/*---------------------------------------------------------------------------*/
static hg_return_t
hg_bulk_complete(struct hg_bulk_op_id *hg_bulk_op_id)
{
hg_context_t *context = hg_bulk_op_id->context;
hg_return_t ret = HG_SUCCESS;
/* Mark operation as completed */
hg_atomic_incr32(&hg_bulk_op_id->completed);
if (hg_bulk_op_id->hg_bulk_origin->eager_mode) {
/* In the case of eager bulk transfer, directly trigger the operation
* to avoid potential deadlocks */
ret = hg_bulk_trigger_entry(hg_bulk_op_id);
if (ret != HG_SUCCESS) {
HG_LOG_ERROR("Could not trigger completion entry");
goto done;
}
} else {
struct hg_completion_entry *hg_completion_entry =
&hg_bulk_op_id->hg_completion_entry;
hg_completion_entry->op_type = HG_BULK;
hg_completion_entry->op_id.hg_bulk_op_id = hg_bulk_op_id;
ret = hg_core_completion_add(context, hg_completion_entry,
hg_bulk_op_id->is_self);
if (ret != HG_SUCCESS) {
HG_LOG_ERROR("Could not add HG completion entry to completion queue");
goto done;
}
}
done:
return ret;
}
/*---------------------------------------------------------------------------*/
hg_return_t
hg_proc_set_buf_ptr(hg_proc_t proc, void *buf_ptr)
{
struct hg_proc *hg_proc = (struct hg_proc *) proc;
ptrdiff_t new_pos, lim_pos;
hg_return_t ret = HG_SUCCESS;
if (!hg_proc) {
HG_LOG_ERROR("Proc is not initialized");
ret = HG_INVALID_PARAM;
goto done;
}
/* Work out new position */
new_pos = (char *) buf_ptr - (char *) hg_proc->current_buf->buf;
lim_pos = (ptrdiff_t) hg_proc->current_buf->size;
if (new_pos > lim_pos) {
HG_LOG_ERROR("Out of memory");
ret = HG_SIZE_ERROR;
return ret;
}
hg_proc->current_buf->buf_ptr = buf_ptr;
hg_proc->current_buf->size_left = hg_proc->current_buf->size -
(hg_size_t) new_pos;
#ifdef HG_HAS_XDR
xdr_setpos(&hg_proc->current_buf->xdr, new_pos);
#endif
done:
return ret;
}
/*---------------------------------------------------------------------------*/
hg_return_t
HG_Bulk_cancel(hg_op_id_t op_id)
{
struct hg_bulk_op_id *hg_bulk_op_id = (struct hg_bulk_op_id *) op_id;
hg_return_t ret = HG_SUCCESS;
if (!hg_bulk_op_id) {
HG_LOG_ERROR("NULL HG bulk operation ID");
ret = HG_INVALID_PARAM;
goto done;
}
if (HG_UTIL_TRUE != hg_atomic_cas32(&hg_bulk_op_id->completed, 1, 0)) {
unsigned int i = 0;
/* Cancel all NA operations issued */
for (i = 0; i < hg_bulk_op_id->op_count; i++) {
na_return_t na_ret;
/* Cancel NA operation */
na_ret = NA_Cancel(HG_Core_class_get_na(hg_bulk_op_id->hg_class),
HG_Core_context_get_na(hg_bulk_op_id->context),
hg_bulk_op_id->na_op_ids[i]);
if (na_ret != NA_SUCCESS) {
HG_LOG_ERROR("Could not cancel op id");
ret = HG_NA_ERROR;
goto done;
}
}
}
done:
return ret;
}
/*---------------------------------------------------------------------------*/
static hg_return_t
hg_test_drc_token_acquire(struct hg_test_info *hg_test_info)
{
hg_return_t ret = HG_SUCCESS;
#ifndef HG_TEST_DRC_IGNORE
int rc;
#endif
/* Acquire credential */
#ifndef HG_TEST_DRC_IGNORE
if (!hg_test_info->credential) {
drc_acquire_again:
rc = drc_acquire(&hg_test_info->credential, 0);
if (rc != DRC_SUCCESS) { /* failed to acquire credential */
if (rc == -DRC_EINVAL) {
sleep(1);
goto drc_acquire_again;
}
HG_LOG_ERROR("drc_acquire() failed (%d, %s)", rc, drc_strerror(-rc));
ret = HG_PROTOCOL_ERROR;
goto done;
}
}
#else
hg_test_info->credential = 12345;
#endif
printf("# Acquired credential %u\n", hg_test_info->credential);
fflush(stdout);
/* Access credential */
#ifndef HG_TEST_DRC_IGNORE
drc_access_again:
rc = drc_access(hg_test_info->credential, 0, &hg_test_info->credential_info);
if (rc != DRC_SUCCESS) { /* failed to access credential */
if (rc == -DRC_EINVAL) {
sleep(1);
goto drc_access_again;
}
HG_LOG_ERROR("drc_access() failed (%d, %s)", rc,
drc_strerror(-rc));
ret = HG_PROTOCOL_ERROR;
goto done;
}
#endif
/* Set cookie for further use */
#ifndef HG_TEST_DRC_IGNORE
hg_test_info->cookie = drc_get_first_cookie(hg_test_info->credential_info);
#else
hg_test_info->cookie = 123456789;
#endif
printf("# Cookie is %u\n", hg_test_info->cookie);
fflush(stdout);
#ifndef HG_TEST_DRC_IGNORE
done:
#endif
return ret;
}
/*---------------------------------------------------------------------------*/
hg_return_t
HG_Bulk_create(hg_class_t *hg_class, hg_uint32_t count, void **buf_ptrs,
const hg_size_t *buf_sizes, hg_uint8_t flags, hg_bulk_t *handle)
{
struct hg_bulk *hg_bulk = NULL;
hg_return_t ret = HG_SUCCESS;
if (!hg_class) {
HG_LOG_ERROR("NULL HG class");
ret = HG_INVALID_PARAM;
goto done;
}
if (!count) {
HG_LOG_ERROR("Invalid number of segments");
ret = HG_INVALID_PARAM;
goto done;
}
if (!buf_sizes) {
HG_LOG_ERROR("NULL segment pointer");
ret = HG_INVALID_PARAM;
goto done;
}
switch (flags) {
case HG_BULK_READWRITE:
break;
case HG_BULK_READ_ONLY:
break;
case HG_BULK_WRITE_ONLY:
break;
default:
HG_LOG_ERROR("Unrecognized handle flag");
ret = HG_INVALID_PARAM;
goto done;
}
ret = hg_bulk_create(hg_class, count, buf_ptrs, buf_sizes, flags, &hg_bulk);
if (ret != HG_SUCCESS) {
HG_LOG_ERROR("Could not create bulk handle");
goto done;
}
*handle = (hg_bulk_t) hg_bulk;
done:
if (ret != HG_SUCCESS) {
hg_bulk_free(hg_bulk);
}
return ret;
}
/*---------------------------------------------------------------------------*/
hg_return_t
hg_test_drc_release(struct hg_test_info *hg_test_info)
{
hg_return_t ret = HG_SUCCESS;
#ifndef HG_TEST_DRC_IGNORE
int rc;
#endif
/* Release the reference to the credential */
#ifndef HG_TEST_DRC_IGNORE
if (hg_test_info->credential_info) {
rc = drc_release_local(&hg_test_info->credential_info);
if (rc != DRC_SUCCESS) { /* failed to release credential info */
HG_LOG_ERROR("Could not release credential info (%d, %s)", rc,
drc_strerror(-rc));
ret = HG_PROTOCOL_ERROR;
goto done;
}
free((void *) hg_test_info->credential_info);
}
if (hg_test_info->wlm_id && hg_test_info->credential) {
rc = drc_revoke(hg_test_info->credential, hg_test_info->wlm_id,
DRC_FLAGS_TARGET_WLM);
if (rc != DRC_SUCCESS) { /* failed to release credential info */
HG_LOG_ERROR("Could not revoke access for %d (%d, %s)",
hg_test_info->wlm_id, rc, drc_strerror(-rc));
ret = HG_PROTOCOL_ERROR;
goto done;
}
}
if (hg_test_info->credential) {
printf("# Releasing credential %u\n", hg_test_info->credential);
rc = drc_release(hg_test_info->credential, 0);
if (rc != DRC_SUCCESS) { /* failed to release credential */
HG_LOG_ERROR("Could not release credential (%d, %s)", rc,
drc_strerror(-rc));
ret = HG_PROTOCOL_ERROR;
goto done;
}
}
done:
#else
(void) hg_test_info;
#endif
return ret;
}
/*---------------------------------------------------------------------------*/
void *
hg_proc_buf_alloc(hg_size_t size)
{
hg_size_t alignment;
void *mem_ptr = NULL;
#ifdef _WIN32
SYSTEM_INFO system_info;
GetSystemInfo (&system_info);
alignment = system_info.dwPageSize;
mem_ptr = _aligned_malloc(size, alignment);
#else
alignment = sysconf(_SC_PAGE_SIZE);
if (posix_memalign(&mem_ptr, alignment, size) != 0) {
HG_LOG_ERROR("posix_memalign failed");
return NULL;
}
#endif
if (mem_ptr) {
memset(mem_ptr, 0, size);
}
return mem_ptr;
}
/*---------------------------------------------------------------------------*/
static int
hg_bulk_transfer_cb(const struct na_cb_info *callback_info)
{
struct hg_bulk_op_id *hg_bulk_op_id =
(struct hg_bulk_op_id *) callback_info->arg;
na_return_t na_ret = NA_SUCCESS;
int ret = 0;
if (callback_info->ret == NA_CANCELED) {
/* If canceled, mark handle as canceled */
hg_atomic_cas32(&hg_bulk_op_id->canceled, 0, 1);
} else if (callback_info->ret != NA_SUCCESS) {
HG_LOG_ERROR("Error in NA callback: %s",
NA_Error_to_string(callback_info->ret));
na_ret = NA_PROTOCOL_ERROR;
goto done;
}
/* When all NA transfers that correspond to bulk operation complete
* add HG user callback to completion queue
*/
if ((unsigned int) hg_atomic_incr32(&hg_bulk_op_id->op_completed_count)
== hg_bulk_op_id->op_count) {
hg_bulk_complete(hg_bulk_op_id);
ret++;
}
done:
(void) na_ret;
return ret;
}
/*---------------------------------------------------------------------------*/
hg_return_t
hg_proc_free(hg_proc_t proc)
{
struct hg_proc *hg_proc = (struct hg_proc *) proc;
hg_return_t ret = HG_SUCCESS;
if (!hg_proc) goto done;
#ifdef HG_HAS_CHECKSUMS
if (hg_proc->proc_buf.checksum != MCHECKSUM_OBJECT_NULL) {
int checksum_ret;
checksum_ret = mchecksum_destroy(hg_proc->proc_buf.checksum);
if (checksum_ret != MCHECKSUM_SUCCESS) {
HG_LOG_ERROR("Could not destroy checksum");
ret = HG_CHECKSUM_ERROR;
}
}
#endif
/* Free extra proc buffer if needed */
if (hg_proc->extra_buf.buf && hg_proc->extra_buf.is_mine) {
free (hg_proc->extra_buf.buf);
hg_proc->extra_buf.buf = NULL;
}
/* Free proc */
free(hg_proc);
hg_proc = NULL;
done:
return ret;
}
/*---------------------------------------------------------------------------*/
hg_return_t
HG_Bulk_access(hg_bulk_t handle, hg_size_t offset, hg_size_t size,
hg_uint8_t flags, hg_uint32_t max_count, void **buf_ptrs,
hg_size_t *buf_sizes, hg_uint32_t *actual_count)
{
struct hg_bulk *hg_bulk = (struct hg_bulk *) handle;
hg_uint32_t count = 0;
hg_return_t ret = HG_SUCCESS;
if (!hg_bulk) {
HG_LOG_ERROR("NULL memory handle passed");
ret = HG_INVALID_PARAM;
goto done;
}
if (!size || !max_count) goto done;
hg_bulk_access(hg_bulk, offset, size, flags, max_count, buf_ptrs,
buf_sizes, &count);
done:
if (ret == HG_SUCCESS) {
if (actual_count) *actual_count = count;
}
return ret;
}
/*---------------------------------------------------------------------------*/
hg_return_t
hg_proc_memcpy(hg_proc_t proc, void *data, hg_size_t data_size)
{
struct hg_proc *hg_proc = (struct hg_proc *) proc;
hg_return_t ret = HG_SUCCESS;
if (!hg_proc) {
HG_LOG_ERROR("Proc is not initialized");
ret = HG_INVALID_PARAM;
goto done;
}
if (hg_proc->op == HG_FREE) goto done;
/* If not enough space allocate extra space if encoding or
* just get extra buffer if decoding */
if (hg_proc->current_buf->size_left < data_size) {
hg_proc_set_size(proc, hg_proc->proc_buf.size +
hg_proc->extra_buf.size + data_size);
}
/* Process data */
hg_proc->current_buf->buf_ptr =
hg_proc_buf_memcpy(hg_proc->current_buf->buf_ptr, data, data_size,
hg_proc->op);
hg_proc->current_buf->size_left -= data_size;
#ifdef HG_HAS_CHECKSUMS
/* Update checksum */
if (hg_proc->current_buf->update_checksum) {
int checksum_ret;
checksum_ret = mchecksum_update(hg_proc->current_buf->checksum, data,
data_size);
if (checksum_ret != MCHECKSUM_SUCCESS) {
HG_LOG_ERROR("Could not update checksum");
ret = HG_CHECKSUM_ERROR;
goto done;
}
}
#endif
done:
return ret;
}
/*---------------------------------------------------------------------------*/
static hg_return_t
hg_test_bulk_transfer_cb(const struct hg_cb_info *hg_cb_info)
{
struct hg_test_bulk_args *bulk_args = (struct hg_test_bulk_args *)
hg_cb_info->arg;
hg_bulk_t local_bulk_handle = hg_cb_info->info.bulk.local_handle;
hg_return_t ret = HG_SUCCESS;
bulk_write_out_t out_struct;
void *buf;
size_t write_ret;
if (hg_cb_info->ret == HG_CANCELED) {
printf("HG_Bulk_transfer() was successfully canceled\n");
/* Fill output structure */
out_struct.ret = 0;
} else if (hg_cb_info->ret != HG_SUCCESS) {
HG_LOG_ERROR("Error in callback");
ret = HG_PROTOCOL_ERROR;
goto done;
}
if (hg_cb_info->ret == HG_SUCCESS) {
/* Call bulk_write */
HG_Bulk_access(local_bulk_handle, 0, bulk_args->nbytes,
HG_BULK_READWRITE, 1, &buf, NULL, NULL);
write_ret = bulk_write(bulk_args->fildes, buf, 0,
bulk_args->nbytes, 1);
/* Fill output structure */
out_struct.ret = write_ret;
}
/* Free block handle */
ret = HG_Bulk_free(local_bulk_handle);
if (ret != HG_SUCCESS) {
fprintf(stderr, "Could not free HG bulk handle\n");
return ret;
}
/* Send response back */
ret = HG_Respond(bulk_args->handle, NULL, NULL, &out_struct);
if (ret != HG_SUCCESS) {
fprintf(stderr, "Could not respond\n");
return ret;
}
done:
HG_Destroy(bulk_args->handle);
free(bulk_args);
return ret;
}
/*---------------------------------------------------------------------------*/
hg_return_t
hg_bulk_trigger_entry(struct hg_bulk_op_id *hg_bulk_op_id)
{
hg_return_t ret = HG_SUCCESS;
/* Execute callback */
if (hg_bulk_op_id->callback) {
struct hg_cb_info hg_cb_info;
hg_cb_info.arg = hg_bulk_op_id->arg;
hg_cb_info.ret =
hg_atomic_get32(&hg_bulk_op_id->canceled) ? HG_CANCELED :
HG_SUCCESS;
hg_cb_info.type = HG_CB_BULK;
hg_cb_info.info.bulk.op = hg_bulk_op_id->op;
hg_cb_info.info.bulk.origin_handle =
(hg_bulk_t) hg_bulk_op_id->hg_bulk_origin;
hg_cb_info.info.bulk.local_handle =
(hg_bulk_t) hg_bulk_op_id->hg_bulk_local;
hg_bulk_op_id->callback(&hg_cb_info);
}
/* Decrement ref_count */
ret = hg_bulk_free(hg_bulk_op_id->hg_bulk_origin);
if (ret != HG_SUCCESS) {
HG_LOG_ERROR("Could not free bulk handle");
goto done;
}
ret = hg_bulk_free(hg_bulk_op_id->hg_bulk_local);
if (ret != HG_SUCCESS) {
HG_LOG_ERROR("Could not free bulk handle");
goto done;
}
/* Free op */
free(hg_bulk_op_id->na_op_ids);
free(hg_bulk_op_id);
done:
return ret;
}
/*---------------------------------------------------------------------------*/
static HG_INLINE hg_return_t
hg_proc_hg_test_drc_grant_out_t(hg_proc_t proc, void *data)
{
hg_test_drc_grant_out_t *struct_data = (hg_test_drc_grant_out_t *) data;
hg_return_t ret = HG_SUCCESS;
#ifdef HG_TEST_DRC_USE_TOKEN
ret = hg_proc_hg_string_t(proc, &struct_data->token);
if (ret != HG_SUCCESS) {
HG_LOG_ERROR("Proc error");
goto done;
}
#else
ret = hg_proc_hg_uint32_t(proc, &struct_data->credential);
if (ret != HG_SUCCESS) {
HG_LOG_ERROR("Proc error");
goto done;
}
#endif
done:
return ret;
}
/*---------------------------------------------------------------------------*/
static HG_INLINE hg_return_t
hg_proc_hg_test_drc_grant_in_t(hg_proc_t proc, void *data)
{
hg_test_drc_grant_in_t *struct_data = (hg_test_drc_grant_in_t *) data;
hg_return_t ret = HG_SUCCESS;
ret = hg_proc_hg_uint32_t(proc, &struct_data->wlm_id);
if (ret != HG_SUCCESS) {
HG_LOG_ERROR("Proc error");
goto done;
}
done:
return ret;
}
/*---------------------------------------------------------------------------*/
static hg_return_t
hg_test_drc_token_request_cb(const struct hg_cb_info *callback_info)
{
hg_request_t *request = (hg_request_t *) callback_info->arg;
hg_return_t ret = HG_SUCCESS;
if (callback_info->ret != HG_SUCCESS) {
HG_LOG_ERROR("Return from callback info is not HG_SUCCESS");
goto done;
}
done:
hg_request_complete(request);
return ret;
}
/*---------------------------------------------------------------------------*/
void *
hg_proc_get_buf_ptr(hg_proc_t proc)
{
struct hg_proc *hg_proc = (struct hg_proc *) proc;
void *ptr = NULL;
if (!hg_proc) {
HG_LOG_ERROR("Proc is not initialized");
goto done;
}
ptr = hg_proc->current_buf->buf_ptr;
done:
return ptr;
}
/*---------------------------------------------------------------------------*/
hg_size_t
hg_proc_get_size_left(hg_proc_t proc)
{
struct hg_proc *hg_proc = (struct hg_proc *) proc;
hg_size_t size = 0;
if (!hg_proc) {
HG_LOG_ERROR("Proc is not initialized");
goto done;
}
size = hg_proc->current_buf->size_left;
done:
return size;
}
/*---------------------------------------------------------------------------*/
hg_size_t
HG_Bulk_get_size(hg_bulk_t handle)
{
hg_size_t ret = 0;
struct hg_bulk *hg_bulk = (struct hg_bulk *) handle;
if (!hg_bulk) {
HG_LOG_ERROR("NULL bulk handle");
goto done;
}
ret = hg_bulk->total_size;
done:
return ret;
}
/*---------------------------------------------------------------------------*/
hg_size_t
hg_proc_get_size(hg_proc_t proc)
{
struct hg_proc *hg_proc = (struct hg_proc *) proc;
hg_size_t size = 0;
if (!hg_proc) {
HG_LOG_ERROR("Proc is not initialized");
goto done;
}
size = hg_proc->proc_buf.size + hg_proc->extra_buf.size;
done:
return size;
}
/*---------------------------------------------------------------------------*/
hg_proc_op_t
hg_proc_get_op(hg_proc_t proc)
{
struct hg_proc *hg_proc = (struct hg_proc *) proc;
hg_proc_op_t proc_op = HG_ENCODE;
if (!hg_proc) {
HG_LOG_ERROR("Proc is not initialized");
goto done;
}
proc_op = hg_proc->op;
done:
return proc_op;
}
/*---------------------------------------------------------------------------*/
hg_class_t *
hg_proc_get_class(hg_proc_t proc)
{
struct hg_proc *hg_proc = (struct hg_proc *) proc;
hg_class_t *hg_class = NULL;
if (!hg_proc) {
HG_LOG_ERROR("Proc is not initialized");
goto done;
}
hg_class = hg_proc->hg_class;
done:
return hg_class;
}
/*---------------------------------------------------------------------------*/
hg_uint32_t
HG_Bulk_get_segment_count(hg_bulk_t handle)
{
struct hg_bulk *hg_bulk = (struct hg_bulk *) handle;
hg_uint32_t ret = 0;
if (!hg_bulk) {
HG_LOG_ERROR("NULL bulk handle");
goto done;
}
ret = hg_bulk->segment_count;
done:
return ret;
}
/**
* Deserialize memcpy
*/
static HG_INLINE hg_return_t
hg_bulk_deserialize_memcpy(const char **src, ssize_t *src_left, void *dest,
size_t n)
{
hg_return_t ret = HG_SUCCESS;
if ((*src_left -= (ssize_t) n) < 0) {
HG_LOG_ERROR("Buffer size too small");
ret = HG_SIZE_ERROR;
goto done;
}
memcpy(dest, *src, n);
*src += n;
done:
return ret;
}
/*---------------------------------------------------------------------------*/
hg_return_t
HG_Bulk_ref_incr(hg_bulk_t handle)
{
struct hg_bulk *hg_bulk = (struct hg_bulk *) handle;
hg_return_t ret = HG_SUCCESS;
if (!hg_bulk) {
HG_LOG_ERROR("NULL memory handle passed");
ret = HG_INVALID_PARAM;
goto done;
}
/* Increment ref count */
hg_atomic_incr32(&hg_bulk->ref_count);
done:
return ret;
}
/*---------------------------------------------------------------------------*/
hg_size_t
hg_proc_get_size_used(hg_proc_t proc)
{
struct hg_proc *hg_proc = (struct hg_proc *) proc;
hg_size_t size = 0;
if (!hg_proc) {
HG_LOG_ERROR("Proc is not initialized");
goto done;
}
if(hg_proc->extra_buf.size > 0)
size = (hg_proc->proc_buf.size + hg_proc->extra_buf.size) - hg_proc->extra_buf.size_left;
else
size = hg_proc->proc_buf.size - hg_proc->proc_buf.size_left;
done:
return size;
}
/*---------------------------------------------------------------------------*/
hg_size_t
HG_Bulk_get_serialize_size(hg_bulk_t handle, hg_bool_t request_eager)
{
struct hg_bulk *hg_bulk = (struct hg_bulk *) handle;
hg_size_t ret = 0;
hg_uint32_t i;
if (!hg_bulk) {
HG_LOG_ERROR("NULL bulk handle");
goto done;
}
/* Permission flags */
ret = sizeof(hg_bulk->flags);
/* Segments */
ret += sizeof(hg_bulk->total_size) + sizeof(hg_bulk->segment_count)
+ hg_bulk->segment_count * sizeof(*hg_bulk->segments);
/* NA mem handles */
ret += sizeof(hg_bulk->na_mem_handle_count);
for (i = 0; i < hg_bulk->na_mem_handle_count; i++) {
na_size_t serialize_size = 0;
if (hg_bulk->na_mem_handles[i]) {
serialize_size = NA_Mem_handle_get_serialize_size(
HG_Core_class_get_na(hg_bulk->hg_class),
hg_bulk->na_mem_handles[i]);
}
ret += sizeof(serialize_size) + serialize_size;
}
/* Eager mode */
ret += sizeof(hg_bulk->eager_mode);
if (request_eager && (hg_bulk->flags == HG_BULK_READ_ONLY))
ret += hg_bulk->total_size;
done:
return ret;
}
/*---------------------------------------------------------------------------*/
hg_return_t
hg_test_drc_acquire(int argc, char *argv[], struct hg_test_info *hg_test_info)
{
struct hg_test_info hg_test_drc_info = { 0 };
struct hg_init_info hg_test_drc_init_info = { 0 };
hg_return_t ret = HG_SUCCESS;
if (!hg_test_info->credential) {
/* Create an NA class with "tcp" protocol */
hg_test_drc_info.na_test_info.extern_init = NA_TRUE;
hg_test_drc_info.na_test_info.protocol = strdup("tcp");
hg_test_drc_info.na_test_info.listen = hg_test_info->na_test_info.listen;
if (NA_Test_init(argc, argv, &hg_test_drc_info.na_test_info) != NA_SUCCESS) {
HG_LOG_ERROR("Could not initialize NA test layer");
ret = HG_NA_ERROR;
goto done;
}
/* Assign NA class */
hg_test_drc_init_info.na_class = hg_test_drc_info.na_test_info.na_class;
/* Init HG HL with init options */
ret = HG_Hl_init_opt(NULL, hg_test_drc_info.na_test_info.listen,
&hg_test_drc_init_info);
if (ret != HG_SUCCESS) {
HG_LOG_ERROR("Could not initialize HG HL");
goto done;
}
hg_test_drc_info.hg_class = HG_CLASS_DEFAULT;
hg_test_drc_info.context = HG_CONTEXT_DEFAULT;
hg_test_drc_info.request_class = HG_REQUEST_CLASS_DEFAULT;
/* Attach test info to class */
HG_Class_set_data(hg_test_drc_info.hg_class, &hg_test_drc_info, NULL);
/* Register routines */
hg_test_drc_register(hg_test_drc_info.hg_class);
/* Acquire DRC token */
if (hg_test_drc_info.na_test_info.listen) {
char addr_string[NA_TEST_MAX_ADDR_NAME];
na_size_t addr_string_len = NA_TEST_MAX_ADDR_NAME;
hg_addr_t self_addr;
ret = hg_test_drc_token_acquire(&hg_test_drc_info);
if (ret != HG_SUCCESS) {
HG_LOG_ERROR("Could not acquire DRC token");
goto done;
}
/* TODO only rank 0 */
ret = HG_Addr_self(hg_test_drc_info.hg_class, &self_addr);
if (ret != HG_SUCCESS) {
HG_LOG_ERROR("Could not get self addr");
goto done;
}
ret = HG_Addr_to_string(hg_test_drc_info.hg_class, addr_string,
&addr_string_len, self_addr);
if (ret != HG_SUCCESS) {
HG_LOG_ERROR("Could not convert addr to string");
goto done;
}
HG_Addr_free(hg_test_drc_info.hg_class, self_addr);
na_test_set_config(addr_string);
/* Used by CTest Test Driver to know when to launch clients */
MERCURY_TESTING_READY_MSG();
/* Progress */
do {
unsigned int total_count = 0;
unsigned int actual_count = 0;
do {
ret = HG_Trigger(hg_test_drc_info.context, 0, 1, &actual_count);
total_count += actual_count;
} while ((ret == HG_SUCCESS) && actual_count);
/* Break as soon as something was triggered */
if (total_count)
break;
ret = HG_Progress(hg_test_drc_info.context, HG_MAX_IDLE_TIME);
} while (ret == HG_SUCCESS || ret == HG_TIMEOUT);
} else {
char test_addr_name[NA_TEST_MAX_ADDR_NAME] = { '\0' };
if (hg_test_drc_info.na_test_info.mpi_comm_rank == 0)
na_test_get_config(test_addr_name, NA_TEST_MAX_ADDR_NAME);
/* Broadcast addr name */
NA_Test_bcast(test_addr_name, NA_TEST_MAX_ADDR_NAME, 0,
&hg_test_drc_info.na_test_info);
hg_test_drc_info.na_test_info.target_name = strdup(test_addr_name);
printf("# Target name read: %s\n",
hg_test_drc_info.na_test_info.target_name);
ret = hg_test_drc_token_request(&hg_test_drc_info);
if (ret != HG_SUCCESS) {
HG_LOG_ERROR("Could not request DRC token");
goto done;
}
}
#ifdef MERCURY_HAS_PARALLEL_TESTING
/* TODO bcast cookie when parallel mode */
#endif
/* Finalize HG HL interface */
ret = HG_Hl_finalize();
if (ret != HG_SUCCESS) {
HG_LOG_ERROR("Could not finalize HG HL");
goto done;
}
/* Finalize NA test class interface */
#ifdef MERCURY_HAS_PARALLEL_TESTING
hg_test_drc_info.na_test_info.mpi_no_finalize = NA_TRUE;
#endif
if (NA_Test_finalize(&hg_test_drc_info.na_test_info) != NA_SUCCESS) {
HG_LOG_ERROR("Could not finalize NA test interface");
ret = HG_NA_ERROR;
goto done;
}
hg_test_info->credential = hg_test_drc_info.credential;
} else {
hg_test_drc_info.credential = hg_test_info->credential;
ret = hg_test_drc_token_acquire(&hg_test_drc_info);
if (ret != HG_SUCCESS) {
HG_LOG_ERROR("Could not acquire DRC token");
goto done;
}
}
/* Copy cookie/credential info */
hg_test_info->wlm_id = hg_test_drc_info.wlm_id;
hg_test_info->credential_info = hg_test_drc_info.credential_info;
hg_test_info->cookie = hg_test_drc_info.cookie;
/* Sleep a few seconds to make sure listener is initialized */
if (!hg_test_drc_info.na_test_info.listen) {
unsigned int sleep_sec = 5;
printf("# Sleeping now for %d seconds...\n", sleep_sec);
fflush(stdout);
sleep(sleep_sec);
}
done:
return ret;
}
/*---------------------------------------------------------------------------*/
static hg_return_t
hg_test_drc_token_request(struct hg_test_info *hg_test_info)
{
hg_request_t *request = NULL;
hg_handle_t handle;
#ifdef HG_TEST_DRC_USE_TOKEN
hg_string_t token;
#else
hg_uint32_t credential;
#endif
hg_test_drc_grant_in_t in_struct;
hg_test_drc_grant_out_t out_struct;
hg_return_t ret = HG_SUCCESS;
#ifndef HG_TEST_DRC_IGNORE
int rc;
#endif
/* Look up target addr using target name info */
ret = HG_Hl_addr_lookup_wait(hg_test_info->context,
hg_test_info->request_class, hg_test_info->na_test_info.target_name,
&hg_test_info->target_addr, HG_MAX_IDLE_TIME);
if (ret != HG_SUCCESS) {
HG_LOG_ERROR("Could not find addr for target %s",
hg_test_info->na_test_info.target_name);
goto done;
}
/* Create new request */
request = hg_request_create(hg_test_info->request_class);
/* Create request with invalid RPC id */
ret = HG_Create(hg_test_info->context, hg_test_info->target_addr,
hg_test_drc_grant_id_g, &handle);
if (ret != HG_SUCCESS) {
HG_LOG_ERROR("Could not create handle");
goto done;
}
/* Get WLM ID and set input */
#ifndef HG_TEST_DRC_IGNORE
in_struct.wlm_id = drc_get_wlm_id();
#else
in_struct.wlm_id = 12340;
#endif
/* Forward call to target addr */
printf("# %u requesting access to remote...\n", in_struct.wlm_id);
fflush(stdout);
ret = HG_Forward(handle, hg_test_drc_token_request_cb, request, &in_struct);
if (ret != HG_SUCCESS) {
HG_LOG_ERROR("Could not forward call with id=%d",
hg_test_drc_grant_id_g);
goto done;
}
/* Wait for completion */
hg_request_wait(request, HG_MAX_IDLE_TIME, NULL);
/* Get output */
ret = HG_Get_output(handle, &out_struct);
if (ret != HG_SUCCESS) {
HG_LOG_ERROR("Could not get output");
goto done;
}
#ifdef HG_TEST_DRC_USE_TOKEN
/* Get token back */
token = out_struct.token;
printf("# Received token %s\n", token);
fflush(stdout);
/* Translate token */
#ifndef HG_TEST_DRC_IGNORE
rc = drc_access_with_token(token, 0, &hg_test_info->credential_info);
if (rc != DRC_SUCCESS) {/* failed to grant access to the credential */
HG_LOG_ERROR("drc_access_with_token() failed (%d, %s)", rc,
drc_strerror(-rc));
ret = HG_PROTOCOL_ERROR;
goto done;
}
#endif
#else
/* Get credential back */
credential = out_struct.credential;
printf("# Received credential %u\n", credential);
fflush(stdout);
/* Access credential */
#ifndef HG_TEST_DRC_IGNORE
drc_access_again:
rc = drc_access(credential, 0, &hg_test_info->credential_info);
if (rc != DRC_SUCCESS) { /* failed to access credential */
if (rc == -DRC_EINVAL) {
sleep(1);
goto drc_access_again;
}
HG_LOG_ERROR("drc_access() failed (%d, %s)", rc,
drc_strerror(-rc));
ret = HG_PROTOCOL_ERROR;
goto done;
}
#endif
#endif
/* Set cookie for further use */
#ifndef HG_TEST_DRC_IGNORE
hg_test_info->cookie = drc_get_first_cookie(hg_test_info->credential_info);
#else
hg_test_info->cookie = 123456789;
#endif
printf("# Cookie is %u\n", hg_test_info->cookie);
fflush(stdout);
/* Clean up resources */
ret = HG_Free_output(handle, &out_struct);
if (ret != HG_SUCCESS) {
HG_LOG_ERROR("Could not free output");
goto done;
}
ret = HG_Destroy(handle);
if (ret != HG_SUCCESS) {
HG_LOG_ERROR("Could not destroy handle");
goto done;
}
hg_request_destroy(request);
/* Free target addr */
ret = HG_Addr_free(hg_test_info->hg_class, hg_test_info->target_addr);
if (ret != HG_SUCCESS) {
HG_LOG_ERROR("Could not free addr");
goto done;
}
done:
return ret;
}
/*---------------------------------------------------------------------------*/
static hg_return_t
hg_test_drc_grant_cb(hg_handle_t handle)
{
const struct hg_info *hg_info = NULL;
struct hg_test_info *hg_test_info = NULL;
hg_test_drc_grant_in_t in_struct;
hg_test_drc_grant_out_t out_struct;
hg_return_t ret = HG_SUCCESS;
#ifdef HG_TEST_DRC_USE_TOKEN
hg_string_t token;
#endif
#ifndef HG_TEST_DRC_IGNORE
int rc;
#endif
/* Get info from handle */
hg_info = HG_Get_info(handle);
/* Get test info */
hg_test_info = (struct hg_test_info *) HG_Class_get_data(hg_info->hg_class);
/* Get input buffer */
ret = HG_Get_input(handle, &in_struct);
if (ret != HG_SUCCESS) {
HG_LOG_ERROR("Could not get input");
goto done;
}
/* Get parameters */
hg_test_info->wlm_id = in_struct.wlm_id;
/* Grant access to another job */
printf("# Granting access to wlm_id %u...\n", hg_test_info->wlm_id);
fflush(stdout);
#ifndef HG_TEST_DRC_IGNORE
drc_grant_again:
rc = drc_grant(hg_test_info->credential, hg_test_info->wlm_id,
DRC_FLAGS_TARGET_WLM);
if (rc != DRC_SUCCESS && rc != -DRC_ALREADY_GRANTED) {
if (rc == -DRC_EINVAL) {
sleep(1);
goto drc_grant_again;
}
HG_LOG_ERROR("drc_grant() to %d failed (%d, %s)", hg_test_info->wlm_id,
rc, drc_strerror(-rc));
ret = HG_PROTOCOL_ERROR;
goto done;
}
#endif
#ifdef HG_TEST_DRC_USE_TOKEN
/* Get the token to pass around to processes in other job */
#ifndef HG_TEST_DRC_IGNORE
rc = drc_get_credential_token(hg_test_info->credential, &token);
if (rc != DRC_SUCCESS) {
HG_LOG_ERROR("drc_get_credential_token() failed (%d, %s)", rc,
drc_strerror(-rc));
ret = HG_PROTOCOL_ERROR;
goto done;
}
#else
token = "my_test_token";
#endif
/* Fill output structure */
printf("# Access granted, token is %s\n", token);
fflush(stdout);
out_struct.token = token;
#else
out_struct.credential = hg_test_info->credential;
#endif
/* Free handle and send response back */
ret = HG_Respond(handle, NULL, NULL, &out_struct);
if (ret != HG_SUCCESS) {
HG_LOG_ERROR("Could not respond");
goto done;
}
HG_Free_input(handle, &in_struct);
HG_Destroy(handle);
done:
return ret;
}
