static int init_mx( MPIDI_PG_t *pg_p )
{
mx_endpoint_addr_t local_endpoint_addr;
mx_return_t ret;
mx_param_t param;
int mpi_errno = MPI_SUCCESS;
int r;
r = MPL_putenv("MX_DISABLE_SHARED=1");
MPIU_ERR_CHKANDJUMP(r, mpi_errno, MPI_ERR_OTHER, "**putenv");
r = MPL_putenv("MX_DISABLE_SELF=1");
MPIU_ERR_CHKANDJUMP(r, mpi_errno, MPI_ERR_OTHER, "**putenv");
ret = mx_init();
MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_init", "**mx_init %s", mx_strerror (ret));
mx_set_error_handler(MX_ERRORS_RETURN);
/*
ret = mx_get_info(NULL, MX_NIC_COUNT, NULL, 0, &nic_count, sizeof(int));
MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_get_info", "**mx_get_info %s", mx_strerror (ret));
count = ++nic_count;
mx_nics = (uint64_t *)MPIU_Malloc(count*sizeof(uint64_t));
ret = mx_get_info(NULL, MX_NIC_IDS, NULL, 0, mx_nics, count*sizeof(uint64_t));
MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_get_info", "**mx_get_info %s", mx_strerror (ret));
do{
ret = mx_nic_id_to_board_number(mx_nics[index],&mx_board_num);
index++;
}while(ret != MX_SUCCESS);
*/
#ifndef USE_CTXT_AS_MARK
param.key = MX_PARAM_CONTEXT_ID;
param.val.context_id.bits = NEM_MX_MATCHING_BITS - SHIFT_TYPE;
param.val.context_id.shift = SHIFT_TYPE;
ret = mx_open_endpoint(MX_ANY_NIC,MX_ANY_ENDPOINT,MPID_NEM_MX_FILTER,¶m,1,&MPID_nem_mx_local_endpoint);
#else
ret = mx_open_endpoint(MX_ANY_NIC,MX_ANY_ENDPOINT,MPID_NEM_MX_FILTER,NULL,0,&MPID_nem_mx_local_endpoint);
#endif
MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_open_endpoint", "**mx_open_endpoint %s", mx_strerror (ret));
ret = mx_get_endpoint_addr(MPID_nem_mx_local_endpoint,&local_endpoint_addr);
MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_get_endpoint_addr", "**mx_get_endpoint_addr %s", mx_strerror (ret));
ret = mx_decompose_endpoint_addr(local_endpoint_addr,&MPID_nem_mx_local_nic_id,&MPID_nem_mx_local_endpoint_id);
MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_decompose_endpoint_addr", "**mx_decompose_endpoint_addr %s", mx_strerror (ret));
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int ompi_mtl_mx_module_init(){
mx_param_t mx_param;
mx_return_t mx_return;
int32_t nic, ep;
/* setup params */
mx_param.key = MX_PARAM_UNEXP_QUEUE_MAX;
mx_param.val.unexp_queue_max = ompi_mtl_mx.mx_unexp_queue_max;
/* get a local endpoint */
nic = ompi_mtl_mx.mx_board_num;
if (nic < 0) {
nic = MX_ANY_NIC;
}
ep = ompi_mtl_mx.mx_endpoint_num;
if (ep < 0) {
ep = MX_ANY_ENDPOINT;
}
mx_return = mx_open_endpoint(nic,
ep,
ompi_mtl_mx.mx_filter,
NULL,
0,
&ompi_mtl_mx.mx_endpoint);
if(mx_return != MX_SUCCESS) {
opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_open_endpoint (error %s)\n", mx_strerror(mx_return));
return OMPI_ERROR;
}
/* get the endpoint address */
mx_return = mx_get_endpoint_addr( ompi_mtl_mx.mx_endpoint,
&ompi_mtl_mx.mx_endpoint_addr);
if(mx_return != MX_SUCCESS) {
opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_get_endpoint_addr (error %s)\n", mx_strerror(mx_return));
return OMPI_ERROR;
}
mx_return = mx_decompose_endpoint_addr( ompi_mtl_mx.mx_endpoint_addr, &(ompi_mtl_mx.mx_addr.nic_id),
&(ompi_mtl_mx.mx_addr.endpoint_id) );
if(mx_return != MX_SUCCESS) {
opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_decompose_endpoint_addr (error %s)\n", mx_strerror(mx_return));
return OMPI_ERROR;
}
opal_output_verbose(10, ompi_mtl_base_framework.framework_output,
"mtl:mx: local nic %d, endpoint %d, got nic %d, ep %d\n", nic, ep,
(int)ompi_mtl_mx.mx_addr.nic_id,
ompi_mtl_mx.mx_addr.endpoint_id);
ompi_modex_send( &mca_mtl_mx_component.super.mtl_version,
&ompi_mtl_mx.mx_addr,
sizeof(mca_mtl_mx_addr_t));
/* register the mtl mx progress function */
opal_progress_register(ompi_mtl_mx_progress);
return OMPI_SUCCESS;
}
int ompi_mtl_mx_module_init(){
mx_param_t mx_param;
mx_return_t mx_return;
/* setup params */
mx_param.key = MX_PARAM_UNEXP_QUEUE_MAX;
mx_param.val.unexp_queue_max = ompi_mtl_mx.mx_unexp_queue_max;
/* get a local endpoint */
mx_return = mx_open_endpoint(MX_ANY_NIC,
MX_ANY_ENDPOINT,
ompi_mtl_mx.mx_filter,
NULL,
0,
&ompi_mtl_mx.mx_endpoint);
if(mx_return != MX_SUCCESS) {
opal_output(ompi_mtl_base_output, "Error in mx_open_endpoint (error %s)\n", mx_strerror(mx_return));
return OMPI_ERROR;
}
/* get the endpoint address */
mx_return = mx_get_endpoint_addr( ompi_mtl_mx.mx_endpoint,
&ompi_mtl_mx.mx_endpoint_addr);
if(mx_return != MX_SUCCESS) {
opal_output(ompi_mtl_base_output, "Error in mx_get_endpoint_addr (error %s)\n", mx_strerror(mx_return));
return OMPI_ERROR;
}
mx_return = mx_decompose_endpoint_addr( ompi_mtl_mx.mx_endpoint_addr, &(ompi_mtl_mx.mx_addr.nic_id),
&(ompi_mtl_mx.mx_addr.endpoint_id) );
if(mx_return != MX_SUCCESS) {
opal_output(ompi_mtl_base_output, "Error in mx_decompose_endpoint_addr (error %s)\n", mx_strerror(mx_return));
return OMPI_ERROR;
}
ompi_modex_send( &mca_mtl_mx_component.super.mtl_version,
&ompi_mtl_mx.mx_addr,
sizeof(mca_mtl_mx_addr_t));
/* register the mtl mx progress function */
opal_progress_register(ompi_mtl_mx_progress);
return OMPI_SUCCESS;
}
int
main(int argc, char **argv)
{
mx_endpoint_t ep;
uint64_t nic_id;
uint16_t my_eid;
uint64_t his_nic_id;
uint32_t board_id;
uint32_t filter;
uint16_t his_eid;
mx_endpoint_addr_t his_addr;
char *rem_host;
int len;
int iter;
int c;
int do_wait;
int do_bothways;
extern char *optarg;
mx_return_t ret;
#if DEBUG
extern int mx_debug_mask;
mx_debug_mask = 0xFFF;
#endif
mx_init();
MX_MUTEX_INIT(&stream_mutex);
/* set up defaults */
rem_host = NULL;
filter = FILTER;
my_eid = DFLT_EID;
his_eid = DFLT_EID;
board_id = MX_ANY_NIC;
len = DFLT_LEN;
iter = DFLT_ITER;
do_wait = 0;
do_bothways = 0;
num_threads = 1;
while ((c = getopt(argc, argv, "hd:e:f:n:b:r:l:N:Vvwx")) != EOF) switch(c) {
case 'd':
rem_host = optarg;
break;
case 'e':
my_eid = atoi(optarg);
break;
case 'f':
filter = atoi(optarg);
break;
case 'n':
sscanf(optarg, "%"SCNx64, &nic_id);
mx_nic_id_to_board_number(nic_id, &board_id);
break;
case 'b':
board_id = atoi(optarg);
break;
case 'r':
his_eid = atoi(optarg);
break;
case 'l':
len = atoi(optarg);
if (len > MAX_LEN) {
fprintf(stderr, "len too large, max is %d\n", MAX_LEN);
exit(1);
}
break;
case 'N':
iter = atoi(optarg);
break;
case 'V':
Verify = 1;
break;
case 'v':
do_verbose = 1;
break;
case 'w':
do_wait = 1;
break;
case 'x':
#if MX_THREAD_SAFE
do_bothways = 1;
#else
fprintf(stderr, "bi-directional mode only supported with threadsafe mx lib\n");
exit(1);
#endif
break;
case 'h':
default:
usage();
exit(1);
}
if (rem_host != NULL)
num_threads += do_bothways;
ret = mx_open_endpoint(board_id, my_eid, filter, NULL, 0, &ep);
if (ret != MX_SUCCESS) {
fprintf(stderr, "Failed to open endpoint %s\n", mx_strerror(ret));
exit(1);
}
/* If no host, we are receiver */
if (rem_host == NULL) {
if (do_verbose)
printf("Starting streaming receiver\n");
if (Verify) {
fprintf(stderr, "-V ignored. Verify must be set by sender\n");
Verify = 0;
}
if (do_wait)
receiver_blocking(ep, MATCH_VAL_MAIN, filter);
else
receiver_polling(ep, MATCH_VAL_MAIN, filter);
} else {
/* get address of destination */
mx_hostname_to_nic_id(rem_host, &his_nic_id);
mx_connect(ep, his_nic_id, his_eid, filter,
MX_INFINITE, &his_addr);
if (do_verbose)
printf("Starting streaming send to host %s\n",
rem_host);
if (Verify) printf("Verifying results\n");
/* start up the sender */
if (do_wait)
sender_blocking(ep, his_addr, iter, len,
do_bothways,MATCH_VAL_MAIN);
else
sender_polling(ep, his_addr, iter, len,
do_bothways, MATCH_VAL_MAIN);
}
mx_close_endpoint(ep);
mx_finalize();
exit(0);
}
