static int init_mx( MPIDI_PG_t *pg_p ) { mx_endpoint_addr_t local_endpoint_addr; mx_return_t ret; mx_param_t param; int mpi_errno = MPI_SUCCESS; int r; r = MPL_putenv("MX_DISABLE_SHARED=1"); MPIU_ERR_CHKANDJUMP(r, mpi_errno, MPI_ERR_OTHER, "**putenv"); r = MPL_putenv("MX_DISABLE_SELF=1"); MPIU_ERR_CHKANDJUMP(r, mpi_errno, MPI_ERR_OTHER, "**putenv"); ret = mx_init(); MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_init", "**mx_init %s", mx_strerror (ret)); mx_set_error_handler(MX_ERRORS_RETURN); /* ret = mx_get_info(NULL, MX_NIC_COUNT, NULL, 0, &nic_count, sizeof(int)); MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_get_info", "**mx_get_info %s", mx_strerror (ret)); count = ++nic_count; mx_nics = (uint64_t *)MPIU_Malloc(count*sizeof(uint64_t)); ret = mx_get_info(NULL, MX_NIC_IDS, NULL, 0, mx_nics, count*sizeof(uint64_t)); MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_get_info", "**mx_get_info %s", mx_strerror (ret)); do{ ret = mx_nic_id_to_board_number(mx_nics[index],&mx_board_num); index++; }while(ret != MX_SUCCESS); */ #ifndef USE_CTXT_AS_MARK param.key = MX_PARAM_CONTEXT_ID; param.val.context_id.bits = NEM_MX_MATCHING_BITS - SHIFT_TYPE; param.val.context_id.shift = SHIFT_TYPE; ret = mx_open_endpoint(MX_ANY_NIC,MX_ANY_ENDPOINT,MPID_NEM_MX_FILTER,¶m,1,&MPID_nem_mx_local_endpoint); #else ret = mx_open_endpoint(MX_ANY_NIC,MX_ANY_ENDPOINT,MPID_NEM_MX_FILTER,NULL,0,&MPID_nem_mx_local_endpoint); #endif MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_open_endpoint", "**mx_open_endpoint %s", mx_strerror (ret)); ret = mx_get_endpoint_addr(MPID_nem_mx_local_endpoint,&local_endpoint_addr); MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_get_endpoint_addr", "**mx_get_endpoint_addr %s", mx_strerror (ret)); ret = mx_decompose_endpoint_addr(local_endpoint_addr,&MPID_nem_mx_local_nic_id,&MPID_nem_mx_local_endpoint_id); MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_decompose_endpoint_addr", "**mx_decompose_endpoint_addr %s", mx_strerror (ret)); fn_exit: return mpi_errno; fn_fail: goto fn_exit; }
int ompi_mtl_mx_module_init(){ mx_param_t mx_param; mx_return_t mx_return; int32_t nic, ep; /* setup params */ mx_param.key = MX_PARAM_UNEXP_QUEUE_MAX; mx_param.val.unexp_queue_max = ompi_mtl_mx.mx_unexp_queue_max; /* get a local endpoint */ nic = ompi_mtl_mx.mx_board_num; if (nic < 0) { nic = MX_ANY_NIC; } ep = ompi_mtl_mx.mx_endpoint_num; if (ep < 0) { ep = MX_ANY_ENDPOINT; } mx_return = mx_open_endpoint(nic, ep, ompi_mtl_mx.mx_filter, NULL, 0, &ompi_mtl_mx.mx_endpoint); if(mx_return != MX_SUCCESS) { opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_open_endpoint (error %s)\n", mx_strerror(mx_return)); return OMPI_ERROR; } /* get the endpoint address */ mx_return = mx_get_endpoint_addr( ompi_mtl_mx.mx_endpoint, &ompi_mtl_mx.mx_endpoint_addr); if(mx_return != MX_SUCCESS) { opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_get_endpoint_addr (error %s)\n", mx_strerror(mx_return)); return OMPI_ERROR; } mx_return = mx_decompose_endpoint_addr( ompi_mtl_mx.mx_endpoint_addr, &(ompi_mtl_mx.mx_addr.nic_id), &(ompi_mtl_mx.mx_addr.endpoint_id) ); if(mx_return != MX_SUCCESS) { opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_decompose_endpoint_addr (error %s)\n", mx_strerror(mx_return)); return OMPI_ERROR; } opal_output_verbose(10, ompi_mtl_base_framework.framework_output, "mtl:mx: local nic %d, endpoint %d, got nic %d, ep %d\n", nic, ep, (int)ompi_mtl_mx.mx_addr.nic_id, ompi_mtl_mx.mx_addr.endpoint_id); ompi_modex_send( &mca_mtl_mx_component.super.mtl_version, &ompi_mtl_mx.mx_addr, sizeof(mca_mtl_mx_addr_t)); /* register the mtl mx progress function */ opal_progress_register(ompi_mtl_mx_progress); return OMPI_SUCCESS; }
int ompi_mtl_mx_module_init(){ mx_param_t mx_param; mx_return_t mx_return; /* setup params */ mx_param.key = MX_PARAM_UNEXP_QUEUE_MAX; mx_param.val.unexp_queue_max = ompi_mtl_mx.mx_unexp_queue_max; /* get a local endpoint */ mx_return = mx_open_endpoint(MX_ANY_NIC, MX_ANY_ENDPOINT, ompi_mtl_mx.mx_filter, NULL, 0, &ompi_mtl_mx.mx_endpoint); if(mx_return != MX_SUCCESS) { opal_output(ompi_mtl_base_output, "Error in mx_open_endpoint (error %s)\n", mx_strerror(mx_return)); return OMPI_ERROR; } /* get the endpoint address */ mx_return = mx_get_endpoint_addr( ompi_mtl_mx.mx_endpoint, &ompi_mtl_mx.mx_endpoint_addr); if(mx_return != MX_SUCCESS) { opal_output(ompi_mtl_base_output, "Error in mx_get_endpoint_addr (error %s)\n", mx_strerror(mx_return)); return OMPI_ERROR; } mx_return = mx_decompose_endpoint_addr( ompi_mtl_mx.mx_endpoint_addr, &(ompi_mtl_mx.mx_addr.nic_id), &(ompi_mtl_mx.mx_addr.endpoint_id) ); if(mx_return != MX_SUCCESS) { opal_output(ompi_mtl_base_output, "Error in mx_decompose_endpoint_addr (error %s)\n", mx_strerror(mx_return)); return OMPI_ERROR; } ompi_modex_send( &mca_mtl_mx_component.super.mtl_version, &ompi_mtl_mx.mx_addr, sizeof(mca_mtl_mx_addr_t)); /* register the mtl mx progress function */ opal_progress_register(ompi_mtl_mx_progress); return OMPI_SUCCESS; }
int main(int argc, char **argv) { mx_endpoint_t ep; uint64_t nic_id; uint16_t my_eid; uint64_t his_nic_id; uint32_t board_id; uint32_t filter; uint16_t his_eid; mx_endpoint_addr_t his_addr; char *rem_host; int len; int iter; int c; int do_wait; int do_bothways; extern char *optarg; mx_return_t ret; #if DEBUG extern int mx_debug_mask; mx_debug_mask = 0xFFF; #endif mx_init(); MX_MUTEX_INIT(&stream_mutex); /* set up defaults */ rem_host = NULL; filter = FILTER; my_eid = DFLT_EID; his_eid = DFLT_EID; board_id = MX_ANY_NIC; len = DFLT_LEN; iter = DFLT_ITER; do_wait = 0; do_bothways = 0; num_threads = 1; while ((c = getopt(argc, argv, "hd:e:f:n:b:r:l:N:Vvwx")) != EOF) switch(c) { case 'd': rem_host = optarg; break; case 'e': my_eid = atoi(optarg); break; case 'f': filter = atoi(optarg); break; case 'n': sscanf(optarg, "%"SCNx64, &nic_id); mx_nic_id_to_board_number(nic_id, &board_id); break; case 'b': board_id = atoi(optarg); break; case 'r': his_eid = atoi(optarg); break; case 'l': len = atoi(optarg); if (len > MAX_LEN) { fprintf(stderr, "len too large, max is %d\n", MAX_LEN); exit(1); } break; case 'N': iter = atoi(optarg); break; case 'V': Verify = 1; break; case 'v': do_verbose = 1; break; case 'w': do_wait = 1; break; case 'x': #if MX_THREAD_SAFE do_bothways = 1; #else fprintf(stderr, "bi-directional mode only supported with threadsafe mx lib\n"); exit(1); #endif break; case 'h': default: usage(); exit(1); } if (rem_host != NULL) num_threads += do_bothways; ret = mx_open_endpoint(board_id, my_eid, filter, NULL, 0, &ep); if (ret != MX_SUCCESS) { fprintf(stderr, "Failed to open endpoint %s\n", mx_strerror(ret)); exit(1); } /* If no host, we are receiver */ if (rem_host == NULL) { if (do_verbose) printf("Starting streaming receiver\n"); if (Verify) { fprintf(stderr, "-V ignored. Verify must be set by sender\n"); Verify = 0; } if (do_wait) receiver_blocking(ep, MATCH_VAL_MAIN, filter); else receiver_polling(ep, MATCH_VAL_MAIN, filter); } else { /* get address of destination */ mx_hostname_to_nic_id(rem_host, &his_nic_id); mx_connect(ep, his_nic_id, his_eid, filter, MX_INFINITE, &his_addr); if (do_verbose) printf("Starting streaming send to host %s\n", rem_host); if (Verify) printf("Verifying results\n"); /* start up the sender */ if (do_wait) sender_blocking(ep, his_addr, iter, len, do_bothways,MATCH_VAL_MAIN); else sender_polling(ep, his_addr, iter, len, do_bothways, MATCH_VAL_MAIN); } mx_close_endpoint(ep); mx_finalize(); exit(0); }