int ompi_common_portals_initialize(ptl_handle_ni_t *ni_handle, bool *accel) { int ret, max_interfaces; ptl_process_id_t ptl_process_id; ptl_interface_t ni_iface = PTL_IFACE_DEFAULT; *accel = false; /* * If we use the YOD launcher we can use the default interface * otherwise we need to use the SeaStar Bridged interface (for CNL/APRUN) */ ni_iface = IFACE_FROM_BRIDGE_AND_NALID(PTL_BRIDGE_UK,PTL_IFACE_SS); /* * Initialize Portals interface */ ret = PtlInit(&max_interfaces); if (PTL_OK != ret) { opal_output(0, "PtlInit failed, returning %d\n", ret); return OMPI_ERR_NOT_AVAILABLE; } /* * Initialize a network device */ ret = PtlNIInit(ni_iface, /* interface to initialize */ PTL_PID_ANY, /* let library assign our pid */ NULL, /* no desired limits */ NULL, /* actual limits */ ni_handle /* our interface handle */ ); if (PTL_OK != ret && PTL_IFACE_DUP != ret) { opal_output(0, "PtlNIInit failed, returning %d (%s : %d)\n", ret, __FILE__, __LINE__); return OMPI_ERROR; } ret = PtlGetId(*ni_handle ,&ptl_process_id); if(PTL_OK != ret) { opal_output(0, "PtlGetId failed, returning %d\n", ret); return OMPI_ERROR; } /* publish my nid/pid info */ ret = ompi_modex_send(&portals_component, &ptl_process_id, sizeof(ptl_process_id_t)); if (OMPI_SUCCESS != ret) { return ret; } return OMPI_SUCCESS; }
int server(void *arg) { int ret; int num_interfaces; ptl_handle_ni_t nih; ptl_handle_eq_t eqh; ptl_ni_limits_t ptl_limits; ptl_event_t ev_t; ptl_event_t *ev=&ev_t; printf("IN SERVER\n"); if ((ret=PtlInit(&num_interfaces)) != PTL_OK) { printf("%s: PtlInit failed: %d\n", __FUNCTION__, ret); exit(1); } printf("%s: PtlInit succeeds (%d:%d)\n", __FUNCTION__, ret, num_interfaces); if (((ret=PtlNIInit( IFACE_FROM_BRIDGE_AND_NALID(PTL_BRIDGE_UK, PTL_IFACE_SS), PTL_PID_ANY, NULL, &ptl_limits, &nih)) != PTL_OK) && (ret != PTL_IFACE_DUP)) { printf("%s: PtlNIInit failed: %d\n", __FUNCTION__, ret); exit(1); } printf("%s: PtlNIInit succeeds (%d)\n", __FUNCTION__, ret); if ((ret=PtlEQAlloc(nih, 4096, NULL, &eqh)) != PTL_OK) { printf("%s: PtlEQAlloc failed: %d\n", __FUNCTION__, ret); exit(1); } iv=11; ret=PtlEQWait(nih,ev); printf("%s: PtlEQAlloc succeeds\n", __FUNCTION__); printf("%d\n",iv); iv=13; while(1); }
int main(int argc, char **argv, char **envp) { int ret; int num_interfaces; ptl_handle_ni_t nih; ptl_handle_eq_t eqh; ptl_ni_limits_t ptl_limits; pid_t child; if ((ret=PtlInit(&num_interfaces)) != PTL_OK) { printf("%s: PtlInit failed: %d\n", FUNCTION_NAME, ret); exit(1); } printf("%s: PtlInit succeeds (%d)\n", FUNCTION_NAME, ret); #ifdef FORK_BEFORE_NI_INIT child = fork(); #endif if ((ret=PtlNIInit( IFACE_FROM_BRIDGE_AND_NALID(PTL_BRIDGE_UK, PTL_IFACE_SS), PTL_PID_ANY, NULL, &ptl_limits, &nih)) != PTL_OK) { printf("%s: PtlNIInit failed: %d\n", FUNCTION_NAME, ret); exit(1); } printf("%s: PtlNIInit succeeds (%d)\n", FUNCTION_NAME, ret); #ifdef FORK_AFTER_NI_INIT child = fork(); #endif if ((ret=PtlEQAlloc(nih, 4096, NULL, &eqh)) != PTL_OK) { printf("%s: PtlEQAlloc failed: %d(%d)\n", FUNCTION_NAME, ret, child); exit(1); } printf("%s: PtlEQAlloc succeeds (%d:%d)\n", FUNCTION_NAME, child, ret); if(child){ MPI_Init(&argc,&argv); MPI_Finalize(); } }
static int component_init(bool enable_progress_threads, bool enable_mpi_threads) { int ret; ptl_ni_limits_t actual; ret = PtlInit(); if (PTL_OK != ret) { opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d: PtlInit failed: %d\n", __FILE__, __LINE__, ret); return OMPI_ERROR; } ret = PtlNIInit(PTL_IFACE_DEFAULT, PTL_NI_PHYSICAL | PTL_NI_MATCHING, PTL_PID_ANY, NULL, &actual, &mca_osc_portals4_component.matching_ni_h); if (PTL_OK != ret) { opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d: PtlNIInit failed: %d\n", __FILE__, __LINE__, ret); return ret; } /* BWB: FIX ME: Need to make sure our ID matches with the MTL... */ mca_osc_portals4_component.matching_atomic_max = actual.max_atomic_size; mca_osc_portals4_component.matching_fetch_atomic_max = actual.max_fetch_atomic_size; mca_osc_portals4_component.matching_atomic_ordered_size = MAX(actual.max_waw_ordered_size, actual.max_war_ordered_size); ret = PtlEQAlloc(mca_osc_portals4_component.matching_ni_h, 4096, &mca_osc_portals4_component.matching_eq_h); if (PTL_OK != ret) { opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d: PtlEQAlloc failed: %d\n", __FILE__, __LINE__, ret); return ret; } ret = PtlPTAlloc(mca_osc_portals4_component.matching_ni_h, 0, mca_osc_portals4_component.matching_eq_h, 4, &mca_osc_portals4_component.matching_pt_idx); if (PTL_OK != ret) { opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d: PtlPTAlloc failed: %d\n", __FILE__, __LINE__, ret); return ret; } OBJ_CONSTRUCT(&mca_osc_portals4_component.requests, opal_free_list_t); ret = opal_free_list_init (&mca_osc_portals4_component.requests, sizeof(ompi_osc_portals4_request_t), opal_cache_line_size, OBJ_CLASS(ompi_osc_portals4_request_t), 0, 0, 8, 0, 8, NULL, 0, NULL, NULL, NULL); if (OMPI_SUCCESS != ret) { opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d: opal_free_list_init failed: %d\n", __FILE__, __LINE__, ret); return ret; } ret = opal_progress_register(progress_callback); if (OMPI_SUCCESS != ret) { opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d: opal_progress_register failed: %d\n", __FILE__, __LINE__, ret); return ret; } return OMPI_SUCCESS; }
int main(int argc, char *argv[]) { ptl_handle_ni_t ni_logical; ptl_pt_index_t logical_pt_index; ptl_process_t myself; struct timeval start, stop; int potato = 0; ENTRY_T potato_catcher; HANDLE_T potato_catcher_handle; ptl_md_t potato_launcher; ptl_handle_md_t potato_launcher_handle; int num_procs; CHECK_RETURNVAL(PtlInit()); CHECK_RETURNVAL(libtest_init()); num_procs = libtest_get_size(); if (NULL != getenv("MAKELEVEL") && num_procs > 2) { return 77; } CHECK_RETURNVAL(PtlNIInit (PTL_IFACE_DEFAULT, NI_TYPE | PTL_NI_LOGICAL, PTL_PID_ANY, NULL, NULL, &ni_logical)); CHECK_RETURNVAL(PtlSetMap(ni_logical, num_procs, libtest_get_mapping(ni_logical))); CHECK_RETURNVAL(PtlGetId(ni_logical, &myself)); CHECK_RETURNVAL(PtlPTAlloc (ni_logical, 0, PTL_EQ_NONE, PTL_PT_ANY, &logical_pt_index)); assert(logical_pt_index == 0); /* Now do the initial setup on ni_logical */ potato_catcher.start = &potato; potato_catcher.length = sizeof(potato); potato_catcher.uid = PTL_UID_ANY; potato_catcher.options = OPTIONS; #if INTERFACE == 1 potato_catcher.match_id.rank = PTL_RANK_ANY; potato_catcher.match_bits = 1; potato_catcher.ignore_bits = ~potato_catcher.match_bits; #endif CHECK_RETURNVAL(PtlCTAlloc(ni_logical, &potato_catcher.ct_handle)); CHECK_RETURNVAL(APPEND (ni_logical, logical_pt_index, &potato_catcher, PTL_PRIORITY_LIST, NULL, &potato_catcher_handle)); /* Now do a barrier (on ni_physical) to make sure that everyone has their * logical interface set up */ libtest_barrier(); /* now I can communicate between ranks with ni_logical */ /* set up the potato launcher */ potato_launcher.start = &potato; potato_launcher.length = sizeof(potato); potato_launcher.options = PTL_MD_EVENT_CT_ACK | PTL_MD_EVENT_CT_SEND; potato_launcher.eq_handle = PTL_EQ_NONE; // i.e. don't queue send events CHECK_RETURNVAL(PtlCTAlloc(ni_logical, &potato_launcher.ct_handle)); CHECK_RETURNVAL(PtlMDBind (ni_logical, &potato_launcher, &potato_launcher_handle)); /* rank 0 starts the potato going */ if (myself.rank == 0) { ptl_process_t nextrank; nextrank.rank = myself.rank + 1; nextrank.rank *= (nextrank.rank <= num_procs - 1); gettimeofday(&start, NULL); CHECK_RETURNVAL(PtlPut(potato_launcher_handle, 0, potato_launcher.length, (LOOPS == 1) ? PTL_OC_ACK_REQ : PTL_NO_ACK_REQ, nextrank, logical_pt_index, 1, 0, NULL, 1)); } { /* the potato-passing loop */ size_t waitfor; ptl_ct_event_t ctc; ptl_process_t nextrank; nextrank.rank = myself.rank + 1; nextrank.rank *= (nextrank.rank <= num_procs - 1); for (waitfor = 1; waitfor <= LOOPS; ++waitfor) { CHECK_RETURNVAL(PtlCTWait(potato_catcher.ct_handle, waitfor, &ctc)); // wait for potato assert(ctc.failure == 0); assert(ctc.success == waitfor); /* I have the potato! */ ++potato; if (potato < LOOPS * (num_procs)) { // otherwise, the recipient may have exited /* Bomb's away! */ if (myself.rank == 0) { CHECK_RETURNVAL(PtlPut(potato_launcher_handle, 0, potato_launcher.length, (waitfor == (LOOPS - 1)) ? PTL_OC_ACK_REQ : PTL_NO_ACK_REQ, nextrank, logical_pt_index, 3, 0, NULL, 2)); } else { CHECK_RETURNVAL(PtlPut(potato_launcher_handle, 0, potato_launcher.length, (waitfor == LOOPS) ? PTL_OC_ACK_REQ : PTL_NO_ACK_REQ, nextrank, logical_pt_index, 3, 0, NULL, 2)); } } } // make sure that last send completed before exiting CHECK_RETURNVAL(PtlCTWait(potato_launcher.ct_handle, LOOPS+1, &ctc)); assert(ctc.failure == 0); } if (myself.rank == 0) { double accumulate = 0.0; gettimeofday(&stop, NULL); accumulate = (stop.tv_sec + stop.tv_usec * 1e-6) - (start.tv_sec + start.tv_usec * 1e-6); /* calculate the average time waiting */ printf("Total time: %g secs\n", accumulate); accumulate /= LOOPS; printf("Average time around the loop: %g microseconds\n", accumulate * 1e6); accumulate /= num_procs; printf("Average catch-to-toss latency: %g microseconds\n", accumulate * 1e6); } /* cleanup */ CHECK_RETURNVAL(PtlMDRelease(potato_launcher_handle)); CHECK_RETURNVAL(PtlCTFree(potato_launcher.ct_handle)); CHECK_RETURNVAL(UNLINK(potato_catcher_handle)); CHECK_RETURNVAL(PtlCTFree(potato_catcher.ct_handle)); /* major cleanup */ CHECK_RETURNVAL(PtlPTFree(ni_logical, logical_pt_index)); CHECK_RETURNVAL(PtlNIFini(ni_logical)); CHECK_RETURNVAL(libtest_fini()); PtlFini(); return 0; }
/* /!\ Called for each processes /!\ */ static int portals4_init_query(bool enable_progress_threads, bool enable_mpi_threads) { int ret; ptl_md_t md; ptl_me_t me; /* Initialize Portals and create a physical, matching interface */ ret = PtlInit(); if (PTL_OK != ret) { opal_output_verbose(1, ompi_coll_base_framework.framework_output, "%s:%d: PtlInit failed: %d\n", __FILE__, __LINE__, ret); return OMPI_ERROR; } ret = PtlNIInit(PTL_IFACE_DEFAULT, PTL_NI_PHYSICAL | PTL_NI_MATCHING, PTL_PID_ANY, NULL, &mca_coll_portals4_component.ni_limits, &mca_coll_portals4_component.ni_h); if (PTL_OK != ret) { opal_output_verbose(1, ompi_coll_base_framework.framework_output, "%s:%d: PtlNIInit failed: %d\n", __FILE__, __LINE__, ret); return OMPI_ERROR; } ret = PtlGetId(mca_coll_portals4_component.ni_h, &mca_coll_portals4_component.id); if (PTL_OK != ret) { opal_output_verbose(1, ompi_coll_base_framework.framework_output, "%s:%d: PtlGetid failed: %d\n", __FILE__, __LINE__, ret); return OMPI_ERROR; } /* FIX ME: Need to make sure our ID matches with the MTL... */ ret = PtlGetUid(mca_coll_portals4_component.ni_h, &mca_coll_portals4_component.uid); if (PTL_OK != ret) { opal_output_verbose(1, ompi_coll_base_framework.framework_output, "%s:%d: PtlGetUid failed: %d\n", __FILE__, __LINE__, ret); return OMPI_ERROR; } ret = PtlEQAlloc(mca_coll_portals4_component.ni_h, MCA_COLL_PORTALS4_EQ_SIZE, &mca_coll_portals4_component.eq_h); if (PTL_OK != ret) { opal_output_verbose(1, ompi_coll_base_framework.framework_output, "%s:%d: PtlEQAlloc failed: %d\n", __FILE__, __LINE__, ret); return OMPI_ERROR; } ret = PtlPTAlloc(mca_coll_portals4_component.ni_h, 0, mca_coll_portals4_component.eq_h, REQ_COLL_TABLE_ID, &mca_coll_portals4_component.pt_idx); if (PTL_OK != ret) { opal_output_verbose(1, ompi_coll_base_framework.framework_output, "%s:%d: PtlPTAlloc failed: %d\n", __FILE__, __LINE__, ret); return OMPI_ERROR; } if (mca_coll_portals4_component.pt_idx != REQ_COLL_TABLE_ID) { opal_output_verbose(1, ompi_coll_base_framework.framework_output, "%s:%d: PtlPTAlloc return wrong pt_idx: %d\n", __FILE__, __LINE__, mca_coll_portals4_component.finish_pt_idx); return OMPI_ERROR; } ret = PtlPTAlloc(mca_coll_portals4_component.ni_h, 0, mca_coll_portals4_component.eq_h, REQ_COLL_FINISH_TABLE_ID, &mca_coll_portals4_component.finish_pt_idx); if (PTL_OK != ret) { opal_output_verbose(1, ompi_coll_base_framework.framework_output, "%s:%d: PtlPTAlloc failed: %d\n", __FILE__, __LINE__, ret); return OMPI_ERROR; } if (mca_coll_portals4_component.finish_pt_idx != REQ_COLL_FINISH_TABLE_ID) { opal_output_verbose(1, ompi_coll_base_framework.framework_output, "%s:%d: PtlPTAlloc return wrong pt_idx: %d\n", __FILE__, __LINE__, mca_coll_portals4_component.finish_pt_idx); return OMPI_ERROR; } /* Bind MD/MDs across all memory. We prefer (for obvious reasons) to have a single MD across all of memory */ memset(&md, 0, sizeof(ptl_md_t)); md.start = 0; md.length = 0; md.options = 0; md.eq_handle = PTL_EQ_NONE; md.ct_handle = PTL_CT_NONE; ret = PtlMDBind(mca_coll_portals4_component.ni_h, &md, &mca_coll_portals4_component.zero_md_h); if (PTL_OK != ret) { opal_output_verbose(1, ompi_coll_base_framework.framework_output, "%s:%d: PtlMDBind failed: %d\n", __FILE__, __LINE__, ret); return OMPI_ERROR; } md.start = 0; md.length = PTL_SIZE_MAX; md.options = 0; md.eq_handle = PTL_EQ_NONE; md.ct_handle = PTL_CT_NONE; ret = PtlMDBind(mca_coll_portals4_component.ni_h, &md, &mca_coll_portals4_component.data_md_h); if (PTL_OK != ret) { opal_output_verbose(1, ompi_coll_base_framework.framework_output, "%s:%d: PtlMDBind failed: %d\n", __FILE__, __LINE__, ret); return OMPI_ERROR; } OPAL_OUTPUT_VERBOSE((90, ompi_coll_base_framework.framework_output, "PtlMDBind start=%p length=%x\n", md.start, md.length)); /* setup finish ack ME */ me.start = NULL; me.length = 0; me.ct_handle = PTL_CT_NONE; me.min_free = 0; me.uid = mca_coll_portals4_component.uid; me.options = PTL_ME_OP_PUT | PTL_ME_EVENT_LINK_DISABLE | PTL_ME_EVENT_UNLINK_DISABLE; me.match_id.phys.nid = PTL_NID_ANY; me.match_id.phys.pid = PTL_PID_ANY; me.match_bits = 0; me.ignore_bits = 0; ret = PtlMEAppend(mca_coll_portals4_component.ni_h, mca_coll_portals4_component.finish_pt_idx, &me, PTL_PRIORITY_LIST, NULL, &mca_coll_portals4_component.finish_me_h); if (PTL_OK != ret) { opal_output_verbose(1, ompi_coll_base_framework.framework_output, "%s:%d: PtlMEAppend of barrier unexpected failed: %d\n", __FILE__, __LINE__, ret); return OMPI_ERROR; } /* This ME is used for RTR exchange only */ me.start = NULL; me.length = 0; me.ct_handle = PTL_CT_NONE; me.min_free = 0; me.uid = mca_coll_portals4_component.uid; me.options = PTL_ME_OP_PUT | PTL_ME_EVENT_SUCCESS_DISABLE | PTL_ME_EVENT_OVER_DISABLE | PTL_ME_EVENT_LINK_DISABLE | PTL_ME_EVENT_UNLINK_DISABLE; me.match_id.phys.nid = PTL_NID_ANY; me.match_id.phys.pid = PTL_PID_ANY; /* Note : the RTR bit must be set to match this ME, * this allows to discriminate the RTR from data flow * (especially for the Barrier operations) */ COLL_PORTALS4_SET_BITS(me.match_bits, 0, 0, 1, 0, 0, 0); me.ignore_bits = ~COLL_PORTALS4_RTR_MASK; ret = PtlMEAppend(mca_coll_portals4_component.ni_h, mca_coll_portals4_component.pt_idx, &me, PTL_OVERFLOW_LIST, NULL, &mca_coll_portals4_component.unex_me_h); if (PTL_OK != ret) { opal_output_verbose(1, ompi_coll_base_framework.framework_output, "%s:%d: PtlMEAppend of barrier unexpected failed: %d\n", __FILE__, __LINE__, ret); return OMPI_ERROR; } /* activate progress callback */ ret = opal_progress_register(portals4_progress); if (OMPI_SUCCESS != ret) { opal_output_verbose(1, ompi_coll_base_framework.framework_output, "%s:%d: opal_progress_register failed: %d\n", __FILE__, __LINE__, ret); return OMPI_ERROR; } return OMPI_SUCCESS; }
int main(int argc, char *argv[]) { ptl_handle_ni_t ni_handle; ptl_process_t *procs; int rank; ptl_pt_index_t pt_index, signal_pt_index; HANDLE_T signal_e_handle; HANDLE_T signal_e2_handle; int num_procs; ptl_handle_eq_t eq_handle; ptl_handle_ct_t ct_handle; ptl_handle_md_t md_handle; ptl_ni_limits_t limits_reqd, limits_actual; ENTRY_T value_e; limits_reqd.max_entries = 1024; limits_reqd.max_unexpected_headers = ITERS*2; limits_reqd.max_mds = 1024; limits_reqd.max_eqs = 1024; limits_reqd.max_cts = 1024; limits_reqd.max_pt_index = 64; limits_reqd.max_iovecs = 1024; limits_reqd.max_list_size = 1024; limits_reqd.max_triggered_ops = 1024; limits_reqd.max_msg_size = 1048576; limits_reqd.max_atomic_size = 1048576; limits_reqd.max_fetch_atomic_size = 1048576; limits_reqd.max_waw_ordered_size = 1048576; limits_reqd.max_war_ordered_size = 1048576; limits_reqd.max_volatile_size = 1048576; limits_reqd.features = 0; CHECK_RETURNVAL(PtlInit()); CHECK_RETURNVAL(libtest_init()); rank = libtest_get_rank(); num_procs = libtest_get_size(); if (num_procs < 2) { fprintf(stderr, "test_flowctl_noeq requires at least two processes\n"); return 77; } int iters; if (num_procs < ITERS) iters = ITERS*2+1; else iters = ITERS; CHECK_RETURNVAL(PtlNIInit(PTL_IFACE_DEFAULT, NI_TYPE | PTL_NI_LOGICAL, PTL_PID_ANY, &limits_reqd, &limits_actual, &ni_handle)); procs = libtest_get_mapping(ni_handle); CHECK_RETURNVAL(PtlSetMap(ni_handle, num_procs, procs)); if (0 == rank) { /* create data PT space */ CHECK_RETURNVAL(PtlEQAlloc(ni_handle, (num_procs - 1) * iters + 64, &eq_handle)); CHECK_RETURNVAL(PtlPTAlloc(ni_handle, PTL_PT_FLOWCTRL, eq_handle, 5, &pt_index)); /* create signal ME */ CHECK_RETURNVAL(PtlCTAlloc(ni_handle, &ct_handle)); CHECK_RETURNVAL(PtlPTAlloc(ni_handle, 1, eq_handle, 6, &signal_pt_index)); value_e.start = NULL; value_e.length = 0; value_e.ct_handle = ct_handle; value_e.uid = PTL_UID_ANY; value_e.options = OPTIONS | PTL_LE_EVENT_CT_COMM; #if INTERFACE == 1 value_e.match_id.rank = PTL_RANK_ANY; value_e.match_bits = 0; value_e.ignore_bits = 0; #endif CHECK_RETURNVAL(APPEND(ni_handle, 5, &value_e, PTL_OVERFLOW_LIST, NULL, &signal_e_handle)); } else { ptl_md_t md; /* 16 extra just in case... */ CHECK_RETURNVAL(PtlEQAlloc(ni_handle, iters*2 + 16, &eq_handle)); md.start = NULL; md.length = 0; md.options = 0; md.eq_handle = eq_handle; md.ct_handle = PTL_CT_NONE; CHECK_RETURNVAL(PtlMDBind(ni_handle, &md, &md_handle)); } fprintf(stderr,"at barrier \n"); libtest_barrier(); if (0 == rank) { ptl_ct_event_t ct; ptl_event_t ev; int ret, count = 0, saw_flowctl = 0; fprintf(stderr,"begin ctwait \n"); /* wait for signal counts */ CHECK_RETURNVAL(PtlCTWait(ct_handle, iters / 2 , &ct)); if (ct.success != iters / 2 || ct.failure != 0) { return 1; } fprintf(stderr,"done CT wait \n"); /* wait for event entries */ while (1) { ret = PtlEQGet(eq_handle, &ev); if (PTL_OK == ret) { count++; fprintf(stderr, "found EQ value \n"); } else if (ret == PTL_EQ_EMPTY) { continue; } else { fprintf(stderr, "0: Unexpected return code from EQGet: %d\n", ret); return 1; } if (ev.type == PTL_EVENT_PT_DISABLED) { saw_flowctl++; break; } } fprintf(stderr, "0: Saw %d flowctl\n", saw_flowctl); if (saw_flowctl == 0) { return 1; } /* Now clear out all of the unexpected messages so we can clean up everything */ CHECK_RETURNVAL(APPEND(ni_handle, 5, &value_e, PTL_PRIORITY_LIST, NULL, &signal_e2_handle)); ret = PTL_OK; while (ret != PTL_EQ_EMPTY) ret = PtlEQGet(eq_handle, &ev); } else { ptl_process_t target; ptl_event_t ev; int ret, count = 0, fails = 0; int i; target.rank = 0; printf("beginning puts \n"); for (i = 0 ; i < iters ; ++i) { CHECK_RETURNVAL(PtlPut(md_handle, 0, 0, PTL_ACK_REQ, target, 5, 0, 0, NULL, 0)); usleep(100); } while (count < iters) { ret = PtlEQGet(eq_handle, &ev); if (PTL_EQ_EMPTY == ret) { continue; } else if (PTL_OK != ret) { fprintf(stderr, "%d: PtlEQGet returned %d\n", rank, ret); return 1; } if (ev.ni_fail_type == PTL_NI_OK) { if (ev.type == PTL_EVENT_SEND) { continue; } else if (ev.type == PTL_EVENT_ACK) { count++; } else { fprintf(stderr, "%d: Unexpected event type %d\n", rank, ev.type); } } else if (ev.ni_fail_type == PTL_NI_PT_DISABLED) { count++; fails++; } else if (ev.ni_fail_type == PTL_EQ_EMPTY) { continue; } else if (ev.ni_fail_type == PTL_EQ_DROPPED) { continue; } else { fprintf(stderr, "%d: Unexpected fail type: %d\n", rank, ev.ni_fail_type); return 1; } } fprintf(stderr, "%d: Saw %d of %d ACKs as fails\n", rank, fails, count); } fprintf(stderr,"at final barrier \n"); libtest_barrier(); if (0 == rank) { CHECK_RETURNVAL(UNLINK(signal_e_handle)); CHECK_RETURNVAL(UNLINK(signal_e2_handle)); CHECK_RETURNVAL(PtlPTFree(ni_handle, signal_pt_index)); CHECK_RETURNVAL(PtlCTFree(ct_handle)); CHECK_RETURNVAL(PtlPTFree(ni_handle, pt_index)); CHECK_RETURNVAL(PtlEQFree(eq_handle)); } else { CHECK_RETURNVAL(PtlMDRelease(md_handle)); CHECK_RETURNVAL(PtlEQFree(eq_handle)); } fprintf(stderr,"final cleanup \n"); CHECK_RETURNVAL(PtlNIFini(ni_handle)); CHECK_RETURNVAL(libtest_fini()); PtlFini(); return 0; }
int main(int argc, char **argv, char **envp) { int i,ret,*npes; int num_interfaces; ptl_handle_ni_t nih; ptl_handle_eq_t eqh; ptl_ni_limits_t ptl_limits; pid_t child; ptl_process_id_t rnk,*procid_map; int spv,*spawned=&spv; if ((ret=PtlInit(&num_interfaces)) != PTL_OK) { printf("%s: PtlInit failed: %d\n", __FUNCTION__, ret); exit(1); } printf("%s: PtlInit succeeds (%d)\n", __FUNCTION__, ret); #ifdef FORK_BEFORE_NI_INIT child = fork(); #endif if((ret=PtlNIInit(IFACE_FROM_BRIDGE_AND_NALID(PTL_BRIDGE_UK, PTL_IFACE_SS), PTL_PID_ANY, NULL, &ptl_limits, &nih)) != PTL_OK) { printf("%s: PtlNIInit failed: %d\n", __FUNCTION__, ret); /*exit(1);*/ } else printf("%s: PtlNIInit succeeds (%d)\n", __FUNCTION__, ret); #ifdef FORK_AFTER_NI_INIT child = fork(); #endif if ((ret=PtlEQAlloc(nih, 4096, NULL, &eqh)) != PTL_OK) { printf("%s: PtlEQAlloc failed: %d(%d)\n", __FUNCTION__, ret, child); exit(1); } printf("%s: PtlEQAlloc succeeds (%d:%d)\n", __FUNCTION__, child, ret); #if 1 if(child){ MPI_Init(&argc,&argv); } if(child){ PMI_Init(spawned); printf("\n%d:spanwned=%d",child,*spawned); if ((ret=PMI_Get_size(npes)) != PMI_SUCCESS) { printf("%s: PMI_Get_size failed: %d\n", __FUNCTION__, ret); /*exit(1);*/ } else printf("%s: PMI_Get_size succeeds (%d)\n", __FUNCTION__,*npes); /*procid_map = (ptl_process_id_t *)malloc(sizeof(ptl_process_id_t)*(*npes)); if(procid_map==NULL)exit(1);*/ if((ret = PMI_CNOS_Get_nidpid_map(&procid_map)) != PMI_SUCCESS) { printf("Getting proc map failed (npes=%d)\n", *npes); } for(i=0;i<*npes;i++){ printf("\npid=%d nid=%d npes=%d(%d)",procid_map[i].pid,procid_map[i].nid,*npes,child); } } #endif if((ret=PtlGetId(nih,&rnk)) !=PTL_OK) { printf("%s: PtlGetId failed: %d(%d)\n", __FUNCTION__, ret, child); exit(1); } printf("%s: nid=%d pid=%d(%d)\n",__FUNCTION__,rnk.nid,rnk.pid,child); if(child){ MPI_Finalize(); printf("%s: mpi_init and finalize succeed(%d)\n",__FUNCTION__,child); } }
int ompi_common_portals_initialize(ptl_handle_ni_t *ni_handle, bool *accel) { int ret; ptl_process_id_t info; if (OPAL_THREAD_ADD32(&usage_count, 1) > 1) return OMPI_SUCCESS; /* if the environment variables for the utcp implementation are already set, assume the user is running without the full Open RTE and is doing RTE testing for a more tightly-coupled platform (like, say, Red Storm). Otherwise, be nice and use the modex to setup everything for the user */ if (NULL == getenv("PTL_MY_RID")) { setup_utcp_params = true; } else { setup_utcp_params = false; } if (setup_utcp_params) { /* Find our contact information and post to registry. Don't initialize Portals until we have everyone's contact information. */ unsigned int nid; p3tcp_my_nid(ptl_ifname, &nid); info.nid = htonl(nid); info.pid = htonl((ptl_pid_t) getpid()); } else { /* Initialize Portals and publish our assigned contact information */ int max_interfaces; unsigned int nptl_procs, rank; ret = PtlInit(&max_interfaces); if (PTL_OK != ret) { opal_output(0, "%5d: PtlInit failed, returning %d\n", getpid(), ret); return OMPI_ERR_NOT_AVAILABLE; } init_called = true; /* tell the UTCP runtime code to read the env variables */ PtlSetRank(PTL_INVALID_HANDLE, -1, -1); /* Initialize a network device */ ret = PtlNIInit(PTL_IFACE_DEFAULT, /* interface to initialize */ PTL_PID_ANY, /* let library assign our pid */ NULL, /* no desired limits */ NULL, /* no need to have limits around */ &active_ni_h /* our interface handle */ ); if (PTL_OK != ret) { opal_output(0, "%5d: PtlNIInit failed, returning %d\n", getpid(), ret); return OMPI_ERR_FATAL; } ret = PtlGetRank(active_ni_h, &rank, &nptl_procs); if (ret != PTL_OK) { opal_output(0, "%5d, PtlGetRank() returned %d", getpid(), ret); return OMPI_ERR_FATAL; } ret = PtlGetRankId(active_ni_h, rank, &info); if (ret != PTL_OK) { opal_output(0, "%5d, PtlGetRank(rank=%d) returned %d", getpid(), rank, ret); return OMPI_ERR_FATAL; } } ret = ompi_modex_send(&portals_component, &info, sizeof(ptl_process_id_t)); if (OMPI_SUCCESS != ret) { return ret; } return OMPI_SUCCESS; }
int ompi_common_portals_ni_initialize(ptl_handle_ni_t *ni_handle, bool *accel) { int ret; *accel = false; OPAL_THREAD_ADD32(&ni_usage_count, 1); if (PTL_INVALID_HANDLE != active_ni_h) { *ni_handle = active_ni_h; return OMPI_SUCCESS; } if (setup_utcp_params) { ompi_proc_t **procs; int my_rid = 0; ptl_process_id_t *info; char *nidmap = NULL, *pidmap = NULL; char *nid_str, *pid_str; size_t map_size = 0; size_t nprocs, size, i; char *tmp; ompi_proc_t* proc_self = ompi_proc_local(); int max_interfaces; /* get our world */ procs = ompi_proc_world(&nprocs); map_size = nprocs * 12 + 1; /* 12 is max length of long in decimal */ nidmap = malloc(map_size); pidmap = malloc(map_size); nid_str = malloc(12 + 1); pid_str = malloc(12 + 1); if (NULL == nidmap || NULL == pidmap || NULL == nid_str || NULL == pid_str) return OMPI_ERROR; for (i = 0 ; i < nprocs ; ++i) { if (proc_self == procs[i]) my_rid = i; ret = ompi_modex_recv(&portals_component, procs[i], (void**) &info, &size); if (OMPI_SUCCESS != ret) { opal_output(0, "%5d: ompi_modex_recv failed: %d", getpid(), ret); return ret; } else if (sizeof(ptl_process_id_t) != size) { opal_output(0, "%5d: ompi_modex_recv returned size %d, expected %d", getpid(), size, sizeof(ptl_process_id_t)); return OMPI_ERROR; } if (i == 0) { snprintf(nidmap, map_size, "%u", ntohl(info->nid)); snprintf(pidmap, map_size, "%u", ntohl(info->pid)); } else { snprintf(nid_str, 12 + 1, ":%u", ntohl(info->nid)); snprintf(pid_str, 12 + 1, ":%u", ntohl(info->pid)); strncat(nidmap, nid_str, 12); strncat(pidmap, pid_str, 12); } free(info); } asprintf(&tmp, "PTL_MY_RID=%u", my_rid); putenv(tmp); asprintf(&tmp, "PTL_NIDMAP=%s", nidmap); putenv(tmp); asprintf(&tmp, "PTL_PIDMAP=%s", pidmap); putenv(tmp); asprintf(&tmp, "PTL_IFACE=%s", ptl_ifname); putenv(tmp); free(pidmap); free(nidmap); free(pid_str); free(nid_str); /* * Initialize Portals */ ret = PtlInit(&max_interfaces); if (PTL_OK != ret) { opal_output(0, "%5d: PtlInit failed, returning %d\n", getpid(), ret); return OMPI_ERR_NOT_AVAILABLE; } init_called = true; /* tell the UTCP runtime code to read the env variables */ PtlSetRank(PTL_INVALID_HANDLE, -1, -1); /* Initialize a network device */ ret = PtlNIInit(PTL_IFACE_DEFAULT, /* interface to initialize */ PTL_PID_ANY, /* let library assign our pid */ NULL, /* no desired limits */ NULL, /* no need to have limits around */ &active_ni_h /* our interface handle */ ); if (PTL_OK != ret) { opal_output(0, "%5d: PtlNIInit failed, returning %d\n", getpid(), ret); return OMPI_ERR_FATAL; } *ni_handle = active_ni_h; return OMPI_SUCCESS; } /* shouldn't ever be able to get here */ return OMPI_ERROR; }
int main(int argc, char **argv, char **envp) { int i,ret,npes; int num_interfaces; ptl_handle_ni_t nih; ptl_handle_eq_t eqh; ptl_ni_limits_t ptl_limits; pid_t child; ptl_process_id_t rnk; child = fork(); if ((ret=PtlInit(&num_interfaces)) != PTL_OK) { printf("%s: PtlInit failed: %d\n", FUNCTION_NAME, ret); exit(1); } printf("%s: PtlInit succeeds (%d)\n", FUNCTION_NAME, ret); if ((ret=PtlNIInit( IFACE_FROM_BRIDGE_AND_NALID(PTL_BRIDGE_UK, PTL_IFACE_SS), PTL_PID_ANY, NULL, &ptl_limits, &nih)) != PTL_OK) { printf("%s: PtlNIInit 1 failed: %d\n", FUNCTION_NAME, ret); } if((ret = PtlNIFini(nih)) !=PTL_OK){ printf("%s: PtlNIFini failed: %d\n", FUNCTION_NAME, ret); } PtlFini(); if ((ret=PtlInit(&num_interfaces)) != PTL_OK) { printf("%s: PtlInit failed: %d\n", FUNCTION_NAME, ret); exit(1); } if ((ret=PtlNIInit( IFACE_FROM_BRIDGE_AND_NALID(PTL_BRIDGE_UK, PTL_IFACE_SS), PTL_PID_ANY, NULL, &ptl_limits, &nih)) != PTL_OK) { printf("%s: PtlNIInit 2 failed: %d\n", FUNCTION_NAME, ret); exit(1); } #if 0 if ((ret=PtlNIInit( IFACE_FROM_BRIDGE_AND_NALID(PTL_BRIDGE_UK, PTL_IFACE_SS), PTL_PID_ANY, NULL, &ptl_limits, &nih)) != PTL_OK) { printf("%s: PtlNIInit failed: %d\n", FUNCTION_NAME, ret); exit(1); } #endif printf("%s: PtlNIInit succeeds (%d)\n", FUNCTION_NAME, ret); if ((ret=PtlEQAlloc(nih, 4096, NULL, &eqh)) != PTL_OK) { printf("%s: PtlEQAlloc failed: %d(%d)\n", FUNCTION_NAME, ret, child); exit(1); } printf("%s: PtlEQAlloc succeeds (%d:%d)\n", FUNCTION_NAME, child, ret); if ((ret=PtlGetId(nih,&rnk)) !=PTL_OK) { printf("%s: PtlGetId failed: %d(%d)\n", FUNCTION_NAME, ret, child); exit(1); } printf("%s: nid=%d pid=%d(%d)\n",FUNCTION_NAME,rnk.nid,rnk.pid,child); if(child){ MPI_Init(&argc,&argv); MPI_Finalize(); printf("%s: mpi_init and finalize succeed(%d)\n",FUNCTION_NAME,child); } }
static int ptl_init(MPIDI_PG_t *pg_p, int pg_rank, char **bc_val_p, int *val_max_sz_p) { int mpi_errno = MPI_SUCCESS; int ret; ptl_md_t md; ptl_ni_limits_t desired; MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_PTL_INIT); MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_PTL_INIT); /* first make sure that our private fields in the vc and req fit into the area provided */ MPIR_Assert(sizeof(MPID_nem_ptl_vc_area) <= MPIDI_NEM_VC_NETMOD_AREA_LEN); MPIR_Assert(sizeof(MPID_nem_ptl_req_area) <= MPIDI_NEM_REQ_NETMOD_AREA_LEN); /* Make sure our IOV is the same as portals4's IOV */ MPIR_Assert(sizeof(ptl_iovec_t) == sizeof(MPL_IOV)); MPIR_Assert(((void*)&(((ptl_iovec_t*)0)->iov_base)) == ((void*)&(((MPL_IOV*)0)->MPL_IOV_BUF))); MPIR_Assert(((void*)&(((ptl_iovec_t*)0)->iov_len)) == ((void*)&(((MPL_IOV*)0)->MPL_IOV_LEN))); MPIR_Assert(sizeof(((ptl_iovec_t*)0)->iov_len) == sizeof(((MPL_IOV*)0)->MPL_IOV_LEN)); mpi_errno = MPIDI_CH3I_Register_anysource_notification(MPID_nem_ptl_anysource_posted, MPID_nem_ptl_anysource_matched); if (mpi_errno) MPIR_ERR_POP(mpi_errno); MPIDI_Anysource_improbe_fn = MPID_nem_ptl_anysource_improbe; /* init portals */ ret = PtlInit(); MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlinit", "**ptlinit %s", MPID_nem_ptl_strerror(ret)); /* do an interface pre-init to get the default limits struct */ ret = PtlNIInit(PTL_IFACE_DEFAULT, PTL_NI_MATCHING | PTL_NI_PHYSICAL, PTL_PID_ANY, NULL, &desired, &MPIDI_nem_ptl_ni); MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlniinit", "**ptlniinit %s", MPID_nem_ptl_strerror(ret)); /* finalize the interface so we can re-init with our desired maximums */ ret = PtlNIFini(MPIDI_nem_ptl_ni); MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlnifini", "**ptlnifini %s", MPID_nem_ptl_strerror(ret)); /* set higher limits if they are determined to be too low */ if (desired.max_unexpected_headers < UNEXPECTED_HDR_COUNT && getenv("PTL_LIM_MAX_UNEXPECTED_HEADERS") == NULL) desired.max_unexpected_headers = UNEXPECTED_HDR_COUNT; if (desired.max_list_size < LIST_SIZE && getenv("PTL_LIM_MAX_LIST_SIZE") == NULL) desired.max_list_size = LIST_SIZE; if (desired.max_entries < ENTRY_COUNT && getenv("PTL_LIM_MAX_ENTRIES") == NULL) desired.max_entries = ENTRY_COUNT; /* do the real init */ ret = PtlNIInit(PTL_IFACE_DEFAULT, PTL_NI_MATCHING | PTL_NI_PHYSICAL, PTL_PID_ANY, &desired, &MPIDI_nem_ptl_ni_limits, &MPIDI_nem_ptl_ni); MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlniinit", "**ptlniinit %s", MPID_nem_ptl_strerror(ret)); /* allocate EQs for each portal */ ret = PtlEQAlloc(MPIDI_nem_ptl_ni, EVENT_COUNT, &MPIDI_nem_ptl_eq); MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptleqalloc", "**ptleqalloc %s", MPID_nem_ptl_strerror(ret)); ret = PtlEQAlloc(MPIDI_nem_ptl_ni, EVENT_COUNT, &MPIDI_nem_ptl_get_eq); MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptleqalloc", "**ptleqalloc %s", MPID_nem_ptl_strerror(ret)); ret = PtlEQAlloc(MPIDI_nem_ptl_ni, EVENT_COUNT, &MPIDI_nem_ptl_control_eq); MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptleqalloc", "**ptleqalloc %s", MPID_nem_ptl_strerror(ret)); ret = PtlEQAlloc(MPIDI_nem_ptl_ni, EVENT_COUNT, &MPIDI_nem_ptl_rpt_eq); MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptleqalloc", "**ptleqalloc %s", MPID_nem_ptl_strerror(ret)); /* allocate a separate EQ for origin events. with this, we can implement rate-limit operations to prevent a locally triggered flow control even */ ret = PtlEQAlloc(MPIDI_nem_ptl_ni, EVENT_COUNT, &MPIDI_nem_ptl_origin_eq); MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptleqalloc", "**ptleqalloc %s", MPID_nem_ptl_strerror(ret)); /* allocate portal for matching messages */ ret = PtlPTAlloc(MPIDI_nem_ptl_ni, PTL_PT_ONLY_USE_ONCE | PTL_PT_ONLY_TRUNCATE | PTL_PT_FLOWCTRL, MPIDI_nem_ptl_eq, PTL_PT_ANY, &MPIDI_nem_ptl_pt); MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlptalloc", "**ptlptalloc %s", MPID_nem_ptl_strerror(ret)); /* allocate portal for large messages where receiver does a get */ ret = PtlPTAlloc(MPIDI_nem_ptl_ni, PTL_PT_ONLY_USE_ONCE | PTL_PT_ONLY_TRUNCATE | PTL_PT_FLOWCTRL, MPIDI_nem_ptl_get_eq, PTL_PT_ANY, &MPIDI_nem_ptl_get_pt); MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlptalloc", "**ptlptalloc %s", MPID_nem_ptl_strerror(ret)); /* allocate portal for MPICH control messages */ ret = PtlPTAlloc(MPIDI_nem_ptl_ni, PTL_PT_ONLY_USE_ONCE | PTL_PT_ONLY_TRUNCATE | PTL_PT_FLOWCTRL, MPIDI_nem_ptl_control_eq, PTL_PT_ANY, &MPIDI_nem_ptl_control_pt); MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlptalloc", "**ptlptalloc %s", MPID_nem_ptl_strerror(ret)); /* allocate portal for MPICH control messages */ ret = PtlPTAlloc(MPIDI_nem_ptl_ni, PTL_PT_ONLY_USE_ONCE | PTL_PT_ONLY_TRUNCATE | PTL_PT_FLOWCTRL, MPIDI_nem_ptl_rpt_eq, PTL_PT_ANY, &MPIDI_nem_ptl_rpt_pt); MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlptalloc", "**ptlptalloc %s", MPID_nem_ptl_strerror(ret)); /* allocate portal for MPICH control messages */ ret = PtlPTAlloc(MPIDI_nem_ptl_ni, PTL_PT_ONLY_USE_ONCE | PTL_PT_ONLY_TRUNCATE | PTL_PT_FLOWCTRL, MPIDI_nem_ptl_rpt_eq, PTL_PT_ANY, &MPIDI_nem_ptl_get_rpt_pt); MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlptalloc", "**ptlptalloc %s", MPID_nem_ptl_strerror(ret)); /* allocate portal for MPICH control messages */ ret = PtlPTAlloc(MPIDI_nem_ptl_ni, PTL_PT_ONLY_USE_ONCE | PTL_PT_ONLY_TRUNCATE | PTL_PT_FLOWCTRL, MPIDI_nem_ptl_rpt_eq, PTL_PT_ANY, &MPIDI_nem_ptl_control_rpt_pt); MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlptalloc", "**ptlptalloc %s", MPID_nem_ptl_strerror(ret)); /* create an MD that covers all of memory */ md.start = 0; md.length = (ptl_size_t)-1; md.options = 0x0; md.eq_handle = MPIDI_nem_ptl_origin_eq; md.ct_handle = PTL_CT_NONE; ret = PtlMDBind(MPIDI_nem_ptl_ni, &md, &MPIDI_nem_ptl_global_md); MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmdbind", "**ptlmdbind %s", MPID_nem_ptl_strerror(ret)); /* currently, rportlas only works with a single NI and EQ */ ret = MPID_nem_ptl_rptl_init(MPIDI_Process.my_pg->size, ORIGIN_EVENTS, get_target_info); MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlniinit", "**ptlniinit %s", MPID_nem_ptl_strerror(ret)); /* allow rportal to manage the primary portal and retransmit if needed */ ret = MPID_nem_ptl_rptl_ptinit(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_origin_eq, MPIDI_nem_ptl_pt, MPIDI_nem_ptl_rpt_pt); MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlptalloc", "**ptlptalloc %s", MPID_nem_ptl_strerror(ret)); /* allow rportal to manage the get and control portals, but we * don't expect retransmission to be needed on the get portal, so * we pass PTL_PT_ANY as the dummy portal. unfortunately, portals * does not have an "invalid" PT constant, which would have been * more appropriate to pass over here. */ ret = MPID_nem_ptl_rptl_ptinit(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_origin_eq, MPIDI_nem_ptl_get_pt, MPIDI_nem_ptl_get_rpt_pt); MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlptalloc", "**ptlptalloc %s", MPID_nem_ptl_strerror(ret)); ret = MPID_nem_ptl_rptl_ptinit(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_origin_eq, MPIDI_nem_ptl_control_pt, MPIDI_nem_ptl_control_rpt_pt); MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlptalloc", "**ptlptalloc %s", MPID_nem_ptl_strerror(ret)); /* create business card */ mpi_errno = get_business_card(pg_rank, bc_val_p, val_max_sz_p); if (mpi_errno) MPIR_ERR_POP(mpi_errno); /* init other modules */ mpi_errno = MPID_nem_ptl_poll_init(); if (mpi_errno) MPIR_ERR_POP(mpi_errno); mpi_errno = MPID_nem_ptl_nm_init(); if (mpi_errno) MPIR_ERR_POP(mpi_errno); fn_exit: MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_PTL_INIT); return mpi_errno; fn_fail: goto fn_exit; }
int main(int argc, char *argv[]) { ptl_handle_ni_t ni_handle; ptl_process_t *procs; int rank; ptl_pt_index_t pt_index, signal_pt_index; HANDLE_T value_e_handle, signal_e_handle; int num_procs; ptl_handle_eq_t eq_handle; ptl_handle_ct_t ct_handle; ptl_handle_md_t md_handle; CHECK_RETURNVAL(PtlInit()); CHECK_RETURNVAL(libtest_init()); rank = libtest_get_rank(); num_procs = libtest_get_size(); if (num_procs < 2) { fprintf(stderr, "test_flowctl_noeq requires at least two processes\n"); return 77; } CHECK_RETURNVAL(PtlNIInit(PTL_IFACE_DEFAULT, NI_TYPE | PTL_NI_LOGICAL, PTL_PID_ANY, NULL, NULL, &ni_handle)); procs = libtest_get_mapping(ni_handle); CHECK_RETURNVAL(PtlSetMap(ni_handle, num_procs, procs)); if (0 == rank) { ENTRY_T value_e; /* create data ME */ CHECK_RETURNVAL(PtlEQAlloc(ni_handle, (num_procs - 1) * ITERS / 2, &eq_handle)); CHECK_RETURNVAL(PtlPTAlloc(ni_handle, PTL_PT_FLOWCTRL, eq_handle, 5, &pt_index)); value_e.start = NULL; value_e.length = 0; value_e.ct_handle = PTL_CT_NONE; value_e.uid = PTL_UID_ANY; value_e.options = OPTIONS; #if INTERFACE == 1 value_e.match_id.rank = PTL_RANK_ANY; value_e.match_bits = 0; value_e.ignore_bits = 0; #endif CHECK_RETURNVAL(APPEND(ni_handle, 5, &value_e, PTL_PRIORITY_LIST, NULL, &value_e_handle)); /* create signal ME */ CHECK_RETURNVAL(PtlCTAlloc(ni_handle, &ct_handle)); CHECK_RETURNVAL(PtlPTAlloc(ni_handle, 0, PTL_EQ_NONE, 6, &signal_pt_index)); value_e.start = NULL; value_e.length = 0; value_e.ct_handle = ct_handle; value_e.uid = PTL_UID_ANY; value_e.options = OPTIONS | PTL_LE_EVENT_SUCCESS_DISABLE | PTL_LE_EVENT_CT_COMM; #if INTERFACE == 1 value_e.match_id.rank = PTL_RANK_ANY; value_e.match_bits = 0; value_e.ignore_bits = 0; #endif CHECK_RETURNVAL(APPEND(ni_handle, 6, &value_e, PTL_PRIORITY_LIST, NULL, &signal_e_handle)); } else { ptl_md_t md; /* 16 extra just in case... */ CHECK_RETURNVAL(PtlEQAlloc(ni_handle, ITERS * 2 + 16, &eq_handle)); md.start = NULL; md.length = 0; md.options = 0; md.eq_handle = eq_handle; md.ct_handle = PTL_CT_NONE; CHECK_RETURNVAL(PtlMDBind(ni_handle, &md, &md_handle)); } libtest_barrier(); if (0 == rank) { ptl_ct_event_t ct; ptl_event_t ev; int ret, count = 0, saw_dropped = 0, saw_flowctl = 0; /* wait for signal counts */ CHECK_RETURNVAL(PtlCTWait(ct_handle, num_procs - 1, &ct)); if (ct.success != num_procs - 1 || ct.failure != 0) { return 1; } /* wait for event entries */ while (count < ITERS * (num_procs - 1)) { ret = PtlEQWait(eq_handle, &ev); if (PTL_OK == ret) { ; } else if (PTL_EQ_DROPPED == ret) { saw_dropped++; if (ev.type == PTL_EVENT_PT_DISABLED){ saw_flowctl++; CHECK_RETURNVAL(PtlPTEnable(ni_handle, pt_index)); } break; } else { fprintf(stderr, "0: Unexpected return code from EQWait: %d\n", ret); return 1; } if (ev.type == PTL_EVENT_PT_DISABLED) { CHECK_RETURNVAL(PtlPTEnable(ni_handle, pt_index)); saw_flowctl++; } else { count++; } } fprintf(stderr, "0: Saw %d dropped, %d flowctl\n", saw_dropped, saw_flowctl); if (saw_flowctl == 0) { return 1; } } else { ptl_process_t target; ptl_event_t ev; int ret, count = 0, fails = 0; int i; int *fail_seen; fail_seen = malloc(sizeof(int) * ITERS); if (NULL == fail_seen) { fprintf(stderr, "%d: malloc failed\n", rank); return 1; } memset(fail_seen, 0, sizeof(int) * ITERS); target.rank = 0; for (i = 0 ; i < ITERS ; ++i) { CHECK_RETURNVAL(PtlPut(md_handle, 0, 0, PTL_ACK_REQ, target, 5, 0, 0, (void*)(size_t)i, 0)); usleep(100); } while (count < ITERS) { ret = PtlEQGet(eq_handle, &ev); if (PTL_EQ_EMPTY == ret) { continue; } else if (PTL_OK != ret) { fprintf(stderr, "%d: PtlEQGet returned %d\n", rank, ret); return 1; } if (ev.ni_fail_type == PTL_NI_OK) { if (ev.type == PTL_EVENT_SEND) { continue; } else if (ev.type == PTL_EVENT_ACK) { count++; } else { fprintf(stderr, "%d: Unexpected event type %d\n", rank, ev.type); } } else if (ev.ni_fail_type == PTL_NI_PT_DISABLED) { int iter = (size_t) ev.user_ptr; if (fail_seen[iter]++ > 0) { fprintf(stderr, "%d: Double report of PT_DISABLED for " "iteration %d\n", rank, iter); return 1; } count++; fails++; } else { fprintf(stderr, "%d: Unexpected fail type: %d\n", rank, ev.ni_fail_type); return 1; } } fprintf(stderr, "%d: Saw %d of %d events as fails\n", rank, fails, count); CHECK_RETURNVAL(PtlPut(md_handle, 0, 0, PTL_NO_ACK_REQ, target, 6, 0, 0, NULL, 0)); /* wait for the send event on the last put */ CHECK_RETURNVAL(PtlEQWait(eq_handle, &ev)); while (fails > 0) { CHECK_RETURNVAL(PtlPut(md_handle, 0, 0, PTL_ACK_REQ, target, 5, 0, 0, NULL, 0)); while (1) { ret = PtlEQWait(eq_handle, &ev); if (PTL_OK != ret) { fprintf(stderr, "%d: PtlEQWait returned %d\n", rank, ret); return 1; } if (ev.ni_fail_type == PTL_NI_OK) { if (ev.type == PTL_EVENT_SEND) { continue; } else if (ev.type == PTL_EVENT_ACK) { fails--; break; } else { fprintf(stderr, "%d: Unexpected event type %d\n", rank, ev.type); } } else if (ev.ni_fail_type == PTL_NI_PT_DISABLED) { break; } else { fprintf(stderr, "%d: Unexpected fail type: %d\n", rank, ev.ni_fail_type); return 1; } } } } libtest_barrier(); if (0 == rank) { CHECK_RETURNVAL(UNLINK(signal_e_handle)); CHECK_RETURNVAL(PtlPTFree(ni_handle, signal_pt_index)); CHECK_RETURNVAL(PtlCTFree(ct_handle)); CHECK_RETURNVAL(UNLINK(value_e_handle)); CHECK_RETURNVAL(PtlPTFree(ni_handle, pt_index)); CHECK_RETURNVAL(PtlEQFree(eq_handle)); } else { CHECK_RETURNVAL(PtlMDRelease(md_handle)); CHECK_RETURNVAL(PtlEQFree(eq_handle)); } CHECK_RETURNVAL(PtlNIFini(ni_handle)); CHECK_RETURNVAL(libtest_fini()); PtlFini(); return 0; }
int shmem_transport_init(long eager_size) { ptl_process_t my_id; int ret; ptl_ni_limits_t ni_req_limits; /* Initialize Portals */ ret = PtlInit(); if (PTL_OK != ret) { fprintf(stderr, "ERROR: PtlInit failed (%d), try setting PTL_IFACE_NAME\n", ret); return 1; } /* Initialize Mutexes */ SHMEM_MUTEX_INIT(shmem_internal_mutex_ptl4_pt_state); SHMEM_MUTEX_INIT(shmem_internal_mutex_ptl4_frag); SHMEM_MUTEX_INIT(shmem_internal_mutex_ptl4_event_slots); SHMEM_MUTEX_INIT(shmem_internal_mutex_ptl4_nb_fence); shmem_transport_portals4_bounce_buffer_size = eager_size; shmem_transport_portals4_bounce_buffers = shmem_free_list_init(sizeof(shmem_transport_portals4_bounce_buffer_t) + eager_size, init_bounce_buffer); shmem_transport_portals4_long_frags = shmem_free_list_init(sizeof(shmem_transport_portals4_long_frag_t), init_long_frag); /* Initialize network */ ni_req_limits.max_entries = 1024; ni_req_limits.max_unexpected_headers = 1024; ni_req_limits.max_mds = 1024; ni_req_limits.max_eqs = 1024; ni_req_limits.max_cts = 1024; ni_req_limits.max_pt_index = 64; ni_req_limits.max_iovecs = 1024; ni_req_limits.max_list_size = 1024; ni_req_limits.max_triggered_ops = 1024; ni_req_limits.max_msg_size = LONG_MAX; ni_req_limits.max_atomic_size = LONG_MAX; ni_req_limits.max_fetch_atomic_size = LONG_MAX; ni_req_limits.max_waw_ordered_size = LONG_MAX; ni_req_limits.max_war_ordered_size = LONG_MAX; ni_req_limits.max_volatile_size = LONG_MAX; ni_req_limits.features = 0; #ifdef ENABLE_REMOTE_VIRTUAL_ADDRESSING ni_req_limits.features |= PTL_TARGET_BIND_INACCESSIBLE; #endif #if WANT_TOTAL_DATA_ORDERING != 0 ni_req_limits.features |= PTL_TOTAL_DATA_ORDERING; #endif ret = PtlNIInit(PTL_IFACE_DEFAULT, PTL_NI_NO_MATCHING | PTL_NI_LOGICAL, PTL_PID_ANY, &ni_req_limits, &ni_limits, &shmem_transport_portals4_ni_h); if (PTL_OK != ret) { fprintf(stderr, "[%03d] ERROR: PtlNIInit failed: %d\n", shmem_internal_my_pe, ret); return ret; } #ifdef ENABLE_REMOTE_VIRTUAL_ADDRESSING if ((PTL_TARGET_BIND_INACCESSIBLE & ni_limits.features) == 0) { fprintf(stderr, "[%03d] ERROR: Remote virtual addressing feature enabled, but Portals\n" "doesn't support PTL_TARGET_BIND_INACCESSIBLE. Aborting.\n", shmem_internal_my_pe); } #endif #if WANT_TOTAL_DATA_ORDERING != 0 if ((PTL_TOTAL_DATA_ORDERING & ni_limits.features) == 0) { if (1 == WANT_TOTAL_DATA_ORDERING) { fprintf(stderr, "[%03d] ERROR: Total data ordering feature enabled, but Portals\n" "doesn't support PTL_TOTAL_DATA_ORDERING. Aborting.\n", shmem_internal_my_pe); } else { shmem_transport_portals4_total_data_ordering = 1; } } #endif /* Note that after this point, one should compare the macro PORTALS4_TOTAL_DATA_ORDERING to 0 /1 to determine if total data ordering is not / is enabled */ ret = PtlGetPhysId(shmem_transport_portals4_ni_h, &my_id); if (PTL_OK != ret) { fprintf(stderr, "[%03d] ERROR: PtlGetPhysId failed: %d\n", shmem_internal_my_pe, ret); return ret; } /* Share information */ ret = shmem_runtime_put("portals4-procid", &my_id, sizeof(my_id)); if (0 != ret) { fprintf(stderr, "[%03d] ERROR: runtime_put failed: %d\n", shmem_internal_my_pe, ret); return ret; } #ifdef ENABLE_REMOTE_VIRTUAL_ADDRESSING /* Make sure the heap and data bases are actually symmetric */ { uint64_t bases[2]; bases[0] = (uintptr_t) shmem_internal_heap_base; bases[1] = (uintptr_t) shmem_internal_data_base; ret = shmem_runtime_put("portals4-bases", bases, sizeof(uint64_t) * 2); if (0 != ret) { fprintf(stderr, "[%03d] ERROR: runtime_put failed: %d\n", shmem_internal_my_pe, ret); return ret; } } #endif return 0; }
int main(int argc, char *argv[]) { ptl_handle_ni_t ni_h; ptl_pt_index_t pt_index; uint64_t *buf; ENTRY_T entry; HANDLE_T entry_h; ptl_md_t md; ptl_handle_md_t md_h; int rank; int num_procs; int ret; ptl_process_t *procs; ptl_handle_eq_t eq_h; ptl_event_t ev; ptl_hdr_data_t rcvd = 0; ptl_hdr_data_t goal = 0; ptl_hdr_data_t hdr_data = 1; ptl_size_t offset = sizeof(uint64_t); uint32_t distance; int sends = 0; CHECK_RETURNVAL(PtlInit()); CHECK_RETURNVAL(libtest_init()); rank = libtest_get_rank(); num_procs = libtest_get_size(); /* This test only succeeds if we have more than one rank */ if (num_procs < 2) return 77; CHECK_RETURNVAL(PtlNIInit(PTL_IFACE_DEFAULT, NI_TYPE | PTL_NI_LOGICAL, PTL_PID_ANY, NULL, NULL, &ni_h)); procs = libtest_get_mapping(ni_h); CHECK_RETURNVAL(PtlSetMap(ni_h, num_procs, procs)); CHECK_RETURNVAL(PtlEQAlloc(ni_h, 1024, &eq_h)); CHECK_RETURNVAL(PtlPTAlloc(ni_h, 0, eq_h, 0, &pt_index)); assert(pt_index == 0); buf = malloc(sizeof(uint64_t) * num_procs); assert(NULL != buf); md.start = buf; md.length = sizeof(uint64_t) * num_procs; md.options = PTL_MD_UNORDERED; md.eq_handle = eq_h; md.ct_handle = PTL_CT_NONE; CHECK_RETURNVAL(PtlMDBind(ni_h, &md, &md_h)); entry.start = buf; entry.length = sizeof(uint64_t) * num_procs; entry.ct_handle = PTL_CT_NONE; entry.uid = PTL_UID_ANY; entry.options = OPTIONS; #if MATCHING == 1 entry.match_id.rank = PTL_RANK_ANY; entry.match_bits = 0; entry.ignore_bits = 0; entry.min_free = 0; #endif CHECK_RETURNVAL(APPEND(ni_h, pt_index, &entry, PTL_PRIORITY_LIST, NULL, &entry_h)); /* ensure ME is linked before the barrier */ CHECK_RETURNVAL(PtlEQWait(eq_h, &ev)); assert( ev.type == PTL_EVENT_LINK ); libtest_barrier(); /* Bruck's Concatenation Algorithm */ memcpy(buf, &rank, sizeof(uint64_t)); for (distance = 1; distance < num_procs; distance *= 2) { ptl_size_t to_xfer; int peer; ptl_process_t proc; if (rank >= distance) { peer = rank - distance; } else { peer = rank + (num_procs - distance); } to_xfer = sizeof(uint64_t) * MIN(distance, num_procs - distance); proc.rank = peer; CHECK_RETURNVAL(PtlPut(md_h, 0, to_xfer, PTL_NO_ACK_REQ, proc, 0, 0, offset, NULL, hdr_data)); sends += 1; /* wait for completion of the proper receive, and keep count of uncompleted sends. "rcvd" is an accumulator to deal with out-of-order receives, which are IDed by the hdr_data */ goal |= hdr_data; while ((rcvd & goal) != goal) { ret = PtlEQWait(eq_h, &ev); switch (ret) { case PTL_OK: if (ev.type == PTL_EVENT_SEND) { sends -= 1; } else { rcvd |= ev.hdr_data; assert(ev.type == PTL_EVENT_PUT); assert(ev.rlength == ev.mlength); assert((ev.rlength == to_xfer) || (ev.hdr_data != hdr_data)); } break; default: fprintf(stderr, "PtlEQWait failure: %d\n", ret); abort(); } } hdr_data <<= 1; offset += to_xfer; } /* wait for any SEND_END events not yet seen */ while (sends) { ret = PtlEQWait(eq_h, &ev); switch (ret) { case PTL_OK: assert( ev.type == PTL_EVENT_SEND ); sends -= 1; break; default: fprintf(stderr, "PtlEQWait failure: %d\n", ret); abort(); } } CHECK_RETURNVAL(UNLINK(entry_h)); CHECK_RETURNVAL(PtlMDRelease(md_h)); free(buf); libtest_barrier(); /* cleanup */ CHECK_RETURNVAL(PtlPTFree(ni_h, pt_index)); CHECK_RETURNVAL(PtlEQFree(eq_h)); CHECK_RETURNVAL(PtlNIFini(ni_h)); CHECK_RETURNVAL(libtest_fini()); PtlFini(); return 0; }
void Init(ArgStruct *p, int* pargc, char*** pargv) { int rc; ptl_pt_index_t pt_handle; /* Initialize Portals and get some runtime info */ rc= PtlInit(); LIBTEST_CHECK(rc, "PtlInit"); libtest_init(); _my_rank= libtest_get_rank(); _nprocs= libtest_get_size(); if (_nprocs < 2) { if (_my_rank == 0) { fprintf(stderr, "Need at least two processes!\n", _my_rank); } exit(-2); } /* ** We need an ni to do barriers and allreduces on. ** It needs to be a non-matching ni. */ rc= PtlNIInit(PTL_IFACE_DEFAULT, PTL_NI_NO_MATCHING | PTL_NI_LOGICAL, PTL_PID_ANY, NULL, NULL, &ni_logical); LIBTEST_CHECK(rc, "PtlNIInit"); rc= PtlSetMap(ni_logical, _nprocs, libtest_get_mapping(ni_logical)); LIBTEST_CHECK(rc, "PtlSetMap"); /* Initialize the barrier in the P4support library. */ libtest_BarrierInit(ni_logical, _my_rank, _nprocs); /* Allocate a Portal Table Index entry for data transmission */ PtlPTAlloc(ni_logical, 0, PTL_EQ_NONE, PTL_XMIT_INDEX, &pt_handle); /* Allocate a Portal Table Index entry to receive an int */ PtlPTAlloc(ni_logical, 0, PTL_EQ_NONE, PTL_SEND_INT_INDEX, &pt_handle); /* Allocate a Portal Table Index entry to receive a double */ PtlPTAlloc(ni_logical, 0, PTL_EQ_NONE, PTL_SEND_DOUBLE_INDEX, &pt_handle); /* Set up the MD to send a single int */ send_int_ct_handle= PTL_INVALID_HANDLE; libtest_CreateMDCT(ni_logical, &send_int, sizeof(int), &send_int_md_handle, &send_int_ct_handle); /* Set up the MD to send a single double */ send_double_ct_handle= PTL_INVALID_HANDLE; libtest_CreateMDCT(ni_logical, &send_double, sizeof(double), &send_double_md_handle, &send_double_ct_handle); /* Create a persistent LE to receive a single int */ recv_int_ct_handle= PTL_INVALID_HANDLE; libtest_CreateLECT(ni_logical, PTL_SEND_INT_INDEX, &recv_int, sizeof(int), &recv_int_le_handle, &recv_int_ct_handle); /* Create a persistent LE to receive a single double */ recv_double_ct_handle= PTL_INVALID_HANDLE; libtest_CreateLECT(ni_logical, PTL_SEND_DOUBLE_INDEX, &recv_double, sizeof(double), &recv_double_le_handle, &recv_double_ct_handle); /* ** Initialize the benchmark data ct handles. Once allocated we'll ** reuse them, instead of reallocating them each time in ** AfterAlignmentInit() */ send_ct_handle= PTL_INVALID_HANDLE; recv_ct_handle= PTL_INVALID_HANDLE; md_handle= PTL_INVALID_HANDLE; md_size= -1; md_buf= NULL; le_handle= PTL_INVALID_HANDLE; le_size= -1; le_buf= NULL; libtest_barrier(); } /* end of Init() */