/*
* delete the entry */
static int unpublish ( char *service_name, ompi_info_t *info )
{
int rc;
#if WANT_PMI2_SUPPORT
if (PMI_SUCCESS != (rc = PMI2_Nameserv_unpublish(service_name, NULL))) {
OMPI_ERROR_LOG(rc);
return OMPI_ERROR;
}
#else
if (PMI_SUCCESS != (rc = PMI_Unpublish_name(service_name))) {
OMPI_ERROR_LOG(rc);
return OMPI_ERROR;
}
#endif
return OMPI_SUCCESS;;
}
static char* lookup ( char *service_name, ompi_info_t *info )
{
char *port=NULL;
int rc;
#if WANT_PMI2_SUPPORT
port = (char*)malloc(1024*sizeof(char)); /* arbitrary size */
if (PMI_SUCCESS != (rc = PMI2_Nameserv_lookup(service_name, NULL, port, 1024))) {
OMPI_ERROR_LOG(rc);
free(port);
return NULL;
}
#else
if (PMI_SUCCESS != (rc = PMI_Lookup_name(service_name, port))) {
OMPI_ERROR_LOG(rc);
return NULL;
}
#endif
return port;
}
int vprotocol_pessimist_event_logger_connect(int el_rank, ompi_communicator_t **el_comm)
{
int rc;
char *port;
int rank;
vprotocol_pessimist_clock_t connect_info[2];
opal_list_t results;
opal_pmix_pdata_t *pdat;
OBJ_CONSTRUCT(&results, opal_list_t);
pdat = OBJ_NEW(opal_pmix_pdata_t);
opal_asprintf(&pdat->value.key, VPROTOCOL_EVENT_LOGGER_NAME_FMT, el_rank);
opal_list_append(&results, &pdat->super);
rc = opal_pmix.lookup(&results, NULL);
if (OPAL_SUCCESS != rc ||
OPAL_STRING != pdat->value.type ||
NULL == pdat->value.data.string) {
OPAL_LIST_DESTRUCT(&results);
return OMPI_ERR_NOT_FOUND;
}
port = strdup(pdat->value.data.string);
OPAL_LIST_DESTRUCT(&results);
V_OUTPUT_VERBOSE(45, "Found port < %s >", port);
rc = ompi_dpm_connect_accept(MPI_COMM_SELF, 0, port, true, el_comm);
if(OMPI_SUCCESS != rc) {
OMPI_ERROR_LOG(rc);
}
/* Send Rank, receive max buffer size and max_clock back */
rank = ompi_comm_rank(&ompi_mpi_comm_world.comm);
rc = mca_pml_v.host_pml.pml_send(&rank, 1, MPI_INTEGER, 0,
VPROTOCOL_PESSIMIST_EVENTLOG_NEW_CLIENT_CMD,
MCA_PML_BASE_SEND_STANDARD,
mca_vprotocol_pessimist.el_comm);
if(OPAL_UNLIKELY(MPI_SUCCESS != rc))
OMPI_ERRHANDLER_INVOKE(mca_vprotocol_pessimist.el_comm, rc,
__FILE__ ": failed sending event logger handshake");
rc = mca_pml_v.host_pml.pml_recv(&connect_info, 2, MPI_UNSIGNED_LONG_LONG,
0, VPROTOCOL_PESSIMIST_EVENTLOG_NEW_CLIENT_CMD,
mca_vprotocol_pessimist.el_comm, MPI_STATUS_IGNORE);
if(OPAL_UNLIKELY(MPI_SUCCESS != rc)) \
OMPI_ERRHANDLER_INVOKE(mca_vprotocol_pessimist.el_comm, rc, \
__FILE__ ": failed receiving event logger handshake");
return rc;
}
int ompi_mpi_finalize(void)
{
int ret;
static int32_t finalize_has_already_started = 0;
opal_list_item_t *item;
struct timeval ompistart, ompistop;
ompi_rte_collective_t *coll;
ompi_proc_t** procs;
size_t nprocs;
/* Be a bit social if an erroneous program calls MPI_FINALIZE in
two different threads, otherwise we may deadlock in
ompi_comm_free() (or run into other nasty lions, tigers, or
bears) */
if (! opal_atomic_cmpset_32(&finalize_has_already_started, 0, 1)) {
/* Note that if we're already finalized, we cannot raise an
MPI exception. The best that we can do is write something
to stderr. */
char hostname[MAXHOSTNAMELEN];
pid_t pid = getpid();
gethostname(hostname, sizeof(hostname));
opal_show_help("help-mpi-runtime.txt",
"mpi_finalize:invoked_multiple_times",
true, hostname, pid);
return MPI_ERR_OTHER;
}
ompi_mpiext_fini();
/* Per MPI-2:4.8, we have to free MPI_COMM_SELF before doing
anything else in MPI_FINALIZE (to include setting up such that
MPI_FINALIZED will return true). */
if (NULL != ompi_mpi_comm_self.comm.c_keyhash) {
ompi_attr_delete_all(COMM_ATTR, &ompi_mpi_comm_self,
ompi_mpi_comm_self.comm.c_keyhash);
OBJ_RELEASE(ompi_mpi_comm_self.comm.c_keyhash);
ompi_mpi_comm_self.comm.c_keyhash = NULL;
}
/* Proceed with MPI_FINALIZE */
ompi_mpi_finalized = true;
/* As finalize is the last legal MPI call, we are allowed to force the release
* of the user buffer used for bsend, before going anywhere further.
*/
(void)mca_pml_base_bsend_detach(NULL, NULL);
nprocs = 0;
procs = ompi_proc_all(&nprocs);
MCA_PML_CALL(del_procs(procs, nprocs));
free(procs);
#if OMPI_ENABLE_PROGRESS_THREADS == 0
opal_progress_set_event_flag(OPAL_EVLOOP_ONCE | OPAL_EVLOOP_NONBLOCK);
#endif
/* Redo ORTE calling opal_progress_event_users_increment() during
MPI lifetime, to get better latency when not using TCP */
opal_progress_event_users_increment();
/* check to see if we want timing information */
if (ompi_enable_timing != 0 && 0 == OMPI_PROC_MY_NAME->vpid) {
gettimeofday(&ompistart, NULL);
}
/* NOTE: MPI-2.1 requires that MPI_FINALIZE is "collective" across
*all* connected processes. This only means that all processes
have to call it. It does *not* mean that all connected
processes need to synchronize (either directly or indirectly).
For example, it is quite easy to construct complicated
scenarios where one job is "connected" to another job via
transitivity, but have no direct knowledge of each other.
Consider the following case: job A spawns job B, and job B
later spawns job C. A "connectedness" graph looks something
like this:
A <--> B <--> C
So what are we *supposed* to do in this case? If job A is
still connected to B when it calls FINALIZE, should it block
until jobs B and C also call FINALIZE?
After lengthy discussions many times over the course of this
project, the issue was finally decided at the Louisville Feb
2009 meeting: no.
Rationale:
- "Collective" does not mean synchronizing. It only means that
every process call it. Hence, in this scenario, every
process in A, B, and C must call FINALIZE.
- KEY POINT: if A calls FINALIZE, then it is erroneous for B or
C to try to communicate with A again.
- Hence, OMPI is *correct* to only effect a barrier across each
jobs' MPI_COMM_WORLD before exiting. Specifically, if A
calls FINALIZE long before B or C, it's *correct* if A exits
at any time (and doesn't notify B or C that it is exiting).
- Arguably, if B or C do try to communicate with the now-gone
A, OMPI should try to print a nice error ("you tried to
communicate with a job that is already gone...") instead of
segv or other Badness. However, that is an *extremely*
difficult problem -- sure, it's easy for A to tell B that it
is finalizing, but how can A tell C? A doesn't even know
about C. You'd need to construct a "connected" graph in a
distributed fashion, which is fraught with race conditions,
etc.
Hence, our conclusion is: OMPI is *correct* in its current
behavior (of only doing a barrier across its own COMM_WORLD)
before exiting. Any problems that occur are as a result of
erroneous MPI applications. We *could* tighten up the erroneous
cases and ensure that we print nice error messages / don't
crash, but that is such a difficult problem that we decided we
have many other, much higher priority issues to handle that deal
with non-erroneous cases. */
/* wait for everyone to reach this point
This is a grpcomm barrier instead of an MPI barrier because an
MPI barrier doesn't ensure that all messages have been transmitted
before exiting, so the possibility of a stranded message exists.
*/
coll = OBJ_NEW(ompi_rte_collective_t);
coll->id = ompi_process_info.peer_fini_barrier;
coll->active = true;
if (OMPI_SUCCESS != (ret = ompi_rte_barrier(coll))) {
OMPI_ERROR_LOG(ret);
return ret;
}
/* wait for barrier to complete */
OMPI_LAZY_WAIT_FOR_COMPLETION(coll->active);
OBJ_RELEASE(coll);
/* check for timing request - get stop time and report elapsed
time if so */
if (ompi_enable_timing && 0 == OMPI_PROC_MY_NAME->vpid) {
gettimeofday(&ompistop, NULL);
opal_output(0, "ompi_mpi_finalize[%ld]: time to execute barrier %ld usec",
(long)OMPI_PROC_MY_NAME->vpid,
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
}
/*
* Shutdown the Checkpoint/Restart Mech.
*/
if (OMPI_SUCCESS != (ret = ompi_cr_finalize())) {
OMPI_ERROR_LOG(ret);
}
/* Shut down any bindings-specific issues: C++, F77, F90 */
/* Remove all memory associated by MPI_REGISTER_DATAREP (per
MPI-2:9.5.3, there is no way for an MPI application to
*un*register datareps, but we don't want the OMPI layer causing
memory leaks). */
while (NULL != (item = opal_list_remove_first(&ompi_registered_datareps))) {
OBJ_RELEASE(item);
}
OBJ_DESTRUCT(&ompi_registered_datareps);
/* Remove all F90 types from the hash tables. As the OBJ_DESTRUCT will
* call a special destructor able to release predefined types, we can
* simply call the OBJ_DESTRUCT on the hash table and all memory will
* be correctly released.
*/
OBJ_DESTRUCT( &ompi_mpi_f90_integer_hashtable );
OBJ_DESTRUCT( &ompi_mpi_f90_real_hashtable );
OBJ_DESTRUCT( &ompi_mpi_f90_complex_hashtable );
/* Free communication objects */
/* free file resources */
if (OMPI_SUCCESS != (ret = ompi_file_finalize())) {
return ret;
}
/* free window resources */
if (OMPI_SUCCESS != (ret = ompi_win_finalize())) {
return ret;
}
if (OMPI_SUCCESS != (ret = ompi_osc_base_finalize())) {
return ret;
}
/* free pml resource */
if(OMPI_SUCCESS != (ret = mca_pml_base_finalize())) {
return ret;
}
/* free communicator resources */
if (OMPI_SUCCESS != (ret = ompi_comm_finalize())) {
return ret;
}
/* free requests */
if (OMPI_SUCCESS != (ret = ompi_request_finalize())) {
return ret;
}
if (OMPI_SUCCESS != (ret = ompi_message_finalize())) {
return ret;
}
/* If requested, print out a list of memory allocated by ALLOC_MEM
but not freed by FREE_MEM */
if (0 != ompi_debug_show_mpi_alloc_mem_leaks) {
mca_mpool_base_tree_print();
}
/* Now that all MPI objects dealing with communications are gone,
shut down MCA types having to do with communications */
if (OMPI_SUCCESS != (ret = mca_base_framework_close(&ompi_pml_base_framework) ) ) {
OMPI_ERROR_LOG(ret);
return ret;
}
/* shut down buffered send code */
mca_pml_base_bsend_fini();
#if OPAL_ENABLE_FT_CR == 1
/*
* Shutdown the CRCP Framework, must happen after PML shutdown
*/
if (OMPI_SUCCESS != (ret = mca_base_framework_close(&ompi_crcp_base_framework) ) ) {
OMPI_ERROR_LOG(ret);
return ret;
}
#endif
/* Free secondary resources */
/* free attr resources */
if (OMPI_SUCCESS != (ret = ompi_attr_finalize())) {
return ret;
}
/* free group resources */
if (OMPI_SUCCESS != (ret = ompi_group_finalize())) {
return ret;
}
/* free proc resources */
if ( OMPI_SUCCESS != (ret = ompi_proc_finalize())) {
return ret;
}
/* finalize the pubsub functions */
if (OMPI_SUCCESS != (ret = mca_base_framework_close(&ompi_pubsub_base_framework) ) ) {
return ret;
}
/* finalize the DPM framework */
if ( OMPI_SUCCESS != (ret = mca_base_framework_close(&ompi_dpm_base_framework))) {
return ret;
}
/* free internal error resources */
if (OMPI_SUCCESS != (ret = ompi_errcode_intern_finalize())) {
return ret;
}
/* free error code resources */
if (OMPI_SUCCESS != (ret = ompi_mpi_errcode_finalize())) {
return ret;
}
/* free errhandler resources */
if (OMPI_SUCCESS != (ret = ompi_errhandler_finalize())) {
return ret;
}
/* Free all other resources */
/* free op resources */
if (OMPI_SUCCESS != (ret = ompi_op_finalize())) {
return ret;
}
/* free ddt resources */
if (OMPI_SUCCESS != (ret = ompi_datatype_finalize())) {
return ret;
}
/* free info resources */
if (OMPI_SUCCESS != (ret = ompi_info_finalize())) {
return ret;
}
/* Close down MCA modules */
/* io is opened lazily, so it's only necessary to close it if it
was actually opened */
if (0 < ompi_io_base_framework.framework_refcnt) {
/* May have been "opened" multiple times. We want it closed now */
ompi_io_base_framework.framework_refcnt = 1;
if (OMPI_SUCCESS != mca_base_framework_close(&ompi_io_base_framework)) {
return ret;
}
}
(void) mca_base_framework_close(&ompi_topo_base_framework);
if (OMPI_SUCCESS != (ret = mca_base_framework_close(&ompi_osc_base_framework))) {
return ret;
}
if (OMPI_SUCCESS != (ret = mca_base_framework_close(&ompi_coll_base_framework))) {
return ret;
}
if (OMPI_SUCCESS != (ret = mca_base_framework_close(&ompi_bml_base_framework))) {
return ret;
}
if (OMPI_SUCCESS != (ret = mca_base_framework_close(&ompi_mpool_base_framework))) {
return ret;
}
if (OMPI_SUCCESS != (ret = mca_base_framework_close(&ompi_rcache_base_framework))) {
return ret;
}
if (OMPI_SUCCESS != (ret = mca_base_framework_close(&ompi_allocator_base_framework))) {
return ret;
}
if (NULL != ompi_mpi_main_thread) {
OBJ_RELEASE(ompi_mpi_main_thread);
ompi_mpi_main_thread = NULL;
}
/* Leave the RTE */
if (OMPI_SUCCESS != (ret = ompi_rte_finalize())) {
return ret;
}
/* now close the rte framework */
if (OMPI_SUCCESS != (ret = mca_base_framework_close(&ompi_rte_base_framework) ) ) {
OMPI_ERROR_LOG(ret);
return ret;
}
if (OPAL_SUCCESS != (ret = opal_finalize_util())) {
return ret;
}
/* All done */
return MPI_SUCCESS;
}
/*
* send connect information to remote endpoint
*/
static int xoob_send_connect_data(mca_btl_base_endpoint_t* endpoint,
uint8_t message_type)
{
opal_buffer_t* buffer = OBJ_NEW(opal_buffer_t);
int rc, srq;
if (NULL == buffer) {
OMPI_ERROR_LOG(OMPI_ERR_OUT_OF_RESOURCE);
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* Bulding standart header that we use in all messages:
* - Message type,
* - Our subnet id
* - Our LID
*/
/* pack the info in the send buffer */
BTL_VERBOSE(("Send pack Message type = %d", message_type));
BTL_VERBOSE(("packing %d of %d\n", 1, OPAL_UINT8));
rc = opal_dss.pack(buffer, &message_type, 1, OPAL_UINT8);
if (OPAL_SUCCESS != rc) {
OMPI_ERROR_LOG(rc);
return rc;
}
BTL_VERBOSE(("Send pack sid = %" PRIx64 "\n", endpoint->subnet_id));
BTL_VERBOSE(("packing %d of %d\n", 1, OPAL_UINT64));
rc = opal_dss.pack(buffer, &endpoint->subnet_id, 1, OPAL_UINT64);
if (OPAL_SUCCESS != rc) {
OMPI_ERROR_LOG(rc);
return rc;
}
BTL_VERBOSE(("Send pack lid = %d", endpoint->endpoint_btl->lid));
BTL_VERBOSE(("packing %d of %d\n", 1, OPAL_UINT16));
rc = opal_dss.pack(buffer, &endpoint->endpoint_btl->lid, 1, OPAL_UINT16);
if (OPAL_SUCCESS != rc) {
OMPI_ERROR_LOG(rc);
return rc;
}
/* Now we append to standart header additional information
* that is required for full (open qp,etc..) connect request and response:
* - qp_num of first qp
* - psn of first qp
* - MTU
*/
if (ENDPOINT_XOOB_CONNECT_REQUEST == message_type ||
ENDPOINT_XOOB_CONNECT_RESPONSE == message_type) {
uint32_t psn, qp_num;
if (ENDPOINT_XOOB_CONNECT_REQUEST == message_type) {
qp_num = endpoint->qps[0].qp->lcl_qp->qp_num;
psn = endpoint->qps[0].qp->lcl_psn;
} else {
qp_num = endpoint->xrc_recv_qp_num;
psn = endpoint->xrc_recv_psn;
}
/* stuff all the QP info into the buffer */
/* we need to send only one QP */
BTL_VERBOSE(("Send pack qp num = %x", qp_num));
BTL_VERBOSE(("packing %d of %d\n", 1, OPAL_UINT32));
rc = opal_dss.pack(buffer, &qp_num, 1, OPAL_UINT32);
if (OPAL_SUCCESS != rc) {
OMPI_ERROR_LOG(rc);
return rc;
}
BTL_VERBOSE(("Send pack lpsn = %d", psn));
BTL_VERBOSE(("packing %d of %d\n", 1, OPAL_UINT32));
rc = opal_dss.pack(buffer, &psn, 1, OPAL_UINT32);
if (OPAL_SUCCESS != rc) {
OMPI_ERROR_LOG(rc);
return rc;
}
BTL_VERBOSE(("Send pack mtu = %d", endpoint->endpoint_btl->device->mtu));
BTL_VERBOSE(("packing %d of %d\n", 1, OPAL_UINT32));
rc = opal_dss.pack(buffer, &endpoint->endpoint_btl->device->mtu, 1,
OPAL_UINT32);
if (OPAL_SUCCESS != rc) {
OMPI_ERROR_LOG(rc);
return rc;
}
}
/* We append to header above additional information
* that is required for full & XRC connect request:
* - The lid ob btl on remote site that we want to connect
*/
if (ENDPOINT_XOOB_CONNECT_REQUEST == message_type ||
ENDPOINT_XOOB_CONNECT_XRC_REQUEST == message_type) {
/* when we are sending request we add remote lid that we want to connect */
BTL_VERBOSE(("Send pack remote lid = %d", endpoint->ib_addr->lid));
BTL_VERBOSE(("packing %d of %d\n", 1, OPAL_UINT16));
rc = opal_dss.pack(buffer, &endpoint->ib_addr->lid, 1, OPAL_UINT16);
if (OPAL_SUCCESS != rc) {
OMPI_ERROR_LOG(rc);
return rc;
}
}
/* when we are sending xrc request we add remote
* recv qp number that we want to connect. */
if (ENDPOINT_XOOB_CONNECT_XRC_REQUEST == message_type) {
BTL_VERBOSE(("Send pack remote qp = %x", endpoint->ib_addr->remote_xrc_rcv_qp_num));
BTL_VERBOSE(("packing %d of %d\n", 1, OPAL_UINT32));
rc = opal_dss.pack(buffer, &endpoint->ib_addr->remote_xrc_rcv_qp_num,
1, OPAL_UINT32);
if (OPAL_SUCCESS != rc) {
OMPI_ERROR_LOG(rc);
return rc;
}
}
/* We append to header above additional information
* that is required for full & XRC connect response:
* - index of our endpoint
* - array of xrc-srq numbers
*/
if (ENDPOINT_XOOB_CONNECT_RESPONSE == message_type ||
ENDPOINT_XOOB_CONNECT_XRC_RESPONSE == message_type) {
/* we need to send the endpoint index for immidate send */
BTL_VERBOSE(("Send pack index = %d", endpoint->index));
BTL_VERBOSE(("packing %d of %d\n", 1, OPAL_UINT32));
rc = opal_dss.pack(buffer, &endpoint->index, 1, OPAL_UINT32);
if (OPAL_SUCCESS != rc) {
OMPI_ERROR_LOG(rc);
return rc;
}
/* on response we add all SRQ numbers */
for (srq = 0; srq < mca_btl_openib_component.num_xrc_qps; srq++) {
BTL_VERBOSE(("Send pack srq[%d] num = %d", srq, endpoint->endpoint_btl->qps[srq].u.srq_qp.srq->xrc_srq_num));
BTL_VERBOSE(("packing %d of %d\n", 1, OPAL_UINT32));
rc = opal_dss.pack(buffer, &endpoint->endpoint_btl->qps[srq].u.srq_qp.srq->xrc_srq_num,
1, OPAL_UINT32);
if (OPAL_SUCCESS != rc) {
OMPI_ERROR_LOG(rc);
return rc;
}
}
}
/* send to remote endpoint */
rc = ompi_rte_send_buffer_nb(&endpoint->endpoint_proc->proc_ompi->proc_name,
buffer, OMPI_RML_TAG_XOPENIB,
xoob_rml_send_cb, NULL);
if (OMPI_SUCCESS != rc) {
OMPI_ERROR_LOG(rc);
return rc;
}
BTL_VERBOSE(("Send QP Info, LID = %d, SUBNET = %" PRIx64 ", Message type = %d",
endpoint->endpoint_btl->lid,
endpoint->subnet_id,
message_type));
return OMPI_SUCCESS;
}
/* Receive connect information to remote endpoint */
static int xoob_receive_connect_data(mca_btl_openib_rem_info_t *info, uint16_t *lid,
uint8_t *message_type, opal_buffer_t* buffer)
{
int cnt = 1, rc, srq;
/* Recv standart header */
BTL_VERBOSE(("unpacking %d of %d\n", cnt, OPAL_UINT8));
rc = opal_dss.unpack(buffer, message_type, &cnt, OPAL_UINT8);
if (OPAL_SUCCESS != rc) {
OMPI_ERROR_LOG(rc);
return OMPI_ERROR;
}
BTL_VERBOSE(("Recv unpack Message type = %d\n", *message_type));
BTL_VERBOSE(("unpacking %d of %d\n", cnt, OPAL_UINT64));
rc = opal_dss.unpack(buffer, &info->rem_subnet_id, &cnt, OPAL_UINT64);
if (OPAL_SUCCESS != rc) {
OMPI_ERROR_LOG(rc);
return OMPI_ERROR;
}
BTL_VERBOSE(("Recv unpack sid = %" PRIx64 "\n", info->rem_subnet_id));
BTL_VERBOSE(("unpacking %d of %d\n", cnt, OPAL_UINT16));
rc = opal_dss.unpack(buffer, &info->rem_lid, &cnt, OPAL_UINT16);
if (OPAL_SUCCESS != rc) {
OMPI_ERROR_LOG(rc);
return OMPI_ERROR;
}
BTL_VERBOSE(("Recv unpack lid = %d", info->rem_lid));
/* Till now we got the standart header, now we continue to recieve data for
* different packet types
*/
if (ENDPOINT_XOOB_CONNECT_REQUEST == *message_type ||
ENDPOINT_XOOB_CONNECT_RESPONSE == *message_type) {
BTL_VERBOSE(("unpacking %d of %d\n", cnt, OPAL_UINT32));
rc = opal_dss.unpack(buffer, &info->rem_qps->rem_qp_num, &cnt,
OPAL_UINT32);
if (OPAL_SUCCESS != rc) {
OMPI_ERROR_LOG(rc);
return OMPI_ERROR;
}
BTL_VERBOSE(("Recv unpack remote qp = %x", info->rem_qps->rem_qp_num));
BTL_VERBOSE(("unpacking %d of %d\n", cnt, OPAL_UINT32));
rc = opal_dss.unpack(buffer, &info->rem_qps->rem_psn, &cnt,
OPAL_UINT32);
if (OPAL_SUCCESS != rc) {
OMPI_ERROR_LOG(rc);
return OMPI_ERROR;
}
BTL_VERBOSE(("Recv unpack remote psn = %d", info->rem_qps->rem_psn));
BTL_VERBOSE(("unpacking %d of %d\n", cnt, OPAL_UINT32));
rc = opal_dss.unpack(buffer, &info->rem_mtu, &cnt, OPAL_UINT32);
if (OPAL_SUCCESS != rc) {
OMPI_ERROR_LOG(rc);
return OMPI_ERROR;
}
BTL_VERBOSE(("Recv unpack remote mtu = %d", info->rem_mtu));
}
if (ENDPOINT_XOOB_CONNECT_REQUEST == *message_type ||
ENDPOINT_XOOB_CONNECT_XRC_REQUEST == *message_type) {
/* unpack requested lid info */
BTL_VERBOSE(("unpacking %d of %d\n", cnt, OPAL_UINT16));
rc = opal_dss.unpack(buffer, lid, &cnt, OPAL_UINT16);
if (OPAL_SUCCESS != rc) {
OMPI_ERROR_LOG(rc);
return OMPI_ERROR;
}
BTL_VERBOSE(("Recv unpack requested lid = %d", *lid));
}
/* Unpack requested recv qp number */
if (ENDPOINT_XOOB_CONNECT_XRC_REQUEST == *message_type) {
BTL_VERBOSE(("unpacking %d of %d\n", cnt, OPAL_UINT32));
/* In XRC request case we will use rem_qp_num as container for requested qp number */
rc = opal_dss.unpack(buffer, &info->rem_qps->rem_qp_num, &cnt,
OPAL_UINT32);
if (OPAL_SUCCESS != rc) {
OMPI_ERROR_LOG(rc);
return rc;
}
BTL_VERBOSE(("Recv unpack requested qp = %x", info->rem_qps->rem_qp_num));
}
if (ENDPOINT_XOOB_CONNECT_RESPONSE == *message_type ||
ENDPOINT_XOOB_CONNECT_XRC_RESPONSE == *message_type) {
BTL_VERBOSE(("unpacking %d of %d\n", cnt, OPAL_UINT32));
rc = opal_dss.unpack(buffer, &info->rem_index, &cnt, OPAL_UINT32);
if (OPAL_SUCCESS != rc) {
OMPI_ERROR_LOG(rc);
return OMPI_ERROR;
}
BTL_VERBOSE(("Recv unpack remote index = %d", info->rem_index));
for (srq = 0; srq < mca_btl_openib_component.num_xrc_qps; srq++) {
BTL_VERBOSE(("unpacking %d of %d\n", cnt, OPAL_UINT32));
rc = opal_dss.unpack(buffer, &info->rem_srqs[srq].rem_srq_num, &cnt, OPAL_UINT32);
if (OPAL_SUCCESS != rc) {
OMPI_ERROR_LOG(rc);
return OMPI_ERROR;
}
BTL_VERBOSE(("Recv unpack remote index srq num[%d]= %d", srq, info->rem_srqs[srq].rem_srq_num));
}
}
return OMPI_SUCCESS;
}
int vprotocol_pessimist_event_logger_connect(int el_rank, ompi_communicator_t **el_comm)
{
int rc;
opal_buffer_t *buffer;
char *port;
ompi_process_name_t el_proc;
char *hnp_uri, *rml_uri;
ompi_rml_tag_t el_tag;
char name[MPI_MAX_PORT_NAME];
int rank;
vprotocol_pessimist_clock_t connect_info[2];
snprintf(name, MPI_MAX_PORT_NAME, VPROTOCOL_EVENT_LOGGER_NAME_FMT, el_rank);
port = ompi_pubsub.lookup(name, MPI_INFO_NULL);
if(NULL == port)
{
return OMPI_ERR_NOT_FOUND;
}
V_OUTPUT_VERBOSE(45, "Found port < %s >", port);
/* separate the string into the HNP and RML URI and tag */
if (OMPI_SUCCESS != (rc = ompi_dpm.parse_port(port, &hnp_uri, &rml_uri, &el_tag))) {
OMPI_ERROR_LOG(rc);
return rc;
}
/* extract the originating proc's name */
if (OMPI_SUCCESS != (rc = ompi_rte_parse_uris(rml_uri, &el_proc, NULL))) {
OMPI_ERROR_LOG(rc);
free(rml_uri); free(hnp_uri);
return rc;
}
/* make sure we can route rml messages to the destination */
if (OMPI_SUCCESS != (rc = ompi_dpm.route_to_port(hnp_uri, &el_proc))) {
OMPI_ERROR_LOG(rc);
free(rml_uri); free(hnp_uri);
return rc;
}
free(rml_uri); free(hnp_uri);
/* Send an rml message to tell the remote end to wake up and jump into
* connect/accept */
buffer = OBJ_NEW(opal_buffer_t);
ompi_rte_send_buffer_nb(&el_proc, buffer, el_tag+1, NULL, NULL);
rc = ompi_dpm.connect_accept(MPI_COMM_SELF, 0, port, true, el_comm);
if(OMPI_SUCCESS != rc) {
OMPI_ERROR_LOG(rc);
}
/* Send Rank, receive max buffer size and max_clock back */
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
rc = mca_pml_v.host_pml.pml_send(&rank, 1, MPI_INTEGER, 0,
VPROTOCOL_PESSIMIST_EVENTLOG_NEW_CLIENT_CMD,
MCA_PML_BASE_SEND_STANDARD,
mca_vprotocol_pessimist.el_comm);
if(OPAL_UNLIKELY(MPI_SUCCESS != rc))
OMPI_ERRHANDLER_INVOKE(mca_vprotocol_pessimist.el_comm, rc,
__FILE__ ": failed sending event logger handshake");
rc = mca_pml_v.host_pml.pml_recv(&connect_info, 2, MPI_UNSIGNED_LONG_LONG,
0, VPROTOCOL_PESSIMIST_EVENTLOG_NEW_CLIENT_CMD,
mca_vprotocol_pessimist.el_comm, MPI_STATUS_IGNORE);
if(OPAL_UNLIKELY(MPI_SUCCESS != rc)) \
OMPI_ERRHANDLER_INVOKE(mca_vprotocol_pessimist.el_comm, rc, \
__FILE__ ": failed receiving event logger handshake");
return rc;
}
int
ompi_proc_unpack(opal_buffer_t* buf,
int proclistsize, ompi_proc_t ***proclist,
bool full_info,
int *newproclistsize, ompi_proc_t ***newproclist)
{
int i;
size_t newprocs_len = 0;
ompi_proc_t **plist=NULL, **newprocs = NULL;
/* do not free plist *ever*, since it is used in the remote group
structure of a communicator */
plist = (ompi_proc_t **) calloc (proclistsize, sizeof (ompi_proc_t *));
if ( NULL == plist ) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* free this on the way out */
newprocs = (ompi_proc_t **) calloc (proclistsize, sizeof (ompi_proc_t *));
if (NULL == newprocs) {
free(plist);
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* cycle through the array of provided procs and unpack
* their info - as packed by ompi_proc_pack
*/
for ( i=0; i<proclistsize; i++ ){
int32_t count=1;
ompi_process_name_t new_name;
uint32_t new_arch;
char *new_hostname;
bool isnew = false;
int rc;
rc = opal_dss.unpack(buf, &new_name, &count, OMPI_NAME);
if (rc != OPAL_SUCCESS) {
OMPI_ERROR_LOG(rc);
free(plist);
free(newprocs);
return rc;
}
if (!full_info) {
rc = opal_dss.unpack(buf, &new_arch, &count, OPAL_UINT32);
if (rc != OPAL_SUCCESS) {
OMPI_ERROR_LOG(rc);
free(plist);
free(newprocs);
return rc;
}
rc = opal_dss.unpack(buf, &new_hostname, &count, OPAL_STRING);
if (rc != OPAL_SUCCESS) {
OMPI_ERROR_LOG(rc);
free(plist);
free(newprocs);
return rc;
}
}
/* see if this proc is already on our ompi_proc_list */
plist[i] = ompi_proc_find_and_add(&new_name, &isnew);
if (isnew) {
/* if not, then it was added, so update the values
* in the proc_t struct with the info that was passed
* to us
*/
newprocs[newprocs_len++] = plist[i];
if (full_info) {
int32_t num_recvd_entries;
int32_t cnt;
int32_t j;
/* unpack the number of entries for this proc */
cnt = 1;
if (OPAL_SUCCESS != (rc = opal_dss.unpack(buf, &num_recvd_entries, &cnt, OPAL_INT32))) {
OMPI_ERROR_LOG(rc);
break;
}
/*
* Extract the attribute names and values
*/
for (j = 0; j < num_recvd_entries; j++) {
opal_value_t *kv;
cnt = 1;
if (OPAL_SUCCESS != (rc = opal_dss.unpack(buf, &kv, &cnt, OPAL_VALUE))) {
OMPI_ERROR_LOG(rc);
break;
}
/* if this is me, dump the data - we already have it in the db */
if (OPAL_EQUAL == ompi_rte_compare_name_fields(OMPI_RTE_CMP_ALL,
OMPI_PROC_MY_NAME, &new_name)) {
OBJ_RELEASE(kv);
} else {
/* store it in the database */
if (OPAL_SUCCESS != (rc = opal_db.store_pointer((opal_identifier_t*)&new_name, kv))) {
OMPI_ERROR_LOG(rc);
OBJ_RELEASE(kv);
}
/* do not release the kv - the db holds that pointer */
}
}
#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
rc = opal_db.fetch((opal_identifier_t*)&new_name, "OMPI_ARCH",
(void**)&new_arch, OPAL_UINT32);
if( OPAL_SUCCESS == rc ) {
new_arch = opal_local_arch;
}
#else
new_arch = opal_local_arch;
#endif
if (ompi_process_info.num_procs < ompi_hostname_cutoff) {
/* retrieve the hostname */
rc = opal_db.fetch_pointer((opal_identifier_t*)&new_name, OMPI_DB_HOSTNAME,
(void**)&new_hostname, OPAL_STRING);
if( OPAL_SUCCESS != rc ) {
new_hostname = NULL;
}
} else {
/* just set the hostname to NULL for now - we'll fill it in
* as modex_recv's are called for procs we will talk to
*/
new_hostname = NULL;
}
}
/* update all the values */
plist[i]->proc_arch = new_arch;
/* if arch is different than mine, create a new convertor for this proc */
if (plist[i]->proc_arch != opal_local_arch) {
#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
OBJ_RELEASE(plist[i]->proc_convertor);
plist[i]->proc_convertor = opal_convertor_create(plist[i]->proc_arch, 0);
#else
opal_show_help("help-mpi-runtime",
"heterogeneous-support-unavailable",
true, ompi_process_info.nodename,
new_hostname == NULL ? "<hostname unavailable>" :
new_hostname);
free(plist);
free(newprocs);
return OMPI_ERR_NOT_SUPPORTED;
#endif
}
if (0 == strcmp(ompi_proc_local_proc->proc_hostname, new_hostname)) {
plist[i]->proc_flags |= (OPAL_PROC_ON_NODE | OPAL_PROC_ON_CU | OPAL_PROC_ON_CLUSTER);
}
/* Save the hostname */
plist[i]->proc_hostname = new_hostname;
} else {
if (full_info) {
int32_t num_recvd_entries;
int32_t j, cnt;
/* discard all keys: they are already locally known */
cnt = 1;
if (OPAL_SUCCESS == (rc = opal_dss.unpack(buf, &num_recvd_entries, &cnt, OPAL_INT32))) {
for (j = 0; j < num_recvd_entries; j++) {
opal_value_t *kv;
cnt = 1;
if (OPAL_SUCCESS != (rc = opal_dss.unpack(buf, &kv, &cnt, OPAL_VALUE))) {
OMPI_ERROR_LOG(rc);
continue;
}
OBJ_RELEASE(kv);
}
} else {
OMPI_ERROR_LOG(rc);
}
}
}
}
if (NULL != newproclistsize) *newproclistsize = newprocs_len;
if (NULL != newproclist) {
*newproclist = newprocs;
} else if (newprocs != NULL) {
free(newprocs);
}
*proclist = plist;
return OMPI_SUCCESS;
}
int
ompi_proc_pack(ompi_proc_t **proclist, int proclistsize,
bool full_info,
opal_buffer_t* buf)
{
int i, rc;
OPAL_THREAD_LOCK(&ompi_proc_lock);
/* cycle through the provided array, packing the OMPI level
* data for each proc. This data may or may not be included
* in any subsequent modex operation, so we include it here
* to ensure completion of a connect/accept handshake. See
* the ompi/mca/dpm framework for an example of where and how
* this info is used.
*
* Eventually, we will review the procedures that call this
* function to see if duplication of communication can be
* reduced. For now, just go ahead and pack the info so it
* can be sent.
*/
for (i=0; i<proclistsize; i++) {
rc = opal_dss.pack(buf, &(proclist[i]->proc_name), 1, OMPI_NAME);
if(rc != OPAL_SUCCESS) {
OMPI_ERROR_LOG(rc);
OPAL_THREAD_UNLOCK(&ompi_proc_lock);
return rc;
}
if (full_info) {
int32_t num_entries;
opal_value_t *kv;
opal_list_t data;
/* fetch all global info we know about the peer - while
* the remote procs may already know some of it, we cannot
* be certain they do. So we must include a full dump of
* everything we know about this proc, excluding INTERNAL
* data that each process computes about its peers
*/
OBJ_CONSTRUCT(&data, opal_list_t);
rc = opal_db.fetch_multiple((opal_identifier_t*)&proclist[i]->proc_name,
OPAL_SCOPE_GLOBAL, NULL, &data);
if (OPAL_SUCCESS != rc) {
OMPI_ERROR_LOG(rc);
num_entries = 0;
} else {
/* count the number of entries we will send */
num_entries = opal_list_get_size(&data);
}
/* put the number of entries into the buffer */
rc = opal_dss.pack(buf, &num_entries, 1, OPAL_INT32);
if (OPAL_SUCCESS != rc) {
OMPI_ERROR_LOG(rc);
break;
}
/* if there are entries, store them */
while (NULL != (kv = (opal_value_t*)opal_list_remove_first(&data))) {
if (OPAL_SUCCESS != (rc = opal_dss.pack(buf, &kv, 1, OPAL_VALUE))) {
OMPI_ERROR_LOG(rc);
break;
}
OBJ_RELEASE(kv);
}
OBJ_DESTRUCT(&data);
} else {
rc = opal_dss.pack(buf, &(proclist[i]->proc_arch), 1, OPAL_UINT32);
if(rc != OPAL_SUCCESS) {
OMPI_ERROR_LOG(rc);
OPAL_THREAD_UNLOCK(&ompi_proc_lock);
return rc;
}
rc = opal_dss.pack(buf, &(proclist[i]->proc_hostname), 1, OPAL_STRING);
if(rc != OPAL_SUCCESS) {
OMPI_ERROR_LOG(rc);
OPAL_THREAD_UNLOCK(&ompi_proc_lock);
return rc;
}
}
}
OPAL_THREAD_UNLOCK(&ompi_proc_lock);
return OMPI_SUCCESS;
}
/*
* For a specific module, see if this proc has matching address/modex
* info. If so, create an endpoint and return it.
*
* Implementation note: This code relies on the order of modules on a local
* side matching the order of the modex entries that we send around, otherwise
* both sides may not agree on a bidirectional connection. It also assumes
* that add_procs will be invoked on the local modules in that same order, for
* the same reason. If those assumptions do not hold, we will need to
* canonicalize this match ordering somehow, probably by (jobid,vpid) pair or
* by the interface MAC or IP address.
*/
static int match_modex(ompi_btl_usnic_module_t *module,
ompi_btl_usnic_proc_t *proc,
int *index_out)
{
int err = OMPI_SUCCESS;
size_t i;
uint32_t num_modules;
ompi_btl_usnic_graph_t *g = NULL;
int nme;
int *me;
bool proc_is_left;
if (NULL == index_out) {
return OMPI_ERR_BAD_PARAM;
}
*index_out = -1;
num_modules = mca_btl_usnic_component.num_modules;
opal_output_verbose(20, USNIC_OUT, "btl:usnic:%s: module=%p proc=%p with dimensions %d x %d",
__func__, (void *)module, (void *)proc,
num_modules, (int)proc->proc_modex_count);
/* We compute an interface match-up table once for each (module,proc) pair
* and cache it in the proc. Store per-proc instead of per-module, since
* MPI dynamic process routines can add procs but not new modules. */
if (NULL == proc->proc_ep_match_table) {
proc->proc_ep_match_table = malloc(num_modules *
sizeof(*proc->proc_ep_match_table));
if (NULL == proc->proc_ep_match_table) {
OMPI_ERROR_LOG(OMPI_ERR_OUT_OF_RESOURCE);
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* initialize to "no matches" */
for (i = 0; i < num_modules; ++i) {
proc->proc_ep_match_table[i] = -1;
}
/* For graphs where all edges are equal (and even for some other
* graphs), two peers making matching calculations with "mirror image"
* graphs might not end up with the same matching. Ensure that both
* sides are always setting up the exact same graph by always putting
* the process with the lower (jobid,vpid) on the "left".
*/
proc_is_left =
(ompi_rte_compare_name_fields(OMPI_RTE_CMP_ALL,
&proc->proc_ompi->proc_name,
&(ompi_proc_local()->proc_name)) < 0);
err = create_proc_module_graph(proc, proc_is_left, &g);
if (OMPI_SUCCESS != err) {
goto out_free_table;
}
nme = 0;
err = ompi_btl_usnic_solve_bipartite_assignment(g, &nme, &me);
if (OMPI_SUCCESS != err) {
OMPI_ERROR_LOG(err);
goto out_free_graph;
}
edge_pairs_to_match_table(proc, proc_is_left, nme, me);
err = ompi_btl_usnic_gr_free(g);
if (OMPI_SUCCESS != err) {
OMPI_ERROR_LOG(err);
return err;
}
}
if (!proc->proc_match_exists) {
opal_output_verbose(5, USNIC_OUT, "btl:usnic:%s: unable to find any valid interface pairs for proc %s",
__func__, OMPI_NAME_PRINT(&proc->proc_ompi->proc_name));
return OMPI_ERR_NOT_FOUND;
}
/* assuming no strange failure cases, this should always be present */
if (NULL != proc->proc_ep_match_table && proc->proc_match_exists) {
for (i = 0; i < num_modules; ++i) {
if (module == mca_btl_usnic_component.usnic_active_modules[i]) {
*index_out = proc->proc_ep_match_table[i];
break;
}
}
}
/* If MTU does not match, throw an error */
/* TODO with UDP, do we still want to enforce this restriction or just take
* the min of the two MTUs? Another choice is to disqualify this pairing
* before running the matching algorithm on it. */
if (*index_out >= 0 &&
proc->proc_modex[*index_out].mtu != module->if_mtu) {
opal_show_help("help-mpi-btl-usnic.txt", "MTU mismatch",
true,
ompi_process_info.nodename,
ibv_get_device_name(module->device),
module->port_num,
module->if_mtu,
(NULL == proc->proc_ompi->proc_hostname) ?
"unknown" : proc->proc_ompi->proc_hostname,
proc->proc_modex[*index_out].mtu);
*index_out = -1;
return OMPI_ERR_UNREACH;
}
return (*index_out == -1 ? OMPI_ERR_NOT_FOUND : OMPI_SUCCESS);
out_free_graph:
ompi_btl_usnic_gr_free(g);
out_free_table:
free(proc->proc_ep_match_table);
proc->proc_ep_match_table = NULL;
proc->proc_match_exists = false;
return err;
}
/**
* Constructs an interface graph from all local modules and the given proc's
* remote interfaces. The resulting vertices will always have the module
* vertices appear before the proc vertices.
*/
static int create_proc_module_graph(
ompi_btl_usnic_proc_t *proc,
bool proc_is_left,
ompi_btl_usnic_graph_t **g_out)
{
int err;
int i, j;
int u, v;
int num_modules;
ompi_btl_usnic_graph_t *g = NULL;
if (NULL == g_out) {
return OMPI_ERR_BAD_PARAM;
}
*g_out = NULL;
num_modules = (int)mca_btl_usnic_component.num_modules;
/* Construct a bipartite graph with remote interfaces on the one side and
* local interfaces (modules) on the other. */
err = ompi_btl_usnic_gr_create(NULL, NULL, &g);
if (OMPI_SUCCESS != err) {
OMPI_ERROR_LOG(err);
goto out;
}
/* create vertices for each interface (local and remote) */
for (i = 0; i < num_modules; ++i) {
int idx = -1;
err = ompi_btl_usnic_gr_add_vertex(g,
mca_btl_usnic_component.usnic_active_modules[i],
&idx);
if (OMPI_SUCCESS != err) {
OMPI_ERROR_LOG(err);
goto out_free_graph;
}
assert(idx == MODULE_VERTEX(i));
}
for (i = 0; i < (int)proc->proc_modex_count; ++i) {
int idx = -1;
err = ompi_btl_usnic_gr_add_vertex(g, &proc->proc_modex[i], &idx);
if (OMPI_SUCCESS != err) {
OMPI_ERROR_LOG(err);
goto out_free_graph;
}
assert(idx == (int)PROC_VERTEX(i));
}
/* now add edges between interfaces that can communicate */
for (i = 0; i < num_modules; ++i) {
for (j = 0; j < (int)proc->proc_modex_count; ++j) {
int64_t weight, cost;
/* assumption: compute_weight returns the same weight on the
* remote process with these arguments (effectively) transposed */
weight = compute_weight(mca_btl_usnic_component.usnic_active_modules[i],
&proc->proc_modex[j]);
opal_output_verbose(20, USNIC_OUT,
"btl:usnic:%s: weight=0x%016" PRIx64 " for edge module[%d] (%p) <--> endpoint[%d] on proc %p",
__func__,
weight, i,
(void *)mca_btl_usnic_component.usnic_active_modules[i],
j, (void *)proc);
if (WEIGHT_UNREACHABLE == weight) {
continue;
} else {
/* the graph code optimizes for minimum *cost*, but we have
* been computing weights (negative costs) */
cost = -weight;
}
assert(INT64_MAX != cost);
assert(INT64_MIN != cost);
if (proc_is_left) {
u = PROC_VERTEX(j);
v = MODULE_VERTEX(i);
} else {
u = MODULE_VERTEX(i);
v = PROC_VERTEX(j);
}
opal_output_verbose(20, USNIC_OUT,
"btl:usnic:%s: adding edge (%d,%d) with cost=%" PRIi64 " for edge module[%d] <--> endpoint[%d]",
__func__, u, v, cost, i, j);
err = ompi_btl_usnic_gr_add_edge(g, u, v, cost,
/*capacity=*/1,
/*e_data=*/NULL);
if (OMPI_SUCCESS != err) {
OMPI_ERROR_LOG(err);
goto out_free_graph;
}
}
}
*g_out = g;
return OMPI_SUCCESS;
out_free_graph:
ompi_btl_usnic_gr_free(g);
out:
return err;
}
/*
* Create an ompi_btl_usnic_proc_t and initialize it with modex info
* and an empty array of endpoints.
*/
static ompi_btl_usnic_proc_t *create_proc(ompi_proc_t *ompi_proc)
{
ompi_btl_usnic_proc_t *proc = NULL;
size_t size;
int rc;
/* Create the proc if it doesn't already exist */
proc = OBJ_NEW(ompi_btl_usnic_proc_t);
if (NULL == proc) {
return NULL;
}
/* Initialize number of peers */
proc->proc_endpoint_count = 0;
proc->proc_ompi = ompi_proc;
/* query for the peer address info */
rc = ompi_modex_recv(&mca_btl_usnic_component.super.btl_version,
ompi_proc, (void*)&proc->proc_modex,
&size);
if (OMPI_SUCCESS != rc) {
opal_show_help("help-mpi-btl-usnic.txt", "internal error during init",
true,
ompi_process_info.nodename,
"<none>", 0,
"ompi_modex_recv() failed", __FILE__, __LINE__,
opal_strerror(rc));
OBJ_RELEASE(proc);
return NULL;
}
if ((size % sizeof(ompi_btl_usnic_addr_t)) != 0) {
char msg[1024];
snprintf(msg, sizeof(msg),
"sizeof(modex for peer %s data) == %d, expected multiple of %d",
OMPI_NAME_PRINT(&ompi_proc->proc_name),
(int) size, (int) sizeof(ompi_btl_usnic_addr_t));
opal_show_help("help-mpi-btl-usnic.txt", "internal error during init",
true,
ompi_process_info.nodename,
"<none>", 0,
"invalid modex data", __FILE__, __LINE__,
msg);
OBJ_RELEASE(proc);
return NULL;
}
proc->proc_modex_count = size / sizeof(ompi_btl_usnic_addr_t);
if (0 == proc->proc_modex_count) {
proc->proc_endpoints = NULL;
OBJ_RELEASE(proc);
return NULL;
}
proc->proc_modex_claimed = (bool*)
calloc(proc->proc_modex_count, sizeof(bool));
if (NULL == proc->proc_modex_claimed) {
OMPI_ERROR_LOG(OMPI_ERR_OUT_OF_RESOURCE);
OBJ_RELEASE(proc);
return NULL;
}
proc->proc_endpoints = (mca_btl_base_endpoint_t**)
calloc(proc->proc_modex_count, sizeof(mca_btl_base_endpoint_t*));
if (NULL == proc->proc_endpoints) {
OMPI_ERROR_LOG(OMPI_ERR_OUT_OF_RESOURCE);
OBJ_RELEASE(proc);
return NULL;
}
return proc;
}
int ompi_mpi_finalize(void)
{
int ret = MPI_SUCCESS;
opal_list_item_t *item;
ompi_proc_t** procs;
size_t nprocs;
OPAL_TIMING_DECLARE(tm);
OPAL_TIMING_INIT_EXT(&tm, OPAL_TIMING_GET_TIME_OF_DAY);
/* Be a bit social if an erroneous program calls MPI_FINALIZE in
two different threads, otherwise we may deadlock in
ompi_comm_free() (or run into other nasty lions, tigers, or
bears).
This lock is held for the duration of ompi_mpi_init() and
ompi_mpi_finalize(). Hence, if we get it, then no other thread
is inside the critical section (and we don't have to check the
*_started bool variables). */
opal_mutex_lock(&ompi_mpi_bootstrap_mutex);
if (!ompi_mpi_initialized || ompi_mpi_finalized) {
/* Note that if we're not initialized or already finalized, we
cannot raise an MPI exception. The best that we can do is
write something to stderr. */
char hostname[MAXHOSTNAMELEN];
pid_t pid = getpid();
gethostname(hostname, sizeof(hostname));
if (ompi_mpi_initialized) {
opal_show_help("help-mpi-runtime.txt",
"mpi_finalize: not initialized",
true, hostname, pid);
} else if (ompi_mpi_finalized) {
opal_show_help("help-mpi-runtime.txt",
"mpi_finalize:invoked_multiple_times",
true, hostname, pid);
}
opal_mutex_unlock(&ompi_mpi_bootstrap_mutex);
return MPI_ERR_OTHER;
}
ompi_mpi_finalize_started = true;
ompi_mpiext_fini();
/* Per MPI-2:4.8, we have to free MPI_COMM_SELF before doing
anything else in MPI_FINALIZE (to include setting up such that
MPI_FINALIZED will return true). */
if (NULL != ompi_mpi_comm_self.comm.c_keyhash) {
ompi_attr_delete_all(COMM_ATTR, &ompi_mpi_comm_self,
ompi_mpi_comm_self.comm.c_keyhash);
OBJ_RELEASE(ompi_mpi_comm_self.comm.c_keyhash);
ompi_mpi_comm_self.comm.c_keyhash = NULL;
}
/* Proceed with MPI_FINALIZE */
ompi_mpi_finalized = true;
/* As finalize is the last legal MPI call, we are allowed to force the release
* of the user buffer used for bsend, before going anywhere further.
*/
(void)mca_pml_base_bsend_detach(NULL, NULL);
#if OPAL_ENABLE_PROGRESS_THREADS == 0
opal_progress_set_event_flag(OPAL_EVLOOP_ONCE | OPAL_EVLOOP_NONBLOCK);
#endif
/* Redo ORTE calling opal_progress_event_users_increment() during
MPI lifetime, to get better latency when not using TCP */
opal_progress_event_users_increment();
/* check to see if we want timing information */
OPAL_TIMING_MSTART((&tm,"time to execute finalize barrier"));
/* NOTE: MPI-2.1 requires that MPI_FINALIZE is "collective" across
*all* connected processes. This only means that all processes
have to call it. It does *not* mean that all connected
processes need to synchronize (either directly or indirectly).
For example, it is quite easy to construct complicated
scenarios where one job is "connected" to another job via
transitivity, but have no direct knowledge of each other.
Consider the following case: job A spawns job B, and job B
later spawns job C. A "connectedness" graph looks something
like this:
A <--> B <--> C
So what are we *supposed* to do in this case? If job A is
still connected to B when it calls FINALIZE, should it block
until jobs B and C also call FINALIZE?
After lengthy discussions many times over the course of this
project, the issue was finally decided at the Louisville Feb
2009 meeting: no.
Rationale:
- "Collective" does not mean synchronizing. It only means that
every process call it. Hence, in this scenario, every
process in A, B, and C must call FINALIZE.
- KEY POINT: if A calls FINALIZE, then it is erroneous for B or
C to try to communicate with A again.
- Hence, OMPI is *correct* to only effect a barrier across each
jobs' MPI_COMM_WORLD before exiting. Specifically, if A
calls FINALIZE long before B or C, it's *correct* if A exits
at any time (and doesn't notify B or C that it is exiting).
- Arguably, if B or C do try to communicate with the now-gone
A, OMPI should try to print a nice error ("you tried to
communicate with a job that is already gone...") instead of
segv or other Badness. However, that is an *extremely*
difficult problem -- sure, it's easy for A to tell B that it
is finalizing, but how can A tell C? A doesn't even know
about C. You'd need to construct a "connected" graph in a
distributed fashion, which is fraught with race conditions,
etc.
Hence, our conclusion is: OMPI is *correct* in its current
behavior (of only doing a barrier across its own COMM_WORLD)
before exiting. Any problems that occur are as a result of
erroneous MPI applications. We *could* tighten up the erroneous
cases and ensure that we print nice error messages / don't
crash, but that is such a difficult problem that we decided we
have many other, much higher priority issues to handle that deal
with non-erroneous cases. */
/* Wait for everyone to reach this point. This is a grpcomm
barrier instead of an MPI barrier for (at least) two reasons:
1. An MPI barrier doesn't ensure that all messages have been
transmitted before exiting (e.g., a BTL can lie and buffer a
message without actually injecting it to the network, and
therefore require further calls to that BTL's progress), so
the possibility of a stranded message exists.
2. If the MPI communication is using an unreliable transport,
there's a problem of knowing that everyone has *left* the
barrier. E.g., one proc can send its ACK to the barrier
message to a peer and then leave the barrier, but the ACK
can get lost and therefore the peer is left in the barrier.
Point #1 has been known for a long time; point #2 emerged after
we added the first unreliable BTL to Open MPI and fixed the
del_procs behavior around May of 2014 (see
https://svn.open-mpi.org/trac/ompi/ticket/4669#comment:4 for
more details). */
opal_pmix.fence(NULL, 0);
/* check for timing request - get stop time and report elapsed
time if so */
OPAL_TIMING_MSTOP(&tm);
OPAL_TIMING_DELTAS(ompi_enable_timing, &tm);
OPAL_TIMING_REPORT(ompi_enable_timing_ext, &tm);
OPAL_TIMING_RELEASE(&tm);
/*
* Shutdown the Checkpoint/Restart Mech.
*/
if (OMPI_SUCCESS != (ret = ompi_cr_finalize())) {
OMPI_ERROR_LOG(ret);
}
/* Shut down any bindings-specific issues: C++, F77, F90 */
/* Remove all memory associated by MPI_REGISTER_DATAREP (per
MPI-2:9.5.3, there is no way for an MPI application to
*un*register datareps, but we don't want the OMPI layer causing
memory leaks). */
while (NULL != (item = opal_list_remove_first(&ompi_registered_datareps))) {
OBJ_RELEASE(item);
}
OBJ_DESTRUCT(&ompi_registered_datareps);
/* Remove all F90 types from the hash tables. As the OBJ_DESTRUCT will
* call a special destructor able to release predefined types, we can
* simply call the OBJ_DESTRUCT on the hash table and all memory will
* be correctly released.
*/
OBJ_DESTRUCT( &ompi_mpi_f90_integer_hashtable );
OBJ_DESTRUCT( &ompi_mpi_f90_real_hashtable );
OBJ_DESTRUCT( &ompi_mpi_f90_complex_hashtable );
/* Free communication objects */
/* free file resources */
if (OMPI_SUCCESS != (ret = ompi_file_finalize())) {
goto done;
}
/* free window resources */
if (OMPI_SUCCESS != (ret = ompi_win_finalize())) {
goto done;
}
if (OMPI_SUCCESS != (ret = ompi_osc_base_finalize())) {
goto done;
}
/* free communicator resources. this MUST come before finalizing the PML
* as this will call into the pml */
if (OMPI_SUCCESS != (ret = ompi_comm_finalize())) {
goto done;
}
/* call del_procs on all allocated procs even though some may not be known
* to the pml layer. the pml layer is expected to be resilient and ignore
* any unknown procs. */
nprocs = 0;
procs = ompi_proc_get_allocated (&nprocs);
MCA_PML_CALL(del_procs(procs, nprocs));
free(procs);
/* free pml resource */
if(OMPI_SUCCESS != (ret = mca_pml_base_finalize())) {
goto done;
}
/* free requests */
if (OMPI_SUCCESS != (ret = ompi_request_finalize())) {
goto done;
}
if (OMPI_SUCCESS != (ret = ompi_message_finalize())) {
goto done;
}
/* If requested, print out a list of memory allocated by ALLOC_MEM
but not freed by FREE_MEM */
if (0 != ompi_debug_show_mpi_alloc_mem_leaks) {
mca_mpool_base_tree_print(ompi_debug_show_mpi_alloc_mem_leaks);
}
/* Now that all MPI objects dealing with communications are gone,
shut down MCA types having to do with communications */
if (OMPI_SUCCESS != (ret = mca_base_framework_close(&ompi_pml_base_framework) ) ) {
OMPI_ERROR_LOG(ret);
goto done;
}
/* shut down buffered send code */
mca_pml_base_bsend_fini();
#if OPAL_ENABLE_FT_CR == 1
/*
* Shutdown the CRCP Framework, must happen after PML shutdown
*/
if (OMPI_SUCCESS != (ret = mca_base_framework_close(&ompi_crcp_base_framework) ) ) {
OMPI_ERROR_LOG(ret);
goto done;
}
#endif
/* Free secondary resources */
/* free attr resources */
if (OMPI_SUCCESS != (ret = ompi_attr_finalize())) {
goto done;
}
/* free group resources */
if (OMPI_SUCCESS != (ret = ompi_group_finalize())) {
goto done;
}
/* finalize the DPM subsystem */
if ( OMPI_SUCCESS != (ret = ompi_dpm_finalize())) {
goto done;
}
/* free internal error resources */
if (OMPI_SUCCESS != (ret = ompi_errcode_intern_finalize())) {
goto done;
}
/* free error code resources */
if (OMPI_SUCCESS != (ret = ompi_mpi_errcode_finalize())) {
goto done;
}
/* free errhandler resources */
if (OMPI_SUCCESS != (ret = ompi_errhandler_finalize())) {
goto done;
}
/* Free all other resources */
/* free op resources */
if (OMPI_SUCCESS != (ret = ompi_op_finalize())) {
goto done;
}
/* free ddt resources */
if (OMPI_SUCCESS != (ret = ompi_datatype_finalize())) {
goto done;
}
/* free info resources */
if (OMPI_SUCCESS != (ret = ompi_info_finalize())) {
goto done;
}
/* Close down MCA modules */
/* io is opened lazily, so it's only necessary to close it if it
was actually opened */
if (0 < ompi_io_base_framework.framework_refcnt) {
/* May have been "opened" multiple times. We want it closed now */
ompi_io_base_framework.framework_refcnt = 1;
if (OMPI_SUCCESS != mca_base_framework_close(&ompi_io_base_framework)) {
goto done;
}
}
(void) mca_base_framework_close(&ompi_topo_base_framework);
if (OMPI_SUCCESS != (ret = mca_base_framework_close(&ompi_osc_base_framework))) {
goto done;
}
if (OMPI_SUCCESS != (ret = mca_base_framework_close(&ompi_coll_base_framework))) {
goto done;
}
if (OMPI_SUCCESS != (ret = mca_base_framework_close(&ompi_bml_base_framework))) {
goto done;
}
if (OMPI_SUCCESS != (ret = mca_base_framework_close(&opal_mpool_base_framework))) {
goto done;
}
if (OMPI_SUCCESS != (ret = mca_base_framework_close(&opal_rcache_base_framework))) {
goto done;
}
if (OMPI_SUCCESS != (ret = mca_base_framework_close(&opal_allocator_base_framework))) {
goto done;
}
/* free proc resources */
if ( OMPI_SUCCESS != (ret = ompi_proc_finalize())) {
goto done;
}
if (NULL != ompi_mpi_main_thread) {
OBJ_RELEASE(ompi_mpi_main_thread);
ompi_mpi_main_thread = NULL;
}
/* Clean up memory/resources from the MPI dynamic process
functionality checker */
ompi_mpi_dynamics_finalize();
/* Leave the RTE */
if (OMPI_SUCCESS != (ret = ompi_rte_finalize())) {
goto done;
}
ompi_rte_initialized = false;
/* now close the rte framework */
if (OMPI_SUCCESS != (ret = mca_base_framework_close(&ompi_rte_base_framework) ) ) {
OMPI_ERROR_LOG(ret);
goto done;
}
if (OPAL_SUCCESS != (ret = opal_finalize_util())) {
goto done;
}
/* All done */
done:
opal_mutex_unlock(&ompi_mpi_bootstrap_mutex);
return ret;
}
/*
* Create an ompi_btl_usnic_proc_t and initialize it with modex info
* and an empty array of endpoints.
*
* Returns OMPI_ERR_UNREACH if we can't reach the peer (i.e., we can't
* find their modex data).
*/
static int create_proc(ompi_proc_t *ompi_proc,
ompi_btl_usnic_proc_t **usnic_proc)
{
ompi_btl_usnic_proc_t *proc = NULL;
size_t size;
int rc;
*usnic_proc = NULL;
/* Create the proc if it doesn't already exist */
proc = OBJ_NEW(ompi_btl_usnic_proc_t);
if (NULL == proc) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* Initialize number of peers */
proc->proc_endpoint_count = 0;
proc->proc_ompi = ompi_proc;
/* query for the peer address info */
rc = ompi_modex_recv(&mca_btl_usnic_component.super.btl_version,
ompi_proc, (void*)&proc->proc_modex,
&size);
/* If this proc simply doesn't have this key, then they're not
running the usnic BTL -- just ignore them. Otherwise, show an
error message. */
if (OPAL_ERR_DATA_VALUE_NOT_FOUND == rc) {
OBJ_RELEASE(proc);
return OMPI_ERR_UNREACH;
} else if (OMPI_SUCCESS != rc) {
opal_show_help("help-mpi-btl-usnic.txt",
"internal error during init",
true,
ompi_process_info.nodename,
"<none>", 0,
"ompi_modex_recv() failed", __FILE__, __LINE__,
opal_strerror(rc));
OBJ_RELEASE(proc);
return OMPI_ERROR;
}
if ((size % sizeof(ompi_btl_usnic_addr_t)) != 0) {
char msg[1024];
snprintf(msg, sizeof(msg),
"sizeof(modex for peer %s data) == %d, expected multiple of %d",
OMPI_NAME_PRINT(&ompi_proc->proc_name),
(int) size, (int) sizeof(ompi_btl_usnic_addr_t));
opal_show_help("help-mpi-btl-usnic.txt", "internal error during init",
true,
ompi_process_info.nodename,
"<none>", 0,
"invalid modex data", __FILE__, __LINE__,
msg);
OBJ_RELEASE(proc);
return OMPI_ERR_VALUE_OUT_OF_BOUNDS;
}
proc->proc_modex_count = size / sizeof(ompi_btl_usnic_addr_t);
if (0 == proc->proc_modex_count) {
proc->proc_endpoints = NULL;
OBJ_RELEASE(proc);
return OMPI_ERR_UNREACH;
}
proc->proc_modex_claimed = (bool*)
calloc(proc->proc_modex_count, sizeof(bool));
if (NULL == proc->proc_modex_claimed) {
OMPI_ERROR_LOG(OMPI_ERR_OUT_OF_RESOURCE);
OBJ_RELEASE(proc);
return OMPI_ERR_OUT_OF_RESOURCE;
}
proc->proc_endpoints = (mca_btl_base_endpoint_t**)
calloc(proc->proc_modex_count, sizeof(mca_btl_base_endpoint_t*));
if (NULL == proc->proc_endpoints) {
OMPI_ERROR_LOG(OMPI_ERR_OUT_OF_RESOURCE);
OBJ_RELEASE(proc);
return OMPI_ERR_OUT_OF_RESOURCE;
}
*usnic_proc = proc;
return OMPI_SUCCESS;
}
