/* * send connect information to remote endpoint */ static int xoob_send_connect_data(mca_btl_base_endpoint_t* endpoint, uint8_t message_type) { opal_buffer_t* buffer = OBJ_NEW(opal_buffer_t); int rc, srq; if (NULL == buffer) { OMPI_ERROR_LOG(OMPI_ERR_OUT_OF_RESOURCE); return OMPI_ERR_OUT_OF_RESOURCE; } /* Bulding standart header that we use in all messages: * - Message type, * - Our subnet id * - Our LID */ /* pack the info in the send buffer */ BTL_VERBOSE(("Send pack Message type = %d", message_type)); BTL_VERBOSE(("packing %d of %d\n", 1, OPAL_UINT8)); rc = opal_dss.pack(buffer, &message_type, 1, OPAL_UINT8); if (OPAL_SUCCESS != rc) { OMPI_ERROR_LOG(rc); return rc; } BTL_VERBOSE(("Send pack sid = %" PRIx64 "\n", endpoint->subnet_id)); BTL_VERBOSE(("packing %d of %d\n", 1, OPAL_UINT64)); rc = opal_dss.pack(buffer, &endpoint->subnet_id, 1, OPAL_UINT64); if (OPAL_SUCCESS != rc) { OMPI_ERROR_LOG(rc); return rc; } BTL_VERBOSE(("Send pack lid = %d", endpoint->endpoint_btl->lid)); BTL_VERBOSE(("packing %d of %d\n", 1, OPAL_UINT16)); rc = opal_dss.pack(buffer, &endpoint->endpoint_btl->lid, 1, OPAL_UINT16); if (OPAL_SUCCESS != rc) { OMPI_ERROR_LOG(rc); return rc; } /* Now we append to standart header additional information * that is required for full (open qp,etc..) connect request and response: * - qp_num of first qp * - psn of first qp * - MTU */ if (ENDPOINT_XOOB_CONNECT_REQUEST == message_type || ENDPOINT_XOOB_CONNECT_RESPONSE == message_type) { uint32_t psn, qp_num; if (ENDPOINT_XOOB_CONNECT_REQUEST == message_type) { qp_num = endpoint->qps[0].qp->lcl_qp->qp_num; psn = endpoint->qps[0].qp->lcl_psn; } else { qp_num = endpoint->xrc_recv_qp_num; psn = endpoint->xrc_recv_psn; } /* stuff all the QP info into the buffer */ /* we need to send only one QP */ BTL_VERBOSE(("Send pack qp num = %x", qp_num)); BTL_VERBOSE(("packing %d of %d\n", 1, OPAL_UINT32)); rc = opal_dss.pack(buffer, &qp_num, 1, OPAL_UINT32); if (OPAL_SUCCESS != rc) { OMPI_ERROR_LOG(rc); return rc; } BTL_VERBOSE(("Send pack lpsn = %d", psn)); BTL_VERBOSE(("packing %d of %d\n", 1, OPAL_UINT32)); rc = opal_dss.pack(buffer, &psn, 1, OPAL_UINT32); if (OPAL_SUCCESS != rc) { OMPI_ERROR_LOG(rc); return rc; } BTL_VERBOSE(("Send pack mtu = %d", endpoint->endpoint_btl->device->mtu)); BTL_VERBOSE(("packing %d of %d\n", 1, OPAL_UINT32)); rc = opal_dss.pack(buffer, &endpoint->endpoint_btl->device->mtu, 1, OPAL_UINT32); if (OPAL_SUCCESS != rc) { OMPI_ERROR_LOG(rc); return rc; } } /* We append to header above additional information * that is required for full & XRC connect request: * - The lid ob btl on remote site that we want to connect */ if (ENDPOINT_XOOB_CONNECT_REQUEST == message_type || ENDPOINT_XOOB_CONNECT_XRC_REQUEST == message_type) { /* when we are sending request we add remote lid that we want to connect */ BTL_VERBOSE(("Send pack remote lid = %d", endpoint->ib_addr->lid)); BTL_VERBOSE(("packing %d of %d\n", 1, OPAL_UINT16)); rc = opal_dss.pack(buffer, &endpoint->ib_addr->lid, 1, OPAL_UINT16); if (OPAL_SUCCESS != rc) { OMPI_ERROR_LOG(rc); return rc; } } /* when we are sending xrc request we add remote * recv qp number that we want to connect. */ if (ENDPOINT_XOOB_CONNECT_XRC_REQUEST == message_type) { BTL_VERBOSE(("Send pack remote qp = %x", endpoint->ib_addr->remote_xrc_rcv_qp_num)); BTL_VERBOSE(("packing %d of %d\n", 1, OPAL_UINT32)); rc = opal_dss.pack(buffer, &endpoint->ib_addr->remote_xrc_rcv_qp_num, 1, OPAL_UINT32); if (OPAL_SUCCESS != rc) { OMPI_ERROR_LOG(rc); return rc; } } /* We append to header above additional information * that is required for full & XRC connect response: * - index of our endpoint * - array of xrc-srq numbers */ if (ENDPOINT_XOOB_CONNECT_RESPONSE == message_type || ENDPOINT_XOOB_CONNECT_XRC_RESPONSE == message_type) { /* we need to send the endpoint index for immidate send */ BTL_VERBOSE(("Send pack index = %d", endpoint->index)); BTL_VERBOSE(("packing %d of %d\n", 1, OPAL_UINT32)); rc = opal_dss.pack(buffer, &endpoint->index, 1, OPAL_UINT32); if (OPAL_SUCCESS != rc) { OMPI_ERROR_LOG(rc); return rc; } /* on response we add all SRQ numbers */ for (srq = 0; srq < mca_btl_openib_component.num_xrc_qps; srq++) { BTL_VERBOSE(("Send pack srq[%d] num = %d", srq, endpoint->endpoint_btl->qps[srq].u.srq_qp.srq->xrc_srq_num)); BTL_VERBOSE(("packing %d of %d\n", 1, OPAL_UINT32)); rc = opal_dss.pack(buffer, &endpoint->endpoint_btl->qps[srq].u.srq_qp.srq->xrc_srq_num, 1, OPAL_UINT32); if (OPAL_SUCCESS != rc) { OMPI_ERROR_LOG(rc); return rc; } } } /* send to remote endpoint */ rc = ompi_rte_send_buffer_nb(&endpoint->endpoint_proc->proc_ompi->proc_name, buffer, OMPI_RML_TAG_XOPENIB, xoob_rml_send_cb, NULL); if (OMPI_SUCCESS != rc) { OMPI_ERROR_LOG(rc); return rc; } BTL_VERBOSE(("Send QP Info, LID = %d, SUBNET = %" PRIx64 ", Message type = %d", endpoint->endpoint_btl->lid, endpoint->subnet_id, message_type)); return OMPI_SUCCESS; }
int vprotocol_pessimist_event_logger_connect(int el_rank, ompi_communicator_t **el_comm) { int rc; opal_buffer_t *buffer; char *port; ompi_process_name_t el_proc; char *hnp_uri, *rml_uri; ompi_rml_tag_t el_tag; char name[MPI_MAX_PORT_NAME]; int rank; vprotocol_pessimist_clock_t connect_info[2]; snprintf(name, MPI_MAX_PORT_NAME, VPROTOCOL_EVENT_LOGGER_NAME_FMT, el_rank); port = ompi_pubsub.lookup(name, MPI_INFO_NULL); if(NULL == port) { return OMPI_ERR_NOT_FOUND; } V_OUTPUT_VERBOSE(45, "Found port < %s >", port); /* separate the string into the HNP and RML URI and tag */ if (OMPI_SUCCESS != (rc = ompi_dpm.parse_port(port, &hnp_uri, &rml_uri, &el_tag))) { OMPI_ERROR_LOG(rc); return rc; } /* extract the originating proc's name */ if (OMPI_SUCCESS != (rc = ompi_rte_parse_uris(rml_uri, &el_proc, NULL))) { OMPI_ERROR_LOG(rc); free(rml_uri); free(hnp_uri); return rc; } /* make sure we can route rml messages to the destination */ if (OMPI_SUCCESS != (rc = ompi_dpm.route_to_port(hnp_uri, &el_proc))) { OMPI_ERROR_LOG(rc); free(rml_uri); free(hnp_uri); return rc; } free(rml_uri); free(hnp_uri); /* Send an rml message to tell the remote end to wake up and jump into * connect/accept */ buffer = OBJ_NEW(opal_buffer_t); ompi_rte_send_buffer_nb(&el_proc, buffer, el_tag+1, NULL, NULL); rc = ompi_dpm.connect_accept(MPI_COMM_SELF, 0, port, true, el_comm); if(OMPI_SUCCESS != rc) { OMPI_ERROR_LOG(rc); } /* Send Rank, receive max buffer size and max_clock back */ MPI_Comm_rank(MPI_COMM_WORLD, &rank); rc = mca_pml_v.host_pml.pml_send(&rank, 1, MPI_INTEGER, 0, VPROTOCOL_PESSIMIST_EVENTLOG_NEW_CLIENT_CMD, MCA_PML_BASE_SEND_STANDARD, mca_vprotocol_pessimist.el_comm); if(OPAL_UNLIKELY(MPI_SUCCESS != rc)) OMPI_ERRHANDLER_INVOKE(mca_vprotocol_pessimist.el_comm, rc, __FILE__ ": failed sending event logger handshake"); rc = mca_pml_v.host_pml.pml_recv(&connect_info, 2, MPI_UNSIGNED_LONG_LONG, 0, VPROTOCOL_PESSIMIST_EVENTLOG_NEW_CLIENT_CMD, mca_vprotocol_pessimist.el_comm, MPI_STATUS_IGNORE); if(OPAL_UNLIKELY(MPI_SUCCESS != rc)) \ OMPI_ERRHANDLER_INVOKE(mca_vprotocol_pessimist.el_comm, rc, \ __FILE__ ": failed receiving event logger handshake"); return rc; }