mca_oob_ud_peer_t *mca_oob_ud_get_peer (struct mca_oob_ud_port_t *port, orte_process_name_t *name, uint32_t qpn, uint32_t qkey, uint16_t lid, uint8_t port_num) { struct ibv_ah_attr ah_attr; mca_oob_ud_peer_t *peer; int rc; rc = mca_oob_ud_peer_lookup (name, &peer); if (ORTE_SUCCESS == rc) { OPAL_OUTPUT_VERBOSE((20, mca_oob_base_output, "%s oob:ud:peer_from_msg_hdr using " "cached peer", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME))); return peer; } OPAL_OUTPUT_VERBOSE((10, mca_oob_base_output, "%s oob:ud:peer_from_msg_hdr creating " "peer from return address", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME))); peer = OBJ_NEW(mca_oob_ud_peer_t); if (NULL == peer) { return NULL; } peer->peer_qpn = qpn; peer->peer_qkey = qkey; peer->peer_name = *name; peer->peer_lid = lid; peer->peer_port = port_num; memset (&ah_attr, 0, sizeof (ah_attr)); ah_attr.dlid = peer->peer_lid; ah_attr.port_num = peer->peer_port; peer->peer_ah = ibv_create_ah (port->device->ib_pd, &ah_attr); if (NULL == peer->peer_ah) { free (peer); return NULL; } peer->peer_context = port->device; OPAL_THREAD_LOCK(&mca_oob_ud_component.ud_lock); opal_hash_table_set_value_uint64(&mca_oob_ud_component.ud_peers, orte_util_hash_name(name), (void *) peer); OPAL_THREAD_UNLOCK(&mca_oob_ud_component.ud_lock); return peer; }
static int mca_oob_ud_set_addr (const orte_process_name_t *name, const char *uri) { mca_oob_ud_peer_t *peer = NULL; int rc; opal_output_verbose(5, orte_oob_base_framework.framework_output, "%s oob:ud:set_addr: setting location for peer %s from %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), ORTE_NAME_PRINT(name), uri); (void) mca_oob_ud_peer_lookup (name, &peer); if (NULL == uri) { if (NULL != peer) { mca_oob_ud_peer_release (peer); } peer = NULL; } else if (NULL == peer) { peer = mca_oob_ud_peer_from_uri (uri); if (NULL == peer) { return ORTE_ERR_BAD_PARAM; } } else { rc = mca_oob_ud_peer_update_with_uri (peer, uri); if (ORTE_SUCCESS != rc) { return rc; } } if (NULL != peer) { peer->peer_name = *name; peer->needs_notification = true; } opal_proc_table_set_value(&mca_oob_ud_module.peers, *name, (void *)peer); return ORTE_SUCCESS; }
int mca_oob_ud_process_send_nb(int fd, short args, void *cbdata) { mca_oob_ud_msg_op_t *op = (mca_oob_ud_msg_op_t*)cbdata; orte_process_name_t hop; mca_oob_ud_peer_t *peer; mca_oob_ud_port_t *port; mca_oob_ud_msg_t *req_msg; mca_oob_ud_req_t *send_req; bool send_eager = false; char *pack_ptr; int rc, size, i; if (OPAL_EQUAL == orte_util_compare_name_fields (ORTE_NS_CMP_ALL, ORTE_PROC_MY_NAME, &op->msg->dst)) { return mca_oob_ud_send_self (op->msg); } /* if we have a route to this peer, then we can reach it */ hop = orte_routed.get_route(&op->msg->dst); if (ORTE_JOBID_INVALID == hop.jobid || ORTE_VPID_INVALID == hop.vpid) { ORTE_ERROR_LOG(ORTE_ERR_UNREACH); return ORTE_ERR_UNREACH; } rc = mca_oob_ud_peer_lookup (&hop, &peer); if(ORTE_SUCCESS != rc || NULL == peer) { ORTE_ERROR_LOG((NULL == peer) ? ORTE_ERR_UNREACH : rc); return (NULL == peer) ? ORTE_ERR_UNREACH : rc; } opal_output_verbose(2, orte_oob_base_framework.framework_output, "%s oob:ud:send_nb to pear %s via hop %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), ORTE_NAME_PRINT(&op->msg->dst), ORTE_NAME_PRINT(&hop)); /* NTH: TODO -- get a random port? */ port = (mca_oob_ud_port_t *) opal_list_get_first (&((mca_oob_ud_device_t *)peer->peer_context)->ports); send_req = OBJ_NEW(mca_oob_ud_req_t); if (!send_req) { ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE); return ORTE_ERR_OUT_OF_RESOURCE; } /* fill in request */ send_req->req_target = op->msg->dst; send_req->req_origin = op->msg->origin; send_req->req_tag = op->msg->tag; if (op->msg->data != NULL) { size = op->msg->count; send_req->req_data_type = MCA_OOB_UD_REQ_TR; send_req->req_data.buf.p = (char *)calloc(size, sizeof(char)); memcpy(send_req->req_data.buf.p, op->msg->data, op->msg->count); send_req->req_data.buf.size = op->msg->count; } else { MCA_OOB_UD_IOV_SIZE(op->msg, size); if (op->msg->iov != NULL) { send_req->req_data_type = MCA_OOB_UD_REQ_IOV; send_req->req_data.iov.uiov = op->msg->iov; send_req->req_data.iov.count = op->msg->count; } else { send_req->req_data_type = MCA_OOB_UD_REQ_BUF; opal_buffer_t *buffer; buffer = OBJ_NEW(opal_buffer_t); if (OPAL_SUCCESS != (rc = opal_dss.copy_payload(buffer, op->msg->buffer))) { ORTE_ERROR_LOG(rc); OBJ_RELEASE(buffer); return rc; } if (OPAL_SUCCESS != (rc = opal_dss.unload(buffer, (void **)&send_req->req_data.buf.p, &send_req->req_data.buf.size))) { ORTE_ERROR_LOG(rc); OBJ_RELEASE(buffer); free(send_req->req_data.buf.p); return rc; } OBJ_RELEASE(buffer); } } send_req->rml_msg = op->msg; send_req->req_cbdata = op->msg->cbdata; send_req->req_peer = peer; send_req->req_mtu = port->mtu; send_req->req_port = port; send_req->req_rc = 0; send_req->state = MCA_OOB_UD_REQ_PENDING; send_req->type = MCA_OOB_UD_REQ_SEND; OBJ_RETAIN(peer); if (size + sizeof (mca_oob_ud_msg_hdr_t) <= (unsigned int)port->mtu) { send_eager = true; } rc = mca_oob_ud_msg_get (port, send_req, &port->listen_qp, peer, false, &req_msg); if (ORTE_SUCCESS != rc) { OBJ_RELEASE (send_req); return rc; } /* fill in message header */ req_msg->hdr->msg_type = MCA_OOB_UD_MSG_REQUEST; req_msg->hdr->msg_rem_ctx = send_req; req_msg->hdr->msg_origin = op->msg->origin; req_msg->hdr->msg_target = op->msg->dst; req_msg->hdr->msg_data.req.data_len = size; req_msg->hdr->msg_data.req.mtu = port->mtu; req_msg->hdr->msg_data.req.tag = op->msg->tag; if (MCA_OOB_UD_REQ_IOV == send_req->req_data_type) { opal_output_verbose(10, orte_oob_base_framework.framework_output, "%s-%s send_nb: tag %d size %lu. msg: %p. peer = %p. req = %p." "count = %d. uiov = %p.\n", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), ORTE_NAME_PRINT(&op->msg->dst), op->msg->tag, (unsigned long)size, (void *) req_msg, (void *) peer, (void *) send_req, send_req->req_data.iov.count, (void *) send_req->req_data.iov.uiov); } else { opal_output_verbose(10, orte_oob_base_framework.framework_output, "%s-%s send_nb: tag %d size %lu. msg: %p. peer = %p. req = %p." "buffer = %p.\n", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), ORTE_NAME_PRINT(&op->msg->dst), op->msg->tag, (unsigned long)size, (void *) req_msg, (void *) peer, (void *) send_req, (void *) send_req->req_data.buf.p); } if (!send_eager) { mca_oob_ud_req_append_to_list (send_req, &mca_oob_ud_component.ud_active_sends); /* send request */ return mca_oob_ud_msg_post_send (req_msg); } pack_ptr = (char *)(req_msg->hdr + 1); if (op->msg->iov != NULL) { for (i = 0 ; i < op->msg->count ; ++i) { memcpy (pack_ptr, op->msg->iov[i].iov_base, op->msg->iov[i].iov_len); pack_ptr += op->msg->iov[i].iov_len; } } else { memcpy(pack_ptr, send_req->req_data.buf.p, send_req->req_data.buf.size); } send_req->req_list = NULL; req_msg->hdr->msg_data.req.data_follows = true; req_msg->cbfunc = mca_oob_ud_send_cb; req_msg->req = send_req; do { /* send request */ rc = mca_oob_ud_msg_post_send (req_msg); if (ORTE_SUCCESS != rc) { ORTE_ERROR_LOG(rc); break; } } while (0); return rc; }