mca_oob_ud_peer_t *mca_oob_ud_get_peer (struct mca_oob_ud_port_t *port,
orte_process_name_t *name,
uint32_t qpn, uint32_t qkey,
uint16_t lid, uint8_t port_num)
{
struct ibv_ah_attr ah_attr;
mca_oob_ud_peer_t *peer;
int rc;
rc = mca_oob_ud_peer_lookup (name, &peer);
if (ORTE_SUCCESS == rc) {
OPAL_OUTPUT_VERBOSE((20, mca_oob_base_output, "%s oob:ud:peer_from_msg_hdr using "
"cached peer", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
return peer;
}
OPAL_OUTPUT_VERBOSE((10, mca_oob_base_output, "%s oob:ud:peer_from_msg_hdr creating "
"peer from return address", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
peer = OBJ_NEW(mca_oob_ud_peer_t);
if (NULL == peer) {
return NULL;
}
peer->peer_qpn = qpn;
peer->peer_qkey = qkey;
peer->peer_name = *name;
peer->peer_lid = lid;
peer->peer_port = port_num;
memset (&ah_attr, 0, sizeof (ah_attr));
ah_attr.dlid = peer->peer_lid;
ah_attr.port_num = peer->peer_port;
peer->peer_ah = ibv_create_ah (port->device->ib_pd, &ah_attr);
if (NULL == peer->peer_ah) {
free (peer);
return NULL;
}
peer->peer_context = port->device;
OPAL_THREAD_LOCK(&mca_oob_ud_component.ud_lock);
opal_hash_table_set_value_uint64(&mca_oob_ud_component.ud_peers,
orte_util_hash_name(name),
(void *) peer);
OPAL_THREAD_UNLOCK(&mca_oob_ud_component.ud_lock);
return peer;
}
static int mca_oob_ud_set_addr (const orte_process_name_t *name, const char *uri)
{
mca_oob_ud_peer_t *peer = NULL;
int rc;
opal_output_verbose(5, orte_oob_base_framework.framework_output,
"%s oob:ud:set_addr: setting location for peer %s from %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), ORTE_NAME_PRINT(name), uri);
(void) mca_oob_ud_peer_lookup (name, &peer);
if (NULL == uri) {
if (NULL != peer) {
mca_oob_ud_peer_release (peer);
}
peer = NULL;
} else if (NULL == peer) {
peer = mca_oob_ud_peer_from_uri (uri);
if (NULL == peer) {
return ORTE_ERR_BAD_PARAM;
}
} else {
rc = mca_oob_ud_peer_update_with_uri (peer, uri);
if (ORTE_SUCCESS != rc) {
return rc;
}
}
if (NULL != peer) {
peer->peer_name = *name;
peer->needs_notification = true;
}
opal_proc_table_set_value(&mca_oob_ud_module.peers,
*name, (void *)peer);
return ORTE_SUCCESS;
}
int mca_oob_ud_process_send_nb(int fd, short args, void *cbdata)
{
mca_oob_ud_msg_op_t *op = (mca_oob_ud_msg_op_t*)cbdata;
orte_process_name_t hop;
mca_oob_ud_peer_t *peer;
mca_oob_ud_port_t *port;
mca_oob_ud_msg_t *req_msg;
mca_oob_ud_req_t *send_req;
bool send_eager = false;
char *pack_ptr;
int rc, size, i;
if (OPAL_EQUAL == orte_util_compare_name_fields
(ORTE_NS_CMP_ALL, ORTE_PROC_MY_NAME, &op->msg->dst)) {
return mca_oob_ud_send_self (op->msg);
}
/* if we have a route to this peer, then we can reach it */
hop = orte_routed.get_route(&op->msg->dst);
if (ORTE_JOBID_INVALID == hop.jobid ||
ORTE_VPID_INVALID == hop.vpid) {
ORTE_ERROR_LOG(ORTE_ERR_UNREACH);
return ORTE_ERR_UNREACH;
}
rc = mca_oob_ud_peer_lookup (&hop, &peer);
if(ORTE_SUCCESS != rc || NULL == peer) {
ORTE_ERROR_LOG((NULL == peer) ? ORTE_ERR_UNREACH : rc);
return (NULL == peer) ? ORTE_ERR_UNREACH : rc;
}
opal_output_verbose(2, orte_oob_base_framework.framework_output,
"%s oob:ud:send_nb to pear %s via hop %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&op->msg->dst), ORTE_NAME_PRINT(&hop));
/* NTH: TODO -- get a random port? */
port = (mca_oob_ud_port_t *) opal_list_get_first (&((mca_oob_ud_device_t *)peer->peer_context)->ports);
send_req = OBJ_NEW(mca_oob_ud_req_t);
if (!send_req) {
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
return ORTE_ERR_OUT_OF_RESOURCE;
}
/* fill in request */
send_req->req_target = op->msg->dst;
send_req->req_origin = op->msg->origin;
send_req->req_tag = op->msg->tag;
if (op->msg->data != NULL) {
size = op->msg->count;
send_req->req_data_type = MCA_OOB_UD_REQ_TR;
send_req->req_data.buf.p = (char *)calloc(size, sizeof(char));
memcpy(send_req->req_data.buf.p, op->msg->data, op->msg->count);
send_req->req_data.buf.size = op->msg->count;
} else {
MCA_OOB_UD_IOV_SIZE(op->msg, size);
if (op->msg->iov != NULL) {
send_req->req_data_type = MCA_OOB_UD_REQ_IOV;
send_req->req_data.iov.uiov = op->msg->iov;
send_req->req_data.iov.count = op->msg->count;
} else {
send_req->req_data_type = MCA_OOB_UD_REQ_BUF;
opal_buffer_t *buffer;
buffer = OBJ_NEW(opal_buffer_t);
if (OPAL_SUCCESS != (rc = opal_dss.copy_payload(buffer, op->msg->buffer))) {
ORTE_ERROR_LOG(rc);
OBJ_RELEASE(buffer);
return rc;
}
if (OPAL_SUCCESS != (rc = opal_dss.unload(buffer, (void **)&send_req->req_data.buf.p, &send_req->req_data.buf.size)))
{
ORTE_ERROR_LOG(rc);
OBJ_RELEASE(buffer);
free(send_req->req_data.buf.p);
return rc;
}
OBJ_RELEASE(buffer);
}
}
send_req->rml_msg = op->msg;
send_req->req_cbdata = op->msg->cbdata;
send_req->req_peer = peer;
send_req->req_mtu = port->mtu;
send_req->req_port = port;
send_req->req_rc = 0;
send_req->state = MCA_OOB_UD_REQ_PENDING;
send_req->type = MCA_OOB_UD_REQ_SEND;
OBJ_RETAIN(peer);
if (size + sizeof (mca_oob_ud_msg_hdr_t) <= (unsigned int)port->mtu) {
send_eager = true;
}
rc = mca_oob_ud_msg_get (port, send_req, &port->listen_qp, peer, false, &req_msg);
if (ORTE_SUCCESS != rc) {
OBJ_RELEASE (send_req);
return rc;
}
/* fill in message header */
req_msg->hdr->msg_type = MCA_OOB_UD_MSG_REQUEST;
req_msg->hdr->msg_rem_ctx = send_req;
req_msg->hdr->msg_origin = op->msg->origin;
req_msg->hdr->msg_target = op->msg->dst;
req_msg->hdr->msg_data.req.data_len = size;
req_msg->hdr->msg_data.req.mtu = port->mtu;
req_msg->hdr->msg_data.req.tag = op->msg->tag;
if (MCA_OOB_UD_REQ_IOV == send_req->req_data_type) {
opal_output_verbose(10, orte_oob_base_framework.framework_output,
"%s-%s send_nb: tag %d size %lu. msg: %p. peer = %p. req = %p."
"count = %d. uiov = %p.\n",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&op->msg->dst),
op->msg->tag, (unsigned long)size,
(void *) req_msg,
(void *) peer, (void *) send_req,
send_req->req_data.iov.count, (void *) send_req->req_data.iov.uiov);
} else {
opal_output_verbose(10, orte_oob_base_framework.framework_output,
"%s-%s send_nb: tag %d size %lu. msg: %p. peer = %p. req = %p."
"buffer = %p.\n",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&op->msg->dst),
op->msg->tag, (unsigned long)size,
(void *) req_msg,
(void *) peer, (void *) send_req, (void *) send_req->req_data.buf.p);
}
if (!send_eager) {
mca_oob_ud_req_append_to_list (send_req, &mca_oob_ud_component.ud_active_sends);
/* send request */
return mca_oob_ud_msg_post_send (req_msg);
}
pack_ptr = (char *)(req_msg->hdr + 1);
if (op->msg->iov != NULL) {
for (i = 0 ; i < op->msg->count ; ++i) {
memcpy (pack_ptr, op->msg->iov[i].iov_base, op->msg->iov[i].iov_len);
pack_ptr += op->msg->iov[i].iov_len;
}
} else {
memcpy(pack_ptr, send_req->req_data.buf.p, send_req->req_data.buf.size);
}
send_req->req_list = NULL;
req_msg->hdr->msg_data.req.data_follows = true;
req_msg->cbfunc = mca_oob_ud_send_cb;
req_msg->req = send_req;
do {
/* send request */
rc = mca_oob_ud_msg_post_send (req_msg);
if (ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
break;
}
} while (0);
return rc;
}
