static void mca_oob_tcp_msg_ident(mca_oob_tcp_msg_t* msg, mca_oob_tcp_peer_t* peer)
{
orte_process_name_t src = msg->msg_hdr.msg_src;
OPAL_THREAD_LOCK(&mca_oob_tcp_component.tcp_lock);
if (orte_util_compare_name_fields(ORTE_NS_CMP_ALL, &peer->peer_name, &src) != OPAL_EQUAL) {
opal_hash_table_remove_value_uint64(&mca_oob_tcp_component.tcp_peers,
orte_util_hash_name(&peer->peer_name));
peer->peer_name = src;
opal_hash_table_set_value_uint64(&mca_oob_tcp_component.tcp_peers,
orte_util_hash_name(&peer->peer_name), peer);
}
OPAL_THREAD_UNLOCK(&mca_oob_tcp_component.tcp_lock);
}
/*
* Look for an existing SCTP process instance based on the globally unique
* process identifier.
*/
mca_btl_sctp_proc_t* mca_btl_sctp_proc_lookup(const orte_process_name_t *name)
{
mca_btl_sctp_proc_t* proc = NULL;
OPAL_THREAD_LOCK(&mca_btl_sctp_component.sctp_lock);
opal_hash_table_get_value_uint64(&mca_btl_sctp_component.sctp_procs,
orte_util_hash_name(name), (void**)&proc);
OPAL_THREAD_UNLOCK(&mca_btl_sctp_component.sctp_lock);
return proc;
}
mca_btl_sctp_proc_t* mca_btl_sctp_proc_create(ompi_proc_t* ompi_proc)
{
int rc;
size_t size;
mca_btl_sctp_proc_t* btl_proc;
uint64_t hash = orte_util_hash_name(&ompi_proc->proc_name);
OPAL_THREAD_LOCK(&mca_btl_sctp_component.sctp_lock);
rc = opal_hash_table_get_value_uint64(&mca_btl_sctp_component.sctp_procs,
hash, (void**)&btl_proc);
if(OMPI_SUCCESS == rc) {
OPAL_THREAD_UNLOCK(&mca_btl_sctp_component.sctp_lock);
return btl_proc;
}
btl_proc = OBJ_NEW(mca_btl_sctp_proc_t);
if(NULL == btl_proc) {
return NULL;
}
btl_proc->proc_ompi = ompi_proc;
btl_proc->proc_name = ompi_proc->proc_name;
/* add to hash table of all proc instance */
opal_hash_table_set_value_uint64(&mca_btl_sctp_component.sctp_procs,
hash, btl_proc);
OPAL_THREAD_UNLOCK(&mca_btl_sctp_component.sctp_lock);
/* lookup sctp parameters exported by this proc */
rc = ompi_modex_recv( &mca_btl_sctp_component.super.btl_version,
ompi_proc,
(void**)&btl_proc->proc_addrs,
&size );
if(rc != OMPI_SUCCESS) {
BTL_ERROR(("mca_base_modex_recv: failed with return value=%d", rc));
OBJ_RELEASE(btl_proc);
return NULL;
}
if(0 != (size % sizeof(mca_btl_sctp_addr_t))) {
BTL_ERROR(("mca_base_modex_recv: invalid size %" PRIsize_t "\n", size));
return NULL;
}
btl_proc->proc_addr_count = size / sizeof(mca_btl_sctp_addr_t);
/* allocate space for endpoint array - one for each exported address */
btl_proc->proc_endpoints = (mca_btl_base_endpoint_t**)
malloc(btl_proc->proc_addr_count * sizeof(mca_btl_base_endpoint_t*));
if(NULL == btl_proc->proc_endpoints) {
OBJ_RELEASE(btl_proc);
return NULL;
}
if(NULL == mca_btl_sctp_component.sctp_local && ompi_proc == ompi_proc_local()) {
mca_btl_sctp_component.sctp_local = btl_proc;
}
return btl_proc;
}
int mca_oob_ud_peer_lookup (const orte_process_name_t *name, mca_oob_ud_peer_t **peer) {
int rc;
*peer = NULL;
rc = opal_hash_table_get_value_uint64(&mca_oob_ud_component.ud_peers,
orte_util_hash_name(name), (void**)peer);
if (OPAL_SUCCESS != rc) {
return ORTE_ERR_UNREACH;
}
return ORTE_SUCCESS;
}
mca_oob_ud_peer_t *mca_oob_ud_get_peer (struct mca_oob_ud_port_t *port,
orte_process_name_t *name,
uint32_t qpn, uint32_t qkey,
uint16_t lid, uint8_t port_num)
{
struct ibv_ah_attr ah_attr;
mca_oob_ud_peer_t *peer;
int rc;
rc = mca_oob_ud_peer_lookup (name, &peer);
if (ORTE_SUCCESS == rc) {
OPAL_OUTPUT_VERBOSE((20, mca_oob_base_output, "%s oob:ud:peer_from_msg_hdr using "
"cached peer", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
return peer;
}
OPAL_OUTPUT_VERBOSE((10, mca_oob_base_output, "%s oob:ud:peer_from_msg_hdr creating "
"peer from return address", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
peer = OBJ_NEW(mca_oob_ud_peer_t);
if (NULL == peer) {
return NULL;
}
peer->peer_qpn = qpn;
peer->peer_qkey = qkey;
peer->peer_name = *name;
peer->peer_lid = lid;
peer->peer_port = port_num;
memset (&ah_attr, 0, sizeof (ah_attr));
ah_attr.dlid = peer->peer_lid;
ah_attr.port_num = peer->peer_port;
peer->peer_ah = ibv_create_ah (port->device->ib_pd, &ah_attr);
if (NULL == peer->peer_ah) {
free (peer);
return NULL;
}
peer->peer_context = port->device;
OPAL_THREAD_LOCK(&mca_oob_ud_component.ud_lock);
opal_hash_table_set_value_uint64(&mca_oob_ud_component.ud_peers,
orte_util_hash_name(name),
(void *) peer);
OPAL_THREAD_UNLOCK(&mca_oob_ud_component.ud_lock);
return peer;
}
void mca_btl_sctp_proc_destruct(mca_btl_sctp_proc_t* stcp_proc)
{
/* remove from list of all proc instances */
OPAL_THREAD_LOCK(&mca_btl_sctp_component.sctp_lock);
opal_hash_table_remove_value_uint64(&mca_btl_sctp_component.sctp_procs,
orte_util_hash_name(&stcp_proc->proc_name));
OPAL_THREAD_UNLOCK(&mca_btl_sctp_component.sctp_lock);
/* release resources */
if(NULL != stcp_proc->proc_endpoints) {
free(stcp_proc->proc_endpoints);
}
OBJ_DESTRUCT(&stcp_proc->proc_lock);
}
/**
* Find modex_proc_data_t container associated with given
* orte_process_name_t.
*
* The global lock should *NOT* be held when
* calling this function.
*/
static modex_proc_data_t*
modex_lookup_orte_proc(const orte_process_name_t *orte_proc)
{
modex_proc_data_t *proc_data = NULL;
OPAL_THREAD_LOCK(&mutex);
opal_hash_table_get_value_uint64(modex_data,
orte_util_hash_name(orte_proc), (void**)&proc_data);
if (NULL == proc_data) {
/* The proc clearly exists, so create a modex structure
for it */
proc_data = OBJ_NEW(modex_proc_data_t);
if (NULL == proc_data) {
opal_output(0, "grpcomm_basic_modex_lookup_orte_proc: unable to allocate modex_proc_data_t\n");
OPAL_THREAD_UNLOCK(&mutex);
return NULL;
}
opal_hash_table_set_value_uint64(modex_data,
orte_util_hash_name(orte_proc), proc_data);
}
OPAL_THREAD_UNLOCK(&mutex);
return proc_data;
}
mca_btl_tcp2_proc_t* mca_btl_tcp2_proc_create(ompi_proc_t* ompi_proc)
{
int rc;
size_t size;
mca_btl_tcp2_proc_t* btl_proc;
uint64_t hash = orte_util_hash_name(&ompi_proc->proc_name);
OPAL_THREAD_LOCK(&mca_btl_tcp2_component.tcp_lock);
rc = opal_hash_table_get_value_uint64(&mca_btl_tcp2_component.tcp_procs,
hash, (void**)&btl_proc);
if(OMPI_SUCCESS == rc) {
OPAL_THREAD_UNLOCK(&mca_btl_tcp2_component.tcp_lock);
return btl_proc;
}
btl_proc = OBJ_NEW(mca_btl_tcp2_proc_t);
if(NULL == btl_proc)
return NULL;
btl_proc->proc_ompi = ompi_proc;
/* add to hash table of all proc instance */
opal_hash_table_set_value_uint64(&mca_btl_tcp2_component.tcp_procs,
hash, btl_proc);
OPAL_THREAD_UNLOCK(&mca_btl_tcp2_component.tcp_lock);
/* lookup tcp parameters exported by this proc */
rc = ompi_modex_recv( &mca_btl_tcp2_component.super.btl_version,
ompi_proc,
(void**)&btl_proc->proc_addrs,
&size );
if(rc != OMPI_SUCCESS) {
BTL_ERROR(("mca_base_modex_recv: failed with return value=%d", rc));
OBJ_RELEASE(btl_proc);
return NULL;
}
if(0 != (size % sizeof(mca_btl_tcp2_addr_t))) {
BTL_ERROR(("mca_base_modex_recv: invalid size %lu: btl-size: %lu\n",
(unsigned long) size, (unsigned long)sizeof(mca_btl_tcp2_addr_t)));
return NULL;
}
btl_proc->proc_addr_count = size / sizeof(mca_btl_tcp2_addr_t);
/* allocate space for endpoint array - one for each exported address */
btl_proc->proc_endpoints = (mca_btl_base_endpoint_t**)
malloc((1 + btl_proc->proc_addr_count) *
sizeof(mca_btl_base_endpoint_t*));
if(NULL == btl_proc->proc_endpoints) {
OBJ_RELEASE(btl_proc);
return NULL;
}
if(NULL == mca_btl_tcp2_component.tcp_local && ompi_proc == ompi_proc_local()) {
mca_btl_tcp2_component.tcp_local = btl_proc;
}
{
/* convert the OMPI addr_family field to OS constants,
* so we can check for AF_INET (or AF_INET6) and don't have
* to deal with byte ordering anymore.
*/
unsigned int i;
for (i = 0; i < btl_proc->proc_addr_count; i++) {
if (MCA_BTL_TCP_AF_INET == btl_proc->proc_addrs[i].addr_family) {
btl_proc->proc_addrs[i].addr_family = AF_INET;
}
#if OPAL_WANT_IPV6
if (MCA_BTL_TCP_AF_INET6 == btl_proc->proc_addrs[i].addr_family) {
btl_proc->proc_addrs[i].addr_family = AF_INET6;
}
#endif
}
}
return btl_proc;
}
