int
shmem_transport_init(long eager_size)
{
ptl_process_t my_id;
int ret;
ptl_ni_limits_t ni_req_limits;
/* Initialize Portals */
ret = PtlInit();
if (PTL_OK != ret) {
fprintf(stderr, "ERROR: PtlInit failed (%d), try setting PTL_IFACE_NAME\n", ret);
return 1;
}
/* Initialize Mutexes */
SHMEM_MUTEX_INIT(shmem_internal_mutex_ptl4_pt_state);
SHMEM_MUTEX_INIT(shmem_internal_mutex_ptl4_frag);
SHMEM_MUTEX_INIT(shmem_internal_mutex_ptl4_event_slots);
SHMEM_MUTEX_INIT(shmem_internal_mutex_ptl4_nb_fence);
shmem_transport_portals4_bounce_buffer_size = eager_size;
shmem_transport_portals4_bounce_buffers =
shmem_free_list_init(sizeof(shmem_transport_portals4_bounce_buffer_t) + eager_size,
init_bounce_buffer);
shmem_transport_portals4_long_frags =
shmem_free_list_init(sizeof(shmem_transport_portals4_long_frag_t),
init_long_frag);
/* Initialize network */
ni_req_limits.max_entries = 1024;
ni_req_limits.max_unexpected_headers = 1024;
ni_req_limits.max_mds = 1024;
ni_req_limits.max_eqs = 1024;
ni_req_limits.max_cts = 1024;
ni_req_limits.max_pt_index = 64;
ni_req_limits.max_iovecs = 1024;
ni_req_limits.max_list_size = 1024;
ni_req_limits.max_triggered_ops = 1024;
ni_req_limits.max_msg_size = LONG_MAX;
ni_req_limits.max_atomic_size = LONG_MAX;
ni_req_limits.max_fetch_atomic_size = LONG_MAX;
ni_req_limits.max_waw_ordered_size = LONG_MAX;
ni_req_limits.max_war_ordered_size = LONG_MAX;
ni_req_limits.max_volatile_size = LONG_MAX;
ni_req_limits.features = 0;
#ifdef ENABLE_REMOTE_VIRTUAL_ADDRESSING
ni_req_limits.features |= PTL_TARGET_BIND_INACCESSIBLE;
#endif
#if WANT_TOTAL_DATA_ORDERING != 0
ni_req_limits.features |= PTL_TOTAL_DATA_ORDERING;
#endif
ret = PtlNIInit(PTL_IFACE_DEFAULT,
PTL_NI_NO_MATCHING | PTL_NI_LOGICAL,
PTL_PID_ANY,
&ni_req_limits,
&ni_limits,
&shmem_transport_portals4_ni_h);
if (PTL_OK != ret) {
fprintf(stderr, "[%03d] ERROR: PtlNIInit failed: %d\n",
shmem_internal_my_pe, ret);
return ret;
}
#ifdef ENABLE_REMOTE_VIRTUAL_ADDRESSING
if ((PTL_TARGET_BIND_INACCESSIBLE & ni_limits.features) == 0) {
fprintf(stderr,
"[%03d] ERROR: Remote virtual addressing feature enabled, but Portals\n"
"doesn't support PTL_TARGET_BIND_INACCESSIBLE. Aborting.\n",
shmem_internal_my_pe);
}
#endif
#if WANT_TOTAL_DATA_ORDERING != 0
if ((PTL_TOTAL_DATA_ORDERING & ni_limits.features) == 0) {
if (1 == WANT_TOTAL_DATA_ORDERING) {
fprintf(stderr,
"[%03d] ERROR: Total data ordering feature enabled, but Portals\n"
"doesn't support PTL_TOTAL_DATA_ORDERING. Aborting.\n",
shmem_internal_my_pe);
} else {
shmem_transport_portals4_total_data_ordering = 1;
}
}
#endif
/* Note that after this point, one should compare the macro
PORTALS4_TOTAL_DATA_ORDERING to 0 /1 to determine if total data
ordering is not / is enabled */
ret = PtlGetPhysId(shmem_transport_portals4_ni_h, &my_id);
if (PTL_OK != ret) {
fprintf(stderr, "[%03d] ERROR: PtlGetPhysId failed: %d\n",
shmem_internal_my_pe, ret);
return ret;
}
/* Share information */
ret = shmem_runtime_put("portals4-procid", &my_id, sizeof(my_id));
if (0 != ret) {
fprintf(stderr, "[%03d] ERROR: runtime_put failed: %d\n",
shmem_internal_my_pe, ret);
return ret;
}
#ifdef ENABLE_REMOTE_VIRTUAL_ADDRESSING
/* Make sure the heap and data bases are actually symmetric */
{
uint64_t bases[2];
bases[0] = (uintptr_t) shmem_internal_heap_base;
bases[1] = (uintptr_t) shmem_internal_data_base;
ret = shmem_runtime_put("portals4-bases", bases, sizeof(uint64_t) * 2);
if (0 != ret) {
fprintf(stderr, "[%03d] ERROR: runtime_put failed: %d\n",
shmem_internal_my_pe, ret);
return ret;
}
}
#endif
return 0;
}
int shmem_transport_init(long eager_size)
{
int ret = 0;
struct fabric_info info = {0};
info.npes = shmem_runtime_get_size();
info.prov_name = shmem_util_getenv_str("OFI_PROVIDER");
if (NULL == info.prov_name)
info.prov_name = shmem_util_getenv_str("OFI_USE_PROVIDER");
info.fabric_name = shmem_util_getenv_str("OFI_FABRIC");
info.domain_name = shmem_util_getenv_str("OFI_DOMAIN");
ret = query_for_fabric(&info);
if(ret!=0)
return ret;
SHMEM_MUTEX_INIT(shmem_transport_ofi_lock);
shmem_transport_ofi_bounce_buffer_size = eager_size;
//init LL for NB buffers
shmem_transport_ofi_bounce_buffers =
shmem_free_list_init(sizeof(shmem_transport_ofi_bounce_buffer_t)
+ eager_size, init_bounce_buffer);
ret = allocate_fabric_resources(&info);
if(ret!=0)
return ret;
ret = allocate_endpoints(&info);
if(ret!=0)
return ret;
ret = allocate_cntr_and_cq();
if(ret!=0)
return ret;
ret = bind_resources_to_and_enable_ep();
if(ret!=0)
return ret;
ret = allocate_recv_cntr_mr();
if(ret!=0)
return ret;
ret = publish_mr_info();
if (ret != 0)
return ret;
ret = atomic_limitations_check();
if(ret!=0)
return ret;
ret = publish_av_info(&info);
if(ret!=0)
return ret;
fi_freeinfo(info.fabrics);
return 0;
}
