static int publish_mr_info(void)
{
#ifndef ENABLE_MR_SCALABLE
{
int err;
uint64_t heap_key, data_key;
heap_key = fi_mr_key(shmem_transport_ofi_target_heap_mrfd);
data_key = fi_mr_key(shmem_transport_ofi_target_data_mrfd);
err = shmem_runtime_put("fi_heap_key", &heap_key, sizeof(uint64_t));
if (err) {
OFI_ERRMSG("Error putting heap key to runtime KVS\n");
return 1;
}
err = shmem_runtime_put("fi_data_key", &data_key, sizeof(uint64_t));
if (err) {
OFI_ERRMSG("Error putting data segment key to runtime KVS\n");
return 1;
}
}
#ifndef ENABLE_REMOTE_VIRTUAL_ADDRESSING
{
int err;
err = shmem_runtime_put("fi_heap_addr", &shmem_internal_heap_base, sizeof(uint8_t*));
if (err) {
OFI_ERRMSG("Error putting heap address to runtime KVS\n");
return 1;
}
err = shmem_runtime_put("fi_data_addr", &shmem_internal_data_base, sizeof(uint8_t*));
if (err) {
OFI_ERRMSG("Error putting data segment address to runtime KVS\n");
return 1;
}
}
#endif /* ENABLE_REMOTE_VIRTUAL_ADDRESSING */
#endif /* ENABLE_MR_SCALABLE */
return 0;
}
static inline int publish_av_info(struct fabric_info *info)
{
int ret = 0;
char epname[128];
size_t epnamelen = sizeof(epname);
#ifdef USE_ON_NODE_COMMS
if(gethostname(myephostname, (EPHOSTNAMELEN - 1)) != 0)
OFI_ERRMSG("gethostname error: %s \n", strerror(errno));
myephostname[EPHOSTNAMELEN-1] = '\0';
ret = shmem_runtime_put("fi_ephostname", myephostname, EPHOSTNAMELEN);
if (ret != 0) {
OFI_ERRMSG("shmem_runtime_put ephostname failed\n");
return ret;
}
#endif
ret = fi_getname((fid_t)shmem_transport_ofi_epfd, epname, &epnamelen);
if(ret!=0 || (epnamelen > sizeof(epname))){
OFI_ERRMSG("fi_getname failed\n");
return ret;
}
ret = shmem_runtime_put("fi_epname", epname, epnamelen);
if (ret != 0) {
OFI_ERRMSG("shmem_runtime_put epname failed\n");
return ret;
}
/* Note: we assume that the length of an address is the same for all
* endpoints. This is safe for most HPC systems, but could be incorrect in
* a heterogeneous context. */
shmem_transport_ofi_addrlen = epnamelen;
return ret;
}
int
shmem_transport_xpmem_init(long eager_size)
{
long page_size = sysconf(_SC_PAGESIZE);
char *base;
size_t len;
int ret;
/* setup data region */
base = FIND_BASE(shmem_internal_data_base, page_size);
len = FIND_LEN(shmem_internal_data_base, shmem_internal_data_length, page_size);
my_info.data_seg = xpmem_make(base, len, XPMEM_PERMIT_MODE, (void*)0666);
if (-1 == my_info.data_seg) {
fprintf(stderr, "[%03d] ERROR: xpmem_make failed: %s\n",
shmem_internal_my_pe, strerror(errno));
return 1;
}
my_info.data_off = (char*) shmem_internal_data_base - (char*) base;
my_info.data_len = len;
/* setup heap region */
base = FIND_BASE(shmem_internal_heap_base, page_size);
len = FIND_LEN(shmem_internal_heap_base, shmem_internal_heap_length, page_size);
my_info.heap_seg = xpmem_make(base, len, XPMEM_PERMIT_MODE, (void*)0666);
if (-1 == my_info.heap_seg) {
fprintf(stderr, "[%03d] ERROR: xpmem_make failed: %s\n",
shmem_internal_my_pe, strerror(errno));
return 1;
}
my_info.heap_off = (char*) shmem_internal_heap_base - (char*) base;
my_info.heap_len = len;
ret = shmem_runtime_put("xpmem-segids", &my_info, sizeof(struct share_info_t));
if (0 != ret) {
fprintf(stderr, "[%03d] ERROR: runtime_put failed: %d\n",
shmem_internal_my_pe, ret);
return 1;
}
return 0;
}
int
shmem_transport_init(long eager_size)
{
ptl_process_t my_id;
int ret;
ptl_ni_limits_t ni_req_limits;
/* Initialize Portals */
ret = PtlInit();
if (PTL_OK != ret) {
fprintf(stderr, "ERROR: PtlInit failed (%d), try setting PTL_IFACE_NAME\n", ret);
return 1;
}
/* Initialize Mutexes */
SHMEM_MUTEX_INIT(shmem_internal_mutex_ptl4_pt_state);
SHMEM_MUTEX_INIT(shmem_internal_mutex_ptl4_frag);
SHMEM_MUTEX_INIT(shmem_internal_mutex_ptl4_event_slots);
SHMEM_MUTEX_INIT(shmem_internal_mutex_ptl4_nb_fence);
shmem_transport_portals4_bounce_buffer_size = eager_size;
shmem_transport_portals4_bounce_buffers =
shmem_free_list_init(sizeof(shmem_transport_portals4_bounce_buffer_t) + eager_size,
init_bounce_buffer);
shmem_transport_portals4_long_frags =
shmem_free_list_init(sizeof(shmem_transport_portals4_long_frag_t),
init_long_frag);
/* Initialize network */
ni_req_limits.max_entries = 1024;
ni_req_limits.max_unexpected_headers = 1024;
ni_req_limits.max_mds = 1024;
ni_req_limits.max_eqs = 1024;
ni_req_limits.max_cts = 1024;
ni_req_limits.max_pt_index = 64;
ni_req_limits.max_iovecs = 1024;
ni_req_limits.max_list_size = 1024;
ni_req_limits.max_triggered_ops = 1024;
ni_req_limits.max_msg_size = LONG_MAX;
ni_req_limits.max_atomic_size = LONG_MAX;
ni_req_limits.max_fetch_atomic_size = LONG_MAX;
ni_req_limits.max_waw_ordered_size = LONG_MAX;
ni_req_limits.max_war_ordered_size = LONG_MAX;
ni_req_limits.max_volatile_size = LONG_MAX;
ni_req_limits.features = 0;
#ifdef ENABLE_REMOTE_VIRTUAL_ADDRESSING
ni_req_limits.features |= PTL_TARGET_BIND_INACCESSIBLE;
#endif
#if WANT_TOTAL_DATA_ORDERING != 0
ni_req_limits.features |= PTL_TOTAL_DATA_ORDERING;
#endif
ret = PtlNIInit(PTL_IFACE_DEFAULT,
PTL_NI_NO_MATCHING | PTL_NI_LOGICAL,
PTL_PID_ANY,
&ni_req_limits,
&ni_limits,
&shmem_transport_portals4_ni_h);
if (PTL_OK != ret) {
fprintf(stderr, "[%03d] ERROR: PtlNIInit failed: %d\n",
shmem_internal_my_pe, ret);
return ret;
}
#ifdef ENABLE_REMOTE_VIRTUAL_ADDRESSING
if ((PTL_TARGET_BIND_INACCESSIBLE & ni_limits.features) == 0) {
fprintf(stderr,
"[%03d] ERROR: Remote virtual addressing feature enabled, but Portals\n"
"doesn't support PTL_TARGET_BIND_INACCESSIBLE. Aborting.\n",
shmem_internal_my_pe);
}
#endif
#if WANT_TOTAL_DATA_ORDERING != 0
if ((PTL_TOTAL_DATA_ORDERING & ni_limits.features) == 0) {
if (1 == WANT_TOTAL_DATA_ORDERING) {
fprintf(stderr,
"[%03d] ERROR: Total data ordering feature enabled, but Portals\n"
"doesn't support PTL_TOTAL_DATA_ORDERING. Aborting.\n",
shmem_internal_my_pe);
} else {
shmem_transport_portals4_total_data_ordering = 1;
}
}
#endif
/* Note that after this point, one should compare the macro
PORTALS4_TOTAL_DATA_ORDERING to 0 /1 to determine if total data
ordering is not / is enabled */
ret = PtlGetPhysId(shmem_transport_portals4_ni_h, &my_id);
if (PTL_OK != ret) {
fprintf(stderr, "[%03d] ERROR: PtlGetPhysId failed: %d\n",
shmem_internal_my_pe, ret);
return ret;
}
/* Share information */
ret = shmem_runtime_put("portals4-procid", &my_id, sizeof(my_id));
if (0 != ret) {
fprintf(stderr, "[%03d] ERROR: runtime_put failed: %d\n",
shmem_internal_my_pe, ret);
return ret;
}
#ifdef ENABLE_REMOTE_VIRTUAL_ADDRESSING
/* Make sure the heap and data bases are actually symmetric */
{
uint64_t bases[2];
bases[0] = (uintptr_t) shmem_internal_heap_base;
bases[1] = (uintptr_t) shmem_internal_data_base;
ret = shmem_runtime_put("portals4-bases", bases, sizeof(uint64_t) * 2);
if (0 != ret) {
fprintf(stderr, "[%03d] ERROR: runtime_put failed: %d\n",
shmem_internal_my_pe, ret);
return ret;
}
}
#endif
return 0;
}
