static int publish_mr_info(void) { #ifndef ENABLE_MR_SCALABLE { int err; uint64_t heap_key, data_key; heap_key = fi_mr_key(shmem_transport_ofi_target_heap_mrfd); data_key = fi_mr_key(shmem_transport_ofi_target_data_mrfd); err = shmem_runtime_put("fi_heap_key", &heap_key, sizeof(uint64_t)); if (err) { OFI_ERRMSG("Error putting heap key to runtime KVS\n"); return 1; } err = shmem_runtime_put("fi_data_key", &data_key, sizeof(uint64_t)); if (err) { OFI_ERRMSG("Error putting data segment key to runtime KVS\n"); return 1; } } #ifndef ENABLE_REMOTE_VIRTUAL_ADDRESSING { int err; err = shmem_runtime_put("fi_heap_addr", &shmem_internal_heap_base, sizeof(uint8_t*)); if (err) { OFI_ERRMSG("Error putting heap address to runtime KVS\n"); return 1; } err = shmem_runtime_put("fi_data_addr", &shmem_internal_data_base, sizeof(uint8_t*)); if (err) { OFI_ERRMSG("Error putting data segment address to runtime KVS\n"); return 1; } } #endif /* ENABLE_REMOTE_VIRTUAL_ADDRESSING */ #endif /* ENABLE_MR_SCALABLE */ return 0; }
static inline int publish_av_info(struct fabric_info *info) { int ret = 0; char epname[128]; size_t epnamelen = sizeof(epname); #ifdef USE_ON_NODE_COMMS if(gethostname(myephostname, (EPHOSTNAMELEN - 1)) != 0) OFI_ERRMSG("gethostname error: %s \n", strerror(errno)); myephostname[EPHOSTNAMELEN-1] = '\0'; ret = shmem_runtime_put("fi_ephostname", myephostname, EPHOSTNAMELEN); if (ret != 0) { OFI_ERRMSG("shmem_runtime_put ephostname failed\n"); return ret; } #endif ret = fi_getname((fid_t)shmem_transport_ofi_epfd, epname, &epnamelen); if(ret!=0 || (epnamelen > sizeof(epname))){ OFI_ERRMSG("fi_getname failed\n"); return ret; } ret = shmem_runtime_put("fi_epname", epname, epnamelen); if (ret != 0) { OFI_ERRMSG("shmem_runtime_put epname failed\n"); return ret; } /* Note: we assume that the length of an address is the same for all * endpoints. This is safe for most HPC systems, but could be incorrect in * a heterogeneous context. */ shmem_transport_ofi_addrlen = epnamelen; return ret; }
int shmem_transport_xpmem_init(long eager_size) { long page_size = sysconf(_SC_PAGESIZE); char *base; size_t len; int ret; /* setup data region */ base = FIND_BASE(shmem_internal_data_base, page_size); len = FIND_LEN(shmem_internal_data_base, shmem_internal_data_length, page_size); my_info.data_seg = xpmem_make(base, len, XPMEM_PERMIT_MODE, (void*)0666); if (-1 == my_info.data_seg) { fprintf(stderr, "[%03d] ERROR: xpmem_make failed: %s\n", shmem_internal_my_pe, strerror(errno)); return 1; } my_info.data_off = (char*) shmem_internal_data_base - (char*) base; my_info.data_len = len; /* setup heap region */ base = FIND_BASE(shmem_internal_heap_base, page_size); len = FIND_LEN(shmem_internal_heap_base, shmem_internal_heap_length, page_size); my_info.heap_seg = xpmem_make(base, len, XPMEM_PERMIT_MODE, (void*)0666); if (-1 == my_info.heap_seg) { fprintf(stderr, "[%03d] ERROR: xpmem_make failed: %s\n", shmem_internal_my_pe, strerror(errno)); return 1; } my_info.heap_off = (char*) shmem_internal_heap_base - (char*) base; my_info.heap_len = len; ret = shmem_runtime_put("xpmem-segids", &my_info, sizeof(struct share_info_t)); if (0 != ret) { fprintf(stderr, "[%03d] ERROR: runtime_put failed: %d\n", shmem_internal_my_pe, ret); return 1; } return 0; }
int shmem_transport_init(long eager_size) { ptl_process_t my_id; int ret; ptl_ni_limits_t ni_req_limits; /* Initialize Portals */ ret = PtlInit(); if (PTL_OK != ret) { fprintf(stderr, "ERROR: PtlInit failed (%d), try setting PTL_IFACE_NAME\n", ret); return 1; } /* Initialize Mutexes */ SHMEM_MUTEX_INIT(shmem_internal_mutex_ptl4_pt_state); SHMEM_MUTEX_INIT(shmem_internal_mutex_ptl4_frag); SHMEM_MUTEX_INIT(shmem_internal_mutex_ptl4_event_slots); SHMEM_MUTEX_INIT(shmem_internal_mutex_ptl4_nb_fence); shmem_transport_portals4_bounce_buffer_size = eager_size; shmem_transport_portals4_bounce_buffers = shmem_free_list_init(sizeof(shmem_transport_portals4_bounce_buffer_t) + eager_size, init_bounce_buffer); shmem_transport_portals4_long_frags = shmem_free_list_init(sizeof(shmem_transport_portals4_long_frag_t), init_long_frag); /* Initialize network */ ni_req_limits.max_entries = 1024; ni_req_limits.max_unexpected_headers = 1024; ni_req_limits.max_mds = 1024; ni_req_limits.max_eqs = 1024; ni_req_limits.max_cts = 1024; ni_req_limits.max_pt_index = 64; ni_req_limits.max_iovecs = 1024; ni_req_limits.max_list_size = 1024; ni_req_limits.max_triggered_ops = 1024; ni_req_limits.max_msg_size = LONG_MAX; ni_req_limits.max_atomic_size = LONG_MAX; ni_req_limits.max_fetch_atomic_size = LONG_MAX; ni_req_limits.max_waw_ordered_size = LONG_MAX; ni_req_limits.max_war_ordered_size = LONG_MAX; ni_req_limits.max_volatile_size = LONG_MAX; ni_req_limits.features = 0; #ifdef ENABLE_REMOTE_VIRTUAL_ADDRESSING ni_req_limits.features |= PTL_TARGET_BIND_INACCESSIBLE; #endif #if WANT_TOTAL_DATA_ORDERING != 0 ni_req_limits.features |= PTL_TOTAL_DATA_ORDERING; #endif ret = PtlNIInit(PTL_IFACE_DEFAULT, PTL_NI_NO_MATCHING | PTL_NI_LOGICAL, PTL_PID_ANY, &ni_req_limits, &ni_limits, &shmem_transport_portals4_ni_h); if (PTL_OK != ret) { fprintf(stderr, "[%03d] ERROR: PtlNIInit failed: %d\n", shmem_internal_my_pe, ret); return ret; } #ifdef ENABLE_REMOTE_VIRTUAL_ADDRESSING if ((PTL_TARGET_BIND_INACCESSIBLE & ni_limits.features) == 0) { fprintf(stderr, "[%03d] ERROR: Remote virtual addressing feature enabled, but Portals\n" "doesn't support PTL_TARGET_BIND_INACCESSIBLE. Aborting.\n", shmem_internal_my_pe); } #endif #if WANT_TOTAL_DATA_ORDERING != 0 if ((PTL_TOTAL_DATA_ORDERING & ni_limits.features) == 0) { if (1 == WANT_TOTAL_DATA_ORDERING) { fprintf(stderr, "[%03d] ERROR: Total data ordering feature enabled, but Portals\n" "doesn't support PTL_TOTAL_DATA_ORDERING. Aborting.\n", shmem_internal_my_pe); } else { shmem_transport_portals4_total_data_ordering = 1; } } #endif /* Note that after this point, one should compare the macro PORTALS4_TOTAL_DATA_ORDERING to 0 /1 to determine if total data ordering is not / is enabled */ ret = PtlGetPhysId(shmem_transport_portals4_ni_h, &my_id); if (PTL_OK != ret) { fprintf(stderr, "[%03d] ERROR: PtlGetPhysId failed: %d\n", shmem_internal_my_pe, ret); return ret; } /* Share information */ ret = shmem_runtime_put("portals4-procid", &my_id, sizeof(my_id)); if (0 != ret) { fprintf(stderr, "[%03d] ERROR: runtime_put failed: %d\n", shmem_internal_my_pe, ret); return ret; } #ifdef ENABLE_REMOTE_VIRTUAL_ADDRESSING /* Make sure the heap and data bases are actually symmetric */ { uint64_t bases[2]; bases[0] = (uintptr_t) shmem_internal_heap_base; bases[1] = (uintptr_t) shmem_internal_data_base; ret = shmem_runtime_put("portals4-bases", bases, sizeof(uint64_t) * 2); if (0 != ret) { fprintf(stderr, "[%03d] ERROR: runtime_put failed: %d\n", shmem_internal_my_pe, ret); return ret; } } #endif return 0; }