shmem_free_list_t*
shmem_free_list_init(unsigned int element_size,
shmem_free_list_item_init_fn_t init_fn)
{
int ret;
shmem_free_list_t *fl = (shmem_free_list_t*) calloc(1, sizeof(shmem_free_list_t));
if (NULL == fl) return NULL;
fl->element_size = element_size;
fl->init_fn = init_fn;
SHMEM_MUTEX_INIT(fl->lock);
ret = shmem_free_list_more(fl);
if (0 != ret) return NULL;
return fl;
}
int oshmem_shmem_init(int argc, char **argv, int requested, int *provided)
{
int ret = OSHMEM_SUCCESS;
OMPI_TIMING_INIT(32);
if (!oshmem_shmem_initialized) {
ret = ompi_mpi_init(argc, argv, requested, provided, true);
OMPI_TIMING_NEXT("ompi_mpi_init");
if (OSHMEM_SUCCESS != ret) {
return ret;
}
PMPI_Comm_dup(MPI_COMM_WORLD, &oshmem_comm_world);
OMPI_TIMING_NEXT("PMPI_Comm_dup");
SHMEM_MUTEX_INIT(shmem_internal_mutex_alloc);
ret = _shmem_init(argc, argv, requested, provided);
OMPI_TIMING_NEXT("_shmem_init");
OMPI_TIMING_IMPORT_OPAL("mca_scoll_mpi_comm_query");
OMPI_TIMING_IMPORT_OPAL("mca_scoll_enable");
OMPI_TIMING_IMPORT_OPAL("mca_scoll_base_select");
if (OSHMEM_SUCCESS != ret) {
return ret;
}
oshmem_shmem_initialized = true;
if (OSHMEM_SUCCESS != shmem_lock_init()) {
SHMEM_API_ERROR( "shmem_lock_init() failed");
return OSHMEM_ERROR;
}
OMPI_TIMING_NEXT("shmem_lock_init");
/* this is a collective op, implies barrier */
MCA_MEMHEAP_CALL(get_all_mkeys());
OMPI_TIMING_NEXT("get_all_mkeys()");
oshmem_shmem_preconnect_all();
OMPI_TIMING_NEXT("shmem_preconnect_all");
#if OSHMEM_OPAL_THREAD_ENABLE
pthread_t thread_id;
int perr;
perr = pthread_create(&thread_id, NULL, &shmem_opal_thread, NULL);
if (0 != perr) {
SHMEM_API_ERROR("cannot create opal thread for SHMEM");
return OSHMEM_ERROR;
}
#endif
OMPI_TIMING_NEXT("THREAD_ENABLE");
}
#ifdef SIGUSR1
signal(SIGUSR1,sighandler__SIGUSR1);
signal(SIGTERM,sighandler__SIGTERM);
#endif
OMPI_TIMING_OUT;
OMPI_TIMING_FINALIZE;
return ret;
}
void
shmem_internal_init(int tl_requested, int *tl_provided)
{
int ret;
int radix = -1, crossover = -1;
long heap_size, eager_size;
int heap_use_malloc = 0;
int runtime_initialized = 0;
int transport_initialized = 0;
#ifdef USE_XPMEM
int xpmem_initialized = 0;
#endif
#ifdef USE_CMA
int cma_initialized = 0;
#endif
ret = shmem_runtime_init();
if (0 != ret) {
fprintf(stderr, "ERROR: runtime init failed: %d\n", ret);
goto cleanup;
}
runtime_initialized = 1;
shmem_internal_my_pe = shmem_runtime_get_rank();
shmem_internal_num_pes = shmem_runtime_get_size();
/* Ensure that the vendor string will not cause an overflow in user code */
if (sizeof(SHMEM_VENDOR_STRING) > SHMEM_MAX_NAME_LEN) {
fprintf(stderr,
"[%03d] ERROR: SHMEM_VENDOR_STRING length (%zu) exceeds SHMEM_MAX_NAME_LEN (%d)\n",
shmem_internal_my_pe, sizeof(SHMEM_VENDOR_STRING), SHMEM_MAX_NAME_LEN);
goto cleanup;
}
/* Process environment variables */
radix = shmem_util_getenv_long("COLL_RADIX", 0, 4);
crossover = shmem_util_getenv_long("COLL_CROSSOVER", 0, 4);
heap_size = shmem_util_getenv_long("SYMMETRIC_SIZE", 1, 512 * 1024 * 1024);
eager_size = shmem_util_getenv_long("BOUNCE_SIZE", 1, 2048);
heap_use_malloc = shmem_util_getenv_long("SYMMETRIC_HEAP_USE_MALLOC", 0, 0);
shmem_internal_debug = (NULL != shmem_util_getenv_str("DEBUG")) ? 1 : 0;
/* huge page support only on Linux for now, default is to use 2MB large pages */
#ifdef __linux__
if (heap_use_malloc == 0) {
shmem_internal_heap_use_huge_pages=
(shmem_util_getenv_str("SYMMETRIC_HEAP_USE_HUGE_PAGES") != NULL) ? 1 : 0;
shmem_internal_heap_huge_page_size = shmem_util_getenv_long("SYMMETRIC_HEAP_PAGE_SIZE",
1,
2 * 1024 * 1024);
}
#endif
/* Find symmetric data */
#ifdef __APPLE__
shmem_internal_data_base = (void*) get_etext();
shmem_internal_data_length = get_end() - get_etext();
#else
shmem_internal_data_base = &data_start;
shmem_internal_data_length = (unsigned long) &end - (unsigned long) &data_start;
#endif
#ifdef USE_ON_NODE_COMMS
shmem_internal_location_array = malloc(sizeof(char) * shmem_internal_num_pes);
if (NULL == shmem_internal_location_array) goto cleanup;
memset(shmem_internal_location_array, -1, shmem_internal_num_pes);
#endif
/* create symmetric heap */
ret = shmem_internal_symmetric_init(heap_size, heap_use_malloc);
if (0 != ret) {
fprintf(stderr,
"[%03d] ERROR: symmetric heap initialization failed: %d\n",
shmem_internal_my_pe, ret);
goto cleanup;
}
/* Initialize transport devices */
ret = shmem_transport_init(eager_size);
if (0 != ret) {
fprintf(stderr,
"[%03d] ERROR: Transport init failed\n",
shmem_internal_my_pe);
goto cleanup;
}
transport_initialized = 1;
#ifdef USE_XPMEM
ret = shmem_transport_xpmem_init(eager_size);
if (0 != ret) {
fprintf(stderr,
"[%03d] ERROR: XPMEM init failed\n",
shmem_internal_my_pe);
goto cleanup;
}
xpmem_initialized = 1;
#endif
#ifdef USE_CMA
shmem_transport_cma_put_max = shmem_util_getenv_long("CMA_PUT_MAX", 1, 8*1024);
shmem_transport_cma_get_max = shmem_util_getenv_long("CMA_GET_MAX", 1, 16*1024);
ret = shmem_transport_cma_init(eager_size);
if (0 != ret) {
fprintf(stderr,
"[%03d] ERROR: CMA init failed\n",
shmem_internal_my_pe);
goto cleanup;
}
cma_initialized = 1;
#endif
/* exchange information */
ret = shmem_runtime_exchange();
if (0 != ret) {
fprintf(stderr, "[%03d] ERROR: runtime exchange failed: %d\n",
shmem_internal_my_pe, ret);
goto cleanup;
}
/* finish transport initialization after information sharing. */
ret = shmem_transport_startup();
if (0 != ret) {
fprintf(stderr,
"[%03d] ERROR: Transport startup failed\n",
shmem_internal_my_pe);
goto cleanup;
}
#ifdef USE_XPMEM
ret = shmem_transport_xpmem_startup();
if (0 != ret) {
fprintf(stderr,
"[%03d] ERROR: XPMEM startup failed\n",
shmem_internal_my_pe);
goto cleanup;
}
#endif
#ifdef USE_CMA
ret = shmem_transport_cma_startup();
if (0 != ret) {
fprintf(stderr,
"[%03d] ERROR: CMA startup failed\n",
shmem_internal_my_pe);
goto cleanup;
}
#endif
ret = shmem_internal_collectives_init(crossover, radix);
if (ret != 0) {
fprintf(stderr,
"[%03d] ERROR: initialization of collectives failed: %d\n",
shmem_internal_my_pe, ret);
goto cleanup;
}
atexit(shmem_internal_shutdown_atexit);
shmem_internal_initialized = 1;
/* set up threading */
SHMEM_MUTEX_INIT(shmem_internal_mutex_alloc);
#ifdef ENABLE_THREADS
shmem_internal_thread_level = tl_requested;
*tl_provided = tl_requested;
#else
shmem_internal_thread_level = SHMEMX_THREAD_SINGLE;
*tl_provided = SHMEMX_THREAD_SINGLE;
#endif
/* get hostname for shmem_getnodename */
if (gethostname(shmem_internal_my_hostname,
sizeof(shmem_internal_my_hostname))) {
snprintf(shmem_internal_my_hostname,
sizeof(shmem_internal_my_hostname),
"ERR: gethostname '%s'?", strerror(errno));
}
/* last minute printing of information */
if (0 == shmem_internal_my_pe) {
if (NULL != shmem_util_getenv_str("VERSION")) {
printf(PACKAGE_STRING "\n");
fflush(NULL);
}
if (NULL != shmem_util_getenv_str("INFO")) {
printf(PACKAGE_STRING "\n\n");
printf("SMA_VERSION %s\n",
(NULL != shmem_util_getenv_str("VERSION")) ? "Set" : "Not set");
printf("\tIf set, print library version at startup\n");
printf("SMA_INFO %s\n",
(NULL != shmem_util_getenv_str("INFO")) ? "Set" : "Not set");
printf("\tIf set, print this help message at startup\n");
printf("SMA_SYMMETRIC_SIZE %ld\n", heap_size);
printf("\tSymmetric heap size\n");
printf("SMA_SYMMETRIC_HEAP_USE_MALLOC %s\n",
(0 != heap_use_malloc) ? "Set" : "Not set");
printf("\tIf set, allocate the symmetric heap using malloc\n");
if (heap_use_malloc == 0) {
printf("SMA_SYMMETRIC_HEAP_USE_HUGE_PAGES %s\n",
shmem_internal_heap_use_huge_pages ? "Yes" : "No");
if (shmem_internal_heap_use_huge_pages) {
printf("SMA_SYMMETRIC_HEAP_PAGE_SIZE %ld \n",
shmem_internal_heap_huge_page_size);
}
printf("\tSymmetric heap use large pages\n");
}
printf("SMA_COLL_CROSSOVER %d\n", crossover);
printf("\tCross-over between linear and tree collectives\n");
printf("SMA_COLL_RADIX %d\n", radix);
printf("\tRadix for tree-based collectives\n");
printf("SMA_BOUNCE_SIZE %ld\n", eager_size);
printf("\tMaximum message size to bounce buffer\n");
printf("SMA_BARRIER_ALGORITHM %s\n", coll_type_str[shmem_internal_barrier_type]);
printf("\tAlgorithm for barrier. Options are auto, linear, tree, dissem\n");
printf("SMA_BCAST_ALGORITHM %s\n", coll_type_str[shmem_internal_bcast_type]);
printf("\tAlgorithm for broadcast. Options are auto, linear, tree\n");
printf("SMA_REDUCE_ALGORITHM %s\n", coll_type_str[shmem_internal_reduce_type]);
printf("\tAlgorithm for reductions. Options are auto, linear, tree, recdbl\n");
printf("SMA_COLLECT_ALGORITHM %s\n", coll_type_str[shmem_internal_collect_type]);
printf("\tAlgorithm for collect. Options are auto, linear\n");
printf("SMA_FCOLLECT_ALGORITHM %s\n", coll_type_str[shmem_internal_fcollect_type]);
printf("\tAlgorithm for fcollect. Options are auto, linear, ring, recdbl\n");
#ifdef USE_CMA
printf("SMA_CMA_PUT_MAX %zu\n", shmem_transport_cma_put_max);
printf("SMA_CMA_GET_MAX %zu\n", shmem_transport_cma_get_max);
#endif /* USE_CMA */
printf("SMA_DEBUG %s\n", (NULL != shmem_util_getenv_str("DEBUG")) ? "On" : "Off");
printf("\tEnable debugging messages.\n");
shmem_transport_print_info();
printf("\n");
fflush(NULL);
}
}
/* finish up */
shmem_runtime_barrier();
return;
cleanup:
if (transport_initialized) {
shmem_transport_fini();
}
#ifdef USE_XPMEM
if (xpmem_initialized) {
shmem_transport_xpmem_fini();
}
#endif
#ifdef USE_CMA
if (cma_initialized) {
shmem_transport_cma_fini();
}
#endif
if (NULL != shmem_internal_data_base) {
shmem_internal_symmetric_fini();
}
if (runtime_initialized) {
shmem_runtime_fini();
}
abort();
}
int
shmem_transport_init(long eager_size)
{
ptl_process_t my_id;
int ret;
ptl_ni_limits_t ni_req_limits;
/* Initialize Portals */
ret = PtlInit();
if (PTL_OK != ret) {
fprintf(stderr, "ERROR: PtlInit failed (%d), try setting PTL_IFACE_NAME\n", ret);
return 1;
}
/* Initialize Mutexes */
SHMEM_MUTEX_INIT(shmem_internal_mutex_ptl4_pt_state);
SHMEM_MUTEX_INIT(shmem_internal_mutex_ptl4_frag);
SHMEM_MUTEX_INIT(shmem_internal_mutex_ptl4_event_slots);
SHMEM_MUTEX_INIT(shmem_internal_mutex_ptl4_nb_fence);
shmem_transport_portals4_bounce_buffer_size = eager_size;
shmem_transport_portals4_bounce_buffers =
shmem_free_list_init(sizeof(shmem_transport_portals4_bounce_buffer_t) + eager_size,
init_bounce_buffer);
shmem_transport_portals4_long_frags =
shmem_free_list_init(sizeof(shmem_transport_portals4_long_frag_t),
init_long_frag);
/* Initialize network */
ni_req_limits.max_entries = 1024;
ni_req_limits.max_unexpected_headers = 1024;
ni_req_limits.max_mds = 1024;
ni_req_limits.max_eqs = 1024;
ni_req_limits.max_cts = 1024;
ni_req_limits.max_pt_index = 64;
ni_req_limits.max_iovecs = 1024;
ni_req_limits.max_list_size = 1024;
ni_req_limits.max_triggered_ops = 1024;
ni_req_limits.max_msg_size = LONG_MAX;
ni_req_limits.max_atomic_size = LONG_MAX;
ni_req_limits.max_fetch_atomic_size = LONG_MAX;
ni_req_limits.max_waw_ordered_size = LONG_MAX;
ni_req_limits.max_war_ordered_size = LONG_MAX;
ni_req_limits.max_volatile_size = LONG_MAX;
ni_req_limits.features = 0;
#ifdef ENABLE_REMOTE_VIRTUAL_ADDRESSING
ni_req_limits.features |= PTL_TARGET_BIND_INACCESSIBLE;
#endif
#if WANT_TOTAL_DATA_ORDERING != 0
ni_req_limits.features |= PTL_TOTAL_DATA_ORDERING;
#endif
ret = PtlNIInit(PTL_IFACE_DEFAULT,
PTL_NI_NO_MATCHING | PTL_NI_LOGICAL,
PTL_PID_ANY,
&ni_req_limits,
&ni_limits,
&shmem_transport_portals4_ni_h);
if (PTL_OK != ret) {
fprintf(stderr, "[%03d] ERROR: PtlNIInit failed: %d\n",
shmem_internal_my_pe, ret);
return ret;
}
#ifdef ENABLE_REMOTE_VIRTUAL_ADDRESSING
if ((PTL_TARGET_BIND_INACCESSIBLE & ni_limits.features) == 0) {
fprintf(stderr,
"[%03d] ERROR: Remote virtual addressing feature enabled, but Portals\n"
"doesn't support PTL_TARGET_BIND_INACCESSIBLE. Aborting.\n",
shmem_internal_my_pe);
}
#endif
#if WANT_TOTAL_DATA_ORDERING != 0
if ((PTL_TOTAL_DATA_ORDERING & ni_limits.features) == 0) {
if (1 == WANT_TOTAL_DATA_ORDERING) {
fprintf(stderr,
"[%03d] ERROR: Total data ordering feature enabled, but Portals\n"
"doesn't support PTL_TOTAL_DATA_ORDERING. Aborting.\n",
shmem_internal_my_pe);
} else {
shmem_transport_portals4_total_data_ordering = 1;
}
}
#endif
/* Note that after this point, one should compare the macro
PORTALS4_TOTAL_DATA_ORDERING to 0 /1 to determine if total data
ordering is not / is enabled */
ret = PtlGetPhysId(shmem_transport_portals4_ni_h, &my_id);
if (PTL_OK != ret) {
fprintf(stderr, "[%03d] ERROR: PtlGetPhysId failed: %d\n",
shmem_internal_my_pe, ret);
return ret;
}
/* Share information */
ret = shmem_runtime_put("portals4-procid", &my_id, sizeof(my_id));
if (0 != ret) {
fprintf(stderr, "[%03d] ERROR: runtime_put failed: %d\n",
shmem_internal_my_pe, ret);
return ret;
}
#ifdef ENABLE_REMOTE_VIRTUAL_ADDRESSING
/* Make sure the heap and data bases are actually symmetric */
{
uint64_t bases[2];
bases[0] = (uintptr_t) shmem_internal_heap_base;
bases[1] = (uintptr_t) shmem_internal_data_base;
ret = shmem_runtime_put("portals4-bases", bases, sizeof(uint64_t) * 2);
if (0 != ret) {
fprintf(stderr, "[%03d] ERROR: runtime_put failed: %d\n",
shmem_internal_my_pe, ret);
return ret;
}
}
#endif
return 0;
}
int shmem_transport_init(long eager_size)
{
int ret = 0;
struct fabric_info info = {0};
info.npes = shmem_runtime_get_size();
info.prov_name = shmem_util_getenv_str("OFI_PROVIDER");
if (NULL == info.prov_name)
info.prov_name = shmem_util_getenv_str("OFI_USE_PROVIDER");
info.fabric_name = shmem_util_getenv_str("OFI_FABRIC");
info.domain_name = shmem_util_getenv_str("OFI_DOMAIN");
ret = query_for_fabric(&info);
if(ret!=0)
return ret;
SHMEM_MUTEX_INIT(shmem_transport_ofi_lock);
shmem_transport_ofi_bounce_buffer_size = eager_size;
//init LL for NB buffers
shmem_transport_ofi_bounce_buffers =
shmem_free_list_init(sizeof(shmem_transport_ofi_bounce_buffer_t)
+ eager_size, init_bounce_buffer);
ret = allocate_fabric_resources(&info);
if(ret!=0)
return ret;
ret = allocate_endpoints(&info);
if(ret!=0)
return ret;
ret = allocate_cntr_and_cq();
if(ret!=0)
return ret;
ret = bind_resources_to_and_enable_ep();
if(ret!=0)
return ret;
ret = allocate_recv_cntr_mr();
if(ret!=0)
return ret;
ret = publish_mr_info();
if (ret != 0)
return ret;
ret = atomic_limitations_check();
if(ret!=0)
return ret;
ret = publish_av_info(&info);
if(ret!=0)
return ret;
fi_freeinfo(info.fabrics);
return 0;
}
