int mca_spml_ikrit_enable(bool enable)
{
SPML_VERBOSE(50, "*** ikrit ENABLED ****");
if (false == enable) {
return OSHMEM_SUCCESS;
}
opal_free_list_init (&mca_spml_base_put_requests,
sizeof(mca_spml_ikrit_put_request_t),
opal_cache_line_size,
OBJ_CLASS(opal_free_list_item_t),
0,
opal_cache_line_size,
mca_spml_ikrit.free_list_num,
mca_spml_ikrit.free_list_max,
mca_spml_ikrit.free_list_inc,
NULL, 0, NULL, NULL, NULL);
opal_free_list_init (&mca_spml_base_get_requests,
sizeof(mca_spml_ikrit_get_request_t),
opal_cache_line_size,
OBJ_CLASS(opal_free_list_item_t),
0,
opal_cache_line_size,
mca_spml_ikrit.free_list_num,
mca_spml_ikrit.free_list_max,
mca_spml_ikrit.free_list_inc,
NULL, 0, NULL, NULL, NULL);
mca_spml_ikrit.enabled = true;
return OSHMEM_SUCCESS;
}
int mca_spml_yoda_enable(bool enable)
{
SPML_VERBOSE(50, "*** yoda ENABLED ****");
if (false == enable) {
return OSHMEM_SUCCESS;
}
OBJ_CONSTRUCT(&mca_spml_yoda.lock, opal_mutex_t);
/**
*If we get here this is the SPML who get selected for the run. We
* should get ownership for the put and get requests list, and
* initialize them with the size of our own requests.
*/
opal_free_list_init (&mca_spml_base_put_requests,
sizeof(mca_spml_yoda_put_request_t),
opal_cache_line_size,
OBJ_CLASS(mca_spml_yoda_put_request_t),
0,
opal_cache_line_size,
mca_spml_yoda.free_list_num,
mca_spml_yoda.free_list_max,
mca_spml_yoda.free_list_inc,
NULL, 0, NULL, NULL, NULL);
opal_free_list_init (&mca_spml_base_get_requests,
sizeof(mca_spml_yoda_get_request_t),
opal_cache_line_size,
OBJ_CLASS(mca_spml_yoda_get_request_t),
0,
opal_cache_line_size,
mca_spml_yoda.free_list_num,
mca_spml_yoda.free_list_max,
mca_spml_yoda.free_list_inc,
NULL, 0, NULL, NULL, NULL);
mca_spml_yoda.enabled = true;
/* The following line resolves the issue with BTL tcp and SPML yoda. In this case the
* atomic_basic_lock(root_rank) function may behave as DoS attack on root_rank, since
* all the procceses will do shmem_int_get from root_rank. These calls would go through
* bml active messaging and will trigger replays in libevent on root rank. If the flag
* OPAL_ENVLOOP_ONCE is not set then libevent will continously progress constantly
* incoming events thus causing root_rank to stuck in libevent loop.
*/
opal_progress_set_event_flag(OPAL_EVLOOP_NONBLOCK | OPAL_EVLOOP_ONCE);
#if OSHMEM_WAIT_COMPLETION_DEBUG == 1
condition_dbg_init();
#endif
return OSHMEM_SUCCESS;
}
static int
mca_btl_scif_setup_mpools (mca_btl_scif_module_t *scif_module)
{
struct mca_mpool_base_resources_t mpool_resources;
int rc;
/* initialize the grdma mpool */
mpool_resources.pool_name = "scif";
mpool_resources.reg_data = (void *) scif_module;
mpool_resources.sizeof_reg = sizeof (mca_btl_scif_reg_t);
mpool_resources.register_mem = scif_reg_mem;
mpool_resources.deregister_mem = scif_dereg_mem;
scif_module->super.btl_mpool =
mca_mpool_base_module_create("grdma", scif_module, &mpool_resources);
if (NULL == scif_module->super.btl_mpool) {
BTL_ERROR(("error creating grdma mpool"));
return OPAL_ERROR;
}
/* setup free lists for fragments. dma fragments will be used for
* rma operations and in-place sends. eager frags will be used for
* buffered sends. */
rc = opal_free_list_init (&scif_module->dma_frags,
sizeof (mca_btl_scif_dma_frag_t), 64,
OBJ_CLASS(mca_btl_scif_dma_frag_t),
128, opal_getpagesize (),
mca_btl_scif_component.scif_free_list_num,
mca_btl_scif_component.scif_free_list_max,
mca_btl_scif_component.scif_free_list_inc,
NULL, 0, NULL, NULL, NULL);
if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
return rc;
}
rc = opal_free_list_init (&scif_module->eager_frags,
sizeof (mca_btl_scif_eager_frag_t), 8,
OBJ_CLASS(mca_btl_scif_eager_frag_t),
128 + scif_module->super.btl_eager_limit, 64,
mca_btl_scif_component.scif_free_list_num,
mca_btl_scif_component.scif_free_list_max,
mca_btl_scif_component.scif_free_list_inc,
NULL, 0, NULL, NULL, NULL);
if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
BTL_ERROR(("error creating eager receive fragment free list"));
return rc;
}
return OPAL_SUCCESS;
}
/*
* Initializes the mpool module.
*/
void mca_mpool_rgpusm_module_init(mca_mpool_rgpusm_module_t* mpool)
{
mpool->super.mpool_component = &mca_mpool_rgpusm_component.super;
mpool->super.mpool_base = NULL; /* no base .. */
mpool->super.mpool_alloc = NULL;
mpool->super.mpool_realloc = NULL;
mpool->super.mpool_free = mca_mpool_rgpusm_free;
mpool->super.mpool_register = mca_mpool_rgpusm_register;
mpool->super.mpool_find = mca_mpool_rgpusm_find;
mpool->super.mpool_deregister = mca_mpool_rgpusm_deregister;
mpool->super.mpool_release_memory = NULL;
mpool->super.mpool_finalize = mca_mpool_rgpusm_finalize;
mpool->super.mpool_ft_event = mca_mpool_rgpusm_ft_event;
mpool->super.rcache =
mca_rcache_base_module_create(mca_mpool_rgpusm_component.rcache_name);
mpool->super.flags = 0;
mpool->resources.reg_data = NULL;
mpool->resources.sizeof_reg = sizeof(struct mca_mpool_common_cuda_reg_t);
mpool->resources.register_mem = cuda_openmemhandle;
mpool->resources.deregister_mem = cuda_closememhandle;
OBJ_CONSTRUCT(&mpool->reg_list, opal_free_list_t);
opal_free_list_init (&mpool->reg_list, mpool->resources.sizeof_reg,
opal_cache_line_size,
OBJ_CLASS(mca_mpool_base_registration_t),
0,opal_cache_line_size,
0, -1, 32, NULL, 0, NULL, NULL, NULL);
OBJ_CONSTRUCT(&mpool->lru_list, opal_list_t);
mpool->stat_cache_hit = mpool->stat_cache_miss = mpool->stat_evicted = 0;
mpool->stat_cache_found = mpool->stat_cache_notfound = 0;
mpool->stat_cache_valid = mpool->stat_cache_invalid = 0;
}
static void initFreeList(void)
{
mca_base_var_register("oshmem", "shmem", "java", "eager",
"Java buffers eager size",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_5,
MCA_BASE_VAR_SCOPE_READONLY,
&oshmem_shmem_java_eager);
OBJ_CONSTRUCT(&shmem_java_buffers, opal_free_list_t);
int r = opal_free_list_init(&shmem_java_buffers,
sizeof(shmem_java_buffer_t),
opal_cache_line_size,
OBJ_CLASS(shmem_java_buffer_t),
0, /* payload size */
0, /* payload align */
2, /* initial elements to alloc */
-1, /* max elements */
2, /* num elements per alloc */
NULL, /* mpool */
0, /* mpool reg flags */
NULL, /* unused0 */
NULL, /* item_init */
NULL /* inem_init context */);
if(r != OPAL_SUCCESS)
{
fprintf(stderr, "Unable to initialize shmem_java_buffers.\n");
exit(1);
}
}
static int
portals4_open(void)
{
int ret;
mca_coll_portals4_component.ni_h = PTL_INVALID_HANDLE;
mca_coll_portals4_component.uid = PTL_UID_ANY;
mca_coll_portals4_component.pt_idx = -1;
mca_coll_portals4_component.finish_pt_idx = -1;
mca_coll_portals4_component.eq_h = PTL_INVALID_HANDLE;
mca_coll_portals4_component.unex_me_h = PTL_INVALID_HANDLE;
mca_coll_portals4_component.finish_me_h = PTL_INVALID_HANDLE;
mca_coll_portals4_component.zero_md_h = PTL_INVALID_HANDLE;
mca_coll_portals4_component.data_md_h = PTL_INVALID_HANDLE;
OBJ_CONSTRUCT(&mca_coll_portals4_component.requests, opal_free_list_t);
ret = opal_free_list_init(&mca_coll_portals4_component.requests,
sizeof(ompi_coll_portals4_request_t),
opal_cache_line_size,
OBJ_CLASS(ompi_coll_portals4_request_t),
0, 0, 8, 0, 8, NULL, 0, NULL, NULL, NULL);
if (OMPI_SUCCESS != ret) {
opal_output_verbose(1, ompi_coll_base_framework.framework_output,
"%s:%d: ompi_free_list_init failed: %d\n",
__FILE__, __LINE__, ret);
return ret;
}
return OMPI_SUCCESS;
}
/*
* Initializes the mpool module.
*/
void mca_mpool_grdma_module_init(mca_mpool_grdma_module_t* mpool, mca_mpool_grdma_pool_t *pool)
{
OBJ_RETAIN(pool);
mpool->pool = pool;
mpool->super.mpool_component = &mca_mpool_grdma_component.super;
mpool->super.mpool_base = NULL; /* no base .. */
mpool->super.mpool_alloc = mca_mpool_grdma_alloc;
mpool->super.mpool_realloc = mca_mpool_grdma_realloc;
mpool->super.mpool_free = mca_mpool_grdma_free;
mpool->super.mpool_register = mca_mpool_grdma_register;
mpool->super.mpool_find = mca_mpool_grdma_find;
mpool->super.mpool_deregister = mca_mpool_grdma_deregister;
mpool->super.mpool_release_memory = mca_mpool_grdma_release_memory;
mpool->super.mpool_finalize = mca_mpool_grdma_finalize;
mpool->super.mpool_ft_event = mca_mpool_grdma_ft_event;
mpool->super.flags = MCA_MPOOL_FLAGS_MPI_ALLOC_MEM;
mpool->super.rcache = pool->rcache;
mpool->stat_cache_hit = mpool->stat_cache_miss = mpool->stat_evicted = 0;
mpool->stat_cache_found = mpool->stat_cache_notfound = 0;
OBJ_CONSTRUCT(&mpool->reg_list, opal_free_list_t);
opal_free_list_init (&mpool->reg_list, mpool->resources.sizeof_reg,
opal_cache_line_size,
OBJ_CLASS(mca_mpool_base_registration_t),
0, opal_cache_line_size, 0, -1, 32, NULL, 0,
NULL, NULL, NULL);
}
/**
* the constructor function. creates the free list to get the nodes from
*
* @param object the tree that is to be used
*
* @retval NONE
*/
static void opal_rb_tree_construct(opal_object_t * object)
{
opal_rb_tree_t * tree = (opal_rb_tree_t *) object;
tree->root_ptr = NULL;
OBJ_CONSTRUCT(&(tree->free_list), opal_free_list_t);
opal_free_list_init (&(tree->free_list), sizeof(opal_rb_tree_node_t),
opal_cache_line_size, OBJ_CLASS(opal_rb_tree_node_t),
0,opal_cache_line_size, 0, -1 , 128, NULL, 0, NULL, NULL, NULL);
}
static void initFreeList(void)
{
OBJ_CONSTRUCT(&ompi_java_buffers, opal_free_list_t);
int r = opal_free_list_init(&ompi_java_buffers, sizeof(ompi_java_buffer_t),
OBJ_CLASS(ompi_java_buffer_t), 2, -1, 2);
if(r != OPAL_SUCCESS)
{
fprintf(stderr, "Unable to initialize ompi_java_buffers.\n");
exit(1);
}
}
int
ompi_osc_pt2pt_component_init(bool enable_progress_threads,
bool enable_mpi_threads)
{
size_t aligned_size;
OBJ_CONSTRUCT(&mca_osc_pt2pt_component.p2p_c_sendreqs, opal_free_list_t);
opal_free_list_init(&mca_osc_pt2pt_component.p2p_c_sendreqs,
sizeof(ompi_osc_pt2pt_sendreq_t),
OBJ_CLASS(ompi_osc_pt2pt_sendreq_t),
1, -1, 1);
OBJ_CONSTRUCT(&mca_osc_pt2pt_component.p2p_c_replyreqs, opal_free_list_t);
opal_free_list_init(&mca_osc_pt2pt_component.p2p_c_replyreqs,
sizeof(ompi_osc_pt2pt_replyreq_t),
OBJ_CLASS(ompi_osc_pt2pt_replyreq_t),
1, -1, 1);
OBJ_CONSTRUCT(&mca_osc_pt2pt_component.p2p_c_longreqs, opal_free_list_t);
opal_free_list_init(&mca_osc_pt2pt_component.p2p_c_longreqs,
sizeof(ompi_osc_pt2pt_longreq_t),
OBJ_CLASS(ompi_osc_pt2pt_longreq_t),
1, -1, 1);
/* adjust size to be multiple of ompi_ptr_t to avoid alignment issues*/
aligned_size = sizeof(ompi_osc_pt2pt_buffer_t) +
(sizeof(ompi_osc_pt2pt_buffer_t) % sizeof(ompi_ptr_t)) +
mca_osc_pt2pt_component.p2p_c_eager_size;
OBJ_CONSTRUCT(&mca_osc_pt2pt_component.p2p_c_buffers, opal_free_list_t);
opal_free_list_init(&mca_osc_pt2pt_component.p2p_c_buffers,
aligned_size,
OBJ_CLASS(ompi_osc_pt2pt_buffer_t),
1, -1, 1);
return OMPI_SUCCESS;
}
/*
* initialize the rb tree
*/
int mca_mpool_base_tree_init(void) {
int rc;
OBJ_CONSTRUCT(&mca_mpool_base_tree, opal_rb_tree_t);
OBJ_CONSTRUCT(&mca_mpool_base_tree_item_free_list, opal_free_list_t);
OBJ_CONSTRUCT(&tree_lock, opal_mutex_t);
rc = opal_free_list_init (&mca_mpool_base_tree_item_free_list,
sizeof(mca_mpool_base_tree_item_t),
opal_cache_line_size,
OBJ_CLASS(mca_mpool_base_tree_item_t),
0,opal_cache_line_size,
0, -1 , 4, NULL, 0, NULL, NULL, NULL);
if(OPAL_SUCCESS == rc) {
rc = opal_rb_tree_init(&mca_mpool_base_tree, mca_mpool_base_tree_node_compare);
}
return rc;
}
/*
* Initializes the rcache module.
*/
void mca_rcache_rgpusm_module_init(mca_rcache_rgpusm_module_t* rcache)
{
rcache->super.rcache_component = &mca_rcache_rgpusm_component.super;
rcache->super.rcache_register = mca_rcache_rgpusm_register;
rcache->super.rcache_find = mca_rcache_rgpusm_find;
rcache->super.rcache_deregister = mca_rcache_rgpusm_deregister;
rcache->super.rcache_finalize = mca_rcache_rgpusm_finalize;
rcache->vma_module = mca_rcache_base_vma_module_alloc ();
OBJ_CONSTRUCT(&rcache->reg_list, opal_free_list_t);
opal_free_list_init (&rcache->reg_list, sizeof(struct mca_rcache_common_cuda_reg_t),
opal_cache_line_size,
OBJ_CLASS(mca_rcache_base_registration_t),
0,opal_cache_line_size,
0, -1, 32, NULL, 0, NULL, NULL, NULL);
OBJ_CONSTRUCT(&rcache->lru_list, opal_list_t);
rcache->stat_cache_hit = rcache->stat_cache_miss = rcache->stat_evicted = 0;
rcache->stat_cache_found = rcache->stat_cache_notfound = 0;
rcache->stat_cache_valid = rcache->stat_cache_invalid = 0;
}
/*
* Initializes the rcache module.
*/
void mca_rcache_grdma_module_init(mca_rcache_grdma_module_t* rcache, mca_rcache_grdma_cache_t *cache)
{
OBJ_RETAIN(cache);
rcache->cache = cache;
rcache->super.rcache_component = &mca_rcache_grdma_component.super;
rcache->super.rcache_register = mca_rcache_grdma_register;
rcache->super.rcache_find = mca_rcache_grdma_find;
rcache->super.rcache_deregister = mca_rcache_grdma_deregister;
rcache->super.rcache_invalidate_range = mca_rcache_grdma_invalidate_range;
rcache->super.rcache_finalize = mca_rcache_grdma_finalize;
rcache->super.rcache_evict = mca_rcache_grdma_evict;
rcache->stat_cache_hit = rcache->stat_cache_miss = rcache->stat_evicted = 0;
rcache->stat_cache_found = rcache->stat_cache_notfound = 0;
OBJ_CONSTRUCT(&rcache->reg_list, opal_free_list_t);
opal_free_list_init (&rcache->reg_list, rcache->resources.sizeof_reg,
opal_cache_line_size,
OBJ_CLASS(mca_rcache_base_registration_t),
0, opal_cache_line_size, 0, -1, 32, NULL, 0,
NULL, NULL, NULL);
}
int ofi_comp_list_init(opal_free_list_t *comp_list)
{
int rc;
OBJ_CONSTRUCT(comp_list, opal_free_list_t);
rc = opal_free_list_init(comp_list,
sizeof(mca_btl_ofi_completion_t),
opal_cache_line_size,
OBJ_CLASS(mca_btl_ofi_completion_t),
0,
0,
128,
-1,
128,
NULL,
0,
NULL,
NULL,
NULL);
if (rc != OPAL_SUCCESS) {
BTL_VERBOSE(("cannot allocate completion freelist"));
}
return rc;
}
/** VPROTOCOL level functions (same as PML one)
*/
static mca_vprotocol_base_module_t *mca_vprotocol_pessimist_component_init( int* priority,
bool enable_progress_threads,
bool enable_mpi_threads)
{
V_OUTPUT_VERBOSE(500, "vprotocol_pessimist: component_init");
*priority = _priority;
/* sanity check */
if(enable_mpi_threads)
{
opal_output(0, "vprotocol_pessimist: component_init: threads are enabled, and not supported by vprotocol pessimist fault tolerant layer, will not load");
return NULL;
}
mca_vprotocol_pessimist.clock = 1;
mca_vprotocol_pessimist.replay = false;
OBJ_CONSTRUCT(&mca_vprotocol_pessimist.replay_events, opal_list_t);
OBJ_CONSTRUCT(&mca_vprotocol_pessimist.pending_events, opal_list_t);
OBJ_CONSTRUCT(&mca_vprotocol_pessimist.events_pool, opal_free_list_t);
opal_free_list_init (&mca_vprotocol_pessimist.events_pool,
sizeof(mca_vprotocol_pessimist_event_t),
opal_cache_line_size,
OBJ_CLASS(mca_vprotocol_pessimist_event_t),
0,opal_cache_line_size,
_free_list_num,
_free_list_max,
_free_list_inc,
NULL, 0, NULL, NULL, NULL);
mca_vprotocol_pessimist.event_buffer_max_length =
_event_buffer_size / sizeof(vprotocol_pessimist_mem_event_t);
mca_vprotocol_pessimist.event_buffer_length = 0;
mca_vprotocol_pessimist.event_buffer =
(vprotocol_pessimist_mem_event_t *) malloc(_event_buffer_size);
mca_vprotocol_pessimist.el_comm = MPI_COMM_NULL;
return &mca_vprotocol_pessimist.super;
}
void hcoll_rte_fns_setup(void)
{
init_module_fns();
OBJ_CONSTRUCT(&mca_coll_hcoll_component.requests, opal_free_list_t);
opal_free_list_init(
&(mca_coll_hcoll_component.requests),
sizeof(ompi_request_t),
/* no special alignment needed */
8,
OBJ_CLASS(ompi_request_t),
/* no payload data */
0, 0,
/* NOTE: hack - need to parametrize this */
10,
-1,
10,
/* No Mpool or init function */
NULL,
0,
NULL,
NULL,
NULL
);
}
static void initFreeList(void)
{
OBJ_CONSTRUCT(&ompi_java_buffers, opal_free_list_t);
int r = opal_free_list_init(&ompi_java_buffers,
sizeof(ompi_java_buffer_t),
opal_cache_line_size,
OBJ_CLASS(ompi_java_buffer_t),
0, /* payload size */
0, /* payload align */
2, /* initial elements to alloc */
-1, /* max elements */
2, /* num elements per alloc */
NULL, /* mpool */
0, /* mpool reg flags */
NULL, /* unused0 */
NULL, /* item_init */
NULL /* inem_init context */);
if(r != OPAL_SUCCESS)
{
fprintf(stderr, "Unable to initialize ompi_java_buffers.\n");
exit(1);
}
}
/*
* Invoked when there's a new communicator that has been created.
* Look at the communicator and decide which set of functions and
* priority we want to return.
*/
mca_coll_base_module_t *
mca_coll_hcoll_comm_query(struct ompi_communicator_t *comm, int *priority)
{
mca_coll_base_module_t *module;
mca_coll_hcoll_module_t *hcoll_module;
ompi_attribute_fn_ptr_union_t del_fn;
ompi_attribute_fn_ptr_union_t copy_fn;
mca_coll_hcoll_component_t *cm;
int err;
int rc;
cm = &mca_coll_hcoll_component;
*priority = 0;
module = NULL;
if (!cm->hcoll_enable) {
return NULL;
}
if (OMPI_COMM_IS_INTER(comm) || ompi_comm_size(comm) < cm->hcoll_np
|| ompi_comm_size(comm) < 2) {
return NULL;
}
if (!cm->libhcoll_initialized)
{
/* libhcoll should be initialized here since current implmentation of
mxm bcol in libhcoll needs world_group fully functional during init
world_group, i.e. ompi_comm_world, is not ready at hcoll component open
call */
opal_progress_register(mca_coll_hcoll_progress);
HCOL_VERBOSE(10,"Calling hcoll_init();");
#if HCOLL_API >= HCOLL_VERSION(3,2)
hcoll_read_init_opts(&cm->init_opts);
cm->init_opts->base_tag = MCA_COLL_BASE_TAG_HCOLL_BASE;
cm->init_opts->max_tag = mca_pml.pml_max_tag;
cm->init_opts->enable_thread_support = ompi_mpi_thread_multiple;
rc = hcoll_init_with_opts(&cm->init_opts);
#else
hcoll_set_runtime_tag_offset(MCA_COLL_BASE_TAG_HCOLL_BASE, mca_pml.pml_max_tag);
rc = hcoll_init();
#endif
if (HCOLL_SUCCESS != rc) {
cm->hcoll_enable = 0;
opal_progress_unregister(mca_coll_hcoll_progress);
HCOL_ERROR("Hcol library init failed");
return NULL;
}
#if HCOLL_API >= HCOLL_VERSION(3,2)
if (cm->using_mem_hooks && cm->init_opts->mem_hook_needed) {
#else
if (cm->using_mem_hooks && hcoll_check_mem_release_cb_needed()) {
#endif
opal_mem_hooks_register_release(mca_coll_hcoll_mem_release_cb, NULL);
} else {
cm->using_mem_hooks = 0;
}
copy_fn.attr_communicator_copy_fn = (MPI_Comm_internal_copy_attr_function*) MPI_COMM_NULL_COPY_FN;
del_fn.attr_communicator_delete_fn = hcoll_comm_attr_del_fn;
err = ompi_attr_create_keyval(COMM_ATTR, copy_fn, del_fn, &hcoll_comm_attr_keyval, NULL ,0, NULL);
if (OMPI_SUCCESS != err) {
cm->hcoll_enable = 0;
hcoll_finalize();
opal_progress_unregister(mca_coll_hcoll_progress);
HCOL_ERROR("Hcol comm keyval create failed");
return NULL;
}
if (mca_coll_hcoll_component.derived_types_support_enabled) {
copy_fn.attr_datatype_copy_fn = (MPI_Type_internal_copy_attr_function *) MPI_TYPE_NULL_COPY_FN;
del_fn.attr_datatype_delete_fn = hcoll_type_attr_del_fn;
err = ompi_attr_create_keyval(TYPE_ATTR, copy_fn, del_fn, &hcoll_type_attr_keyval, NULL ,0, NULL);
if (OMPI_SUCCESS != err) {
cm->hcoll_enable = 0;
hcoll_finalize();
opal_progress_unregister(mca_coll_hcoll_progress);
HCOL_ERROR("Hcol type keyval create failed");
return NULL;
}
}
OBJ_CONSTRUCT(&cm->dtypes, opal_free_list_t);
opal_free_list_init(&cm->dtypes, sizeof(mca_coll_hcoll_dtype_t),
8, OBJ_CLASS(mca_coll_hcoll_dtype_t), 0, 0,
32, -1, 32, NULL, 0, NULL, NULL, NULL);
}
hcoll_module = OBJ_NEW(mca_coll_hcoll_module_t);
if (!hcoll_module) {
if (!cm->libhcoll_initialized) {
cm->hcoll_enable = 0;
hcoll_finalize();
opal_progress_unregister(mca_coll_hcoll_progress);
}
return NULL;
}
hcoll_module->comm = comm;
HCOL_VERBOSE(10,"Creating hcoll_context for comm %p, comm_id %d, comm_size %d",
(void*)comm,comm->c_contextid,ompi_comm_size(comm));
hcoll_module->hcoll_context =
hcoll_create_context((rte_grp_handle_t)comm);
if (NULL == hcoll_module->hcoll_context) {
HCOL_VERBOSE(1,"hcoll_create_context returned NULL");
OBJ_RELEASE(hcoll_module);
if (!cm->libhcoll_initialized) {
cm->hcoll_enable = 0;
hcoll_finalize();
opal_progress_unregister(mca_coll_hcoll_progress);
}
return NULL;
}
hcoll_module->super.coll_module_enable = mca_coll_hcoll_module_enable;
hcoll_module->super.coll_barrier = hcoll_collectives.coll_barrier ? mca_coll_hcoll_barrier : NULL;
hcoll_module->super.coll_bcast = hcoll_collectives.coll_bcast ? mca_coll_hcoll_bcast : NULL;
hcoll_module->super.coll_allgather = hcoll_collectives.coll_allgather ? mca_coll_hcoll_allgather : NULL;
hcoll_module->super.coll_allgatherv = hcoll_collectives.coll_allgatherv ? mca_coll_hcoll_allgatherv : NULL;
hcoll_module->super.coll_allreduce = hcoll_collectives.coll_allreduce ? mca_coll_hcoll_allreduce : NULL;
hcoll_module->super.coll_alltoall = hcoll_collectives.coll_alltoall ? mca_coll_hcoll_alltoall : NULL;
hcoll_module->super.coll_alltoallv = hcoll_collectives.coll_alltoallv ? mca_coll_hcoll_alltoallv : NULL;
hcoll_module->super.coll_gatherv = hcoll_collectives.coll_gatherv ? mca_coll_hcoll_gatherv : NULL;
hcoll_module->super.coll_reduce = hcoll_collectives.coll_reduce ? mca_coll_hcoll_reduce : NULL;
hcoll_module->super.coll_ibarrier = hcoll_collectives.coll_ibarrier ? mca_coll_hcoll_ibarrier : NULL;
hcoll_module->super.coll_ibcast = hcoll_collectives.coll_ibcast ? mca_coll_hcoll_ibcast : NULL;
hcoll_module->super.coll_iallgather = hcoll_collectives.coll_iallgather ? mca_coll_hcoll_iallgather : NULL;
#if HCOLL_API >= HCOLL_VERSION(3,5)
hcoll_module->super.coll_iallgatherv = hcoll_collectives.coll_iallgatherv ? mca_coll_hcoll_iallgatherv : NULL;
#else
hcoll_module->super.coll_iallgatherv = NULL;
#endif
hcoll_module->super.coll_iallreduce = hcoll_collectives.coll_iallreduce ? mca_coll_hcoll_iallreduce : NULL;
#if HCOLL_API >= HCOLL_VERSION(3,5)
hcoll_module->super.coll_ireduce = hcoll_collectives.coll_ireduce ? mca_coll_hcoll_ireduce : NULL;
#else
hcoll_module->super.coll_ireduce = NULL;
#endif
hcoll_module->super.coll_gather = /*hcoll_collectives.coll_gather ? mca_coll_hcoll_gather :*/ NULL;
hcoll_module->super.coll_igatherv = hcoll_collectives.coll_igatherv ? mca_coll_hcoll_igatherv : NULL;
hcoll_module->super.coll_ialltoall = /*hcoll_collectives.coll_ialltoall ? mca_coll_hcoll_ialltoall : */ NULL;
#if HCOLL_API >= HCOLL_VERSION(3,7)
hcoll_module->super.coll_ialltoallv = hcoll_collectives.coll_ialltoallv ? mca_coll_hcoll_ialltoallv : NULL;
#else
hcoll_module->super.coll_ialltoallv = NULL;
#endif
*priority = cm->hcoll_priority;
module = &hcoll_module->super;
if (!cm->libhcoll_initialized) {
cm->libhcoll_initialized = true;
}
return module;
}
OBJ_CLASS_INSTANCE(mca_coll_hcoll_module_t,
mca_coll_base_module_t,
mca_coll_hcoll_module_construct,
mca_coll_hcoll_module_destruct);
static int
component_init(bool enable_progress_threads, bool enable_mpi_threads)
{
int ret;
ptl_ni_limits_t actual;
ret = PtlInit();
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_osc_base_framework.framework_output,
"%s:%d: PtlInit failed: %d\n",
__FILE__, __LINE__, ret);
return OMPI_ERROR;
}
ret = PtlNIInit(PTL_IFACE_DEFAULT,
PTL_NI_PHYSICAL | PTL_NI_MATCHING,
PTL_PID_ANY,
NULL,
&actual,
&mca_osc_portals4_component.matching_ni_h);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_osc_base_framework.framework_output,
"%s:%d: PtlNIInit failed: %d\n",
__FILE__, __LINE__, ret);
return ret;
}
/* BWB: FIX ME: Need to make sure our ID matches with the MTL... */
mca_osc_portals4_component.matching_atomic_max = actual.max_atomic_size;
mca_osc_portals4_component.matching_fetch_atomic_max = actual.max_fetch_atomic_size;
mca_osc_portals4_component.matching_atomic_ordered_size =
MAX(actual.max_waw_ordered_size, actual.max_war_ordered_size);
ret = PtlEQAlloc(mca_osc_portals4_component.matching_ni_h,
4096,
&mca_osc_portals4_component.matching_eq_h);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_osc_base_framework.framework_output,
"%s:%d: PtlEQAlloc failed: %d\n",
__FILE__, __LINE__, ret);
return ret;
}
ret = PtlPTAlloc(mca_osc_portals4_component.matching_ni_h,
0,
mca_osc_portals4_component.matching_eq_h,
4,
&mca_osc_portals4_component.matching_pt_idx);
if (PTL_OK != ret) {
opal_output_verbose(1, ompi_osc_base_framework.framework_output,
"%s:%d: PtlPTAlloc failed: %d\n",
__FILE__, __LINE__, ret);
return ret;
}
OBJ_CONSTRUCT(&mca_osc_portals4_component.requests, opal_free_list_t);
ret = opal_free_list_init (&mca_osc_portals4_component.requests,
sizeof(ompi_osc_portals4_request_t),
opal_cache_line_size,
OBJ_CLASS(ompi_osc_portals4_request_t),
0, 0, 8, 0, 8, NULL, 0, NULL, NULL, NULL);
if (OMPI_SUCCESS != ret) {
opal_output_verbose(1, ompi_osc_base_framework.framework_output,
"%s:%d: opal_free_list_init failed: %d\n",
__FILE__, __LINE__, ret);
return ret;
}
ret = opal_progress_register(progress_callback);
if (OMPI_SUCCESS != ret) {
opal_output_verbose(1, ompi_osc_base_framework.framework_output,
"%s:%d: opal_progress_register failed: %d\n",
__FILE__, __LINE__, ret);
return ret;
}
return OMPI_SUCCESS;
}
static int
sm_btl_first_time_init(mca_btl_sm_t *sm_btl,
int32_t my_smp_rank,
int n)
{
size_t length, length_payload;
sm_fifo_t *my_fifos;
int my_mem_node, num_mem_nodes, i, rc;
mca_mpool_base_resources_t *res = NULL;
mca_btl_sm_component_t* m = &mca_btl_sm_component;
/* Assume we don't have hwloc support and fill in dummy info */
mca_btl_sm_component.mem_node = my_mem_node = 0;
mca_btl_sm_component.num_mem_nodes = num_mem_nodes = 1;
#if OPAL_HAVE_HWLOC
/* If we have hwloc support, then get accurate information */
if (NULL != opal_hwloc_topology) {
i = opal_hwloc_base_get_nbobjs_by_type(opal_hwloc_topology,
HWLOC_OBJ_NODE, 0,
OPAL_HWLOC_AVAILABLE);
/* If we find >0 NUMA nodes, then investigate further */
if (i > 0) {
int numa=0, w;
unsigned n_bound=0;
hwloc_cpuset_t avail;
hwloc_obj_t obj;
/* JMS This tells me how many numa nodes are *available*,
but it's not how many are being used *by this job*.
Note that this is the value we've previously used (from
the previous carto-based implementation), but it really
should be improved to be how many NUMA nodes are being
used *in this job*. */
mca_btl_sm_component.num_mem_nodes = num_mem_nodes = i;
/* if we are not bound, then there is nothing further to do */
if (NULL != ompi_process_info.cpuset) {
/* count the number of NUMA nodes to which we are bound */
for (w=0; w < i; w++) {
if (NULL == (obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology,
HWLOC_OBJ_NODE, 0, w,
OPAL_HWLOC_AVAILABLE))) {
continue;
}
/* get that NUMA node's available cpus */
avail = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, obj);
/* see if we intersect */
if (hwloc_bitmap_intersects(avail, opal_hwloc_my_cpuset)) {
n_bound++;
numa = w;
}
}
/* if we are located on more than one NUMA, or we didn't find
* a NUMA we are on, then not much we can do
*/
if (1 == n_bound) {
mca_btl_sm_component.mem_node = my_mem_node = numa;
} else {
mca_btl_sm_component.mem_node = my_mem_node = -1;
}
}
}
}
#endif
if (NULL == (res = calloc(1, sizeof(*res)))) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* lookup shared memory pool */
mca_btl_sm_component.sm_mpools =
(mca_mpool_base_module_t **)calloc(num_mem_nodes,
sizeof(mca_mpool_base_module_t *));
/* Disable memory binding, because each MPI process will claim pages in the
* mpool for their local NUMA node */
res->mem_node = -1;
if (OMPI_SUCCESS != (rc = setup_mpool_base_resources(m, res))) {
free(res);
return rc;
}
/* now that res is fully populated, create the thing */
mca_btl_sm_component.sm_mpools[0] =
mca_mpool_base_module_create(mca_btl_sm_component.sm_mpool_name,
sm_btl, res);
/* Sanity check to ensure that we found it */
if (NULL == mca_btl_sm_component.sm_mpools[0]) {
free(res);
return OMPI_ERR_OUT_OF_RESOURCE;
}
mca_btl_sm_component.sm_mpool = mca_btl_sm_component.sm_mpools[0];
mca_btl_sm_component.sm_mpool_base =
mca_btl_sm_component.sm_mpools[0]->mpool_base(mca_btl_sm_component.sm_mpools[0]);
/* create a list of peers */
mca_btl_sm_component.sm_peers = (struct mca_btl_base_endpoint_t**)
calloc(n, sizeof(struct mca_btl_base_endpoint_t*));
if (NULL == mca_btl_sm_component.sm_peers) {
free(res);
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* remember that node rank zero is already attached */
if (0 != my_smp_rank) {
if (OMPI_SUCCESS != (rc = sm_segment_attach(m))) {
free(res);
return rc;
}
}
/* it is now safe to free the mpool resources */
free(res);
/* check to make sure number of local procs is within the
* specified limits */
if(mca_btl_sm_component.sm_max_procs > 0 &&
mca_btl_sm_component.num_smp_procs + n >
mca_btl_sm_component.sm_max_procs) {
return OMPI_ERROR;
}
mca_btl_sm_component.shm_fifo = (volatile sm_fifo_t **)mca_btl_sm_component.sm_seg->module_data_addr;
mca_btl_sm_component.shm_bases = (char**)(mca_btl_sm_component.shm_fifo + n);
mca_btl_sm_component.shm_mem_nodes = (uint16_t*)(mca_btl_sm_component.shm_bases + n);
/* set the base of the shared memory segment */
mca_btl_sm_component.shm_bases[mca_btl_sm_component.my_smp_rank] =
(char*)mca_btl_sm_component.sm_mpool_base;
mca_btl_sm_component.shm_mem_nodes[mca_btl_sm_component.my_smp_rank] =
(uint16_t)my_mem_node;
/* initialize the array of fifo's "owned" by this process */
if(NULL == (my_fifos = (sm_fifo_t*)mpool_calloc(FIFO_MAP_NUM(n), sizeof(sm_fifo_t))))
return OMPI_ERR_OUT_OF_RESOURCE;
mca_btl_sm_component.shm_fifo[mca_btl_sm_component.my_smp_rank] = my_fifos;
/* cache the pointer to the 2d fifo array. These addresses
* are valid in the current process space */
mca_btl_sm_component.fifo = (sm_fifo_t**)malloc(sizeof(sm_fifo_t*) * n);
if(NULL == mca_btl_sm_component.fifo)
return OMPI_ERR_OUT_OF_RESOURCE;
mca_btl_sm_component.fifo[mca_btl_sm_component.my_smp_rank] = my_fifos;
mca_btl_sm_component.mem_nodes = (uint16_t *) malloc(sizeof(uint16_t) * n);
if(NULL == mca_btl_sm_component.mem_nodes)
return OMPI_ERR_OUT_OF_RESOURCE;
/* initialize fragment descriptor free lists */
/* allocation will be for the fragment descriptor and payload buffer */
length = sizeof(mca_btl_sm_frag1_t);
length_payload =
sizeof(mca_btl_sm_hdr_t) + mca_btl_sm_component.eager_limit;
i = ompi_free_list_init_new(&mca_btl_sm_component.sm_frags_eager, length,
opal_cache_line_size, OBJ_CLASS(mca_btl_sm_frag1_t),
length_payload, opal_cache_line_size,
mca_btl_sm_component.sm_free_list_num,
mca_btl_sm_component.sm_free_list_max,
mca_btl_sm_component.sm_free_list_inc,
mca_btl_sm_component.sm_mpool);
if ( OMPI_SUCCESS != i )
return i;
length = sizeof(mca_btl_sm_frag2_t);
length_payload =
sizeof(mca_btl_sm_hdr_t) + mca_btl_sm_component.max_frag_size;
i = ompi_free_list_init_new(&mca_btl_sm_component.sm_frags_max, length,
opal_cache_line_size, OBJ_CLASS(mca_btl_sm_frag2_t),
length_payload, opal_cache_line_size,
mca_btl_sm_component.sm_free_list_num,
mca_btl_sm_component.sm_free_list_max,
mca_btl_sm_component.sm_free_list_inc,
mca_btl_sm_component.sm_mpool);
if ( OMPI_SUCCESS != i )
return i;
i = ompi_free_list_init_new(&mca_btl_sm_component.sm_frags_user,
sizeof(mca_btl_sm_user_t),
opal_cache_line_size, OBJ_CLASS(mca_btl_sm_user_t),
sizeof(mca_btl_sm_hdr_t), opal_cache_line_size,
mca_btl_sm_component.sm_free_list_num,
mca_btl_sm_component.sm_free_list_max,
mca_btl_sm_component.sm_free_list_inc,
mca_btl_sm_component.sm_mpool);
if ( OMPI_SUCCESS != i )
return i;
mca_btl_sm_component.num_outstanding_frags = 0;
mca_btl_sm_component.num_pending_sends = 0;
i = opal_free_list_init(&mca_btl_sm_component.pending_send_fl,
sizeof(btl_sm_pending_send_item_t),
OBJ_CLASS(opal_free_list_item_t),
16, -1, 32);
if ( OMPI_SUCCESS != i )
return i;
/* set flag indicating btl has been inited */
sm_btl->btl_inited = true;
return OMPI_SUCCESS;
}
static int vader_btl_first_time_init(mca_btl_vader_t *vader_btl, int n)
{
mca_btl_vader_component_t *component = &mca_btl_vader_component;
int rc;
/* generate the endpoints */
component->endpoints = (struct mca_btl_base_endpoint_t *) calloc (n + 1, sizeof (struct mca_btl_base_endpoint_t));
if (NULL == component->endpoints) {
return OPAL_ERR_OUT_OF_RESOURCE;
}
component->endpoints[n].peer_smp_rank = -1;
component->fbox_in_endpoints = calloc (n + 1, sizeof (void *));
if (NULL == component->fbox_in_endpoints) {
free(component->endpoints);
return OPAL_ERR_OUT_OF_RESOURCE;
}
component->segment_offset = MCA_BTL_VADER_FIFO_SIZE;
/* initialize fragment descriptor free lists */
/* initialize free list for small send and inline fragments */
rc = opal_free_list_init (&component->vader_frags_user,
sizeof(mca_btl_vader_frag_t),
opal_cache_line_size, OBJ_CLASS(mca_btl_vader_frag_t),
0, opal_cache_line_size,
component->vader_free_list_num,
component->vader_free_list_max,
component->vader_free_list_inc,
NULL, 0, NULL, mca_btl_vader_frag_init,
(void *)(intptr_t) mca_btl_vader_component.max_inline_send);
if (OPAL_SUCCESS != rc) {
return rc;
}
/* initialize free list for buffered send fragments */
rc = opal_free_list_init (&component->vader_frags_eager,
sizeof (mca_btl_vader_frag_t),
opal_cache_line_size, OBJ_CLASS(mca_btl_vader_frag_t),
0, opal_cache_line_size,
component->vader_free_list_num,
component->vader_free_list_max,
component->vader_free_list_inc,
NULL, 0, NULL, mca_btl_vader_frag_init,
(void *)(intptr_t) mca_btl_vader.super.btl_eager_limit);
if (OPAL_SUCCESS != rc) {
return rc;
}
if (MCA_BTL_VADER_XPMEM != mca_btl_vader_component.single_copy_mechanism) {
/* initialize free list for buffered send fragments */
rc = opal_free_list_init (&component->vader_frags_max_send,
sizeof (mca_btl_vader_frag_t),
opal_cache_line_size, OBJ_CLASS(mca_btl_vader_frag_t),
0, opal_cache_line_size,
component->vader_free_list_num,
component->vader_free_list_max,
component->vader_free_list_inc,
NULL, 0, NULL, mca_btl_vader_frag_init,
(void *)(intptr_t) mca_btl_vader.super.btl_max_send_size);
if (OPAL_SUCCESS != rc) {
return rc;
}
}
/* set flag indicating btl has been inited */
vader_btl->btl_inited = true;
return OPAL_SUCCESS;
}
static int component_init(bool enable_progress_threads, bool enable_mpi_threads) {
ucp_config_t *config = NULL;
ucp_params_t context_params;
bool progress_registered = false, requests_created = false;
int ret = OMPI_SUCCESS;
ucs_status_t status;
mca_osc_ucx_component.ucp_context = NULL;
mca_osc_ucx_component.ucp_worker = NULL;
mca_osc_ucx_component.enable_mpi_threads = enable_mpi_threads;
status = ucp_config_read("MPI", NULL, &config);
if (UCS_OK != status) {
opal_output_verbose(1, ompi_osc_base_framework.framework_output,
"%s:%d: ucp_config_read failed: %d\n",
__FILE__, __LINE__, status);
return OMPI_ERROR;
}
OBJ_CONSTRUCT(&mca_osc_ucx_component.requests, opal_free_list_t);
requests_created = true;
ret = opal_free_list_init (&mca_osc_ucx_component.requests,
sizeof(ompi_osc_ucx_request_t),
opal_cache_line_size,
OBJ_CLASS(ompi_osc_ucx_request_t),
0, 0, 8, 0, 8, NULL, 0, NULL, NULL, NULL);
if (OMPI_SUCCESS != ret) {
opal_output_verbose(1, ompi_osc_base_framework.framework_output,
"%s:%d: opal_free_list_init failed: %d\n",
__FILE__, __LINE__, ret);
goto error;
}
mca_osc_ucx_component.num_incomplete_req_ops = 0;
ret = opal_progress_register(progress_callback);
progress_registered = true;
if (OMPI_SUCCESS != ret) {
opal_output_verbose(1, ompi_osc_base_framework.framework_output,
"%s:%d: opal_progress_register failed: %d\n",
__FILE__, __LINE__, ret);
goto error;
}
/* initialize UCP context */
memset(&context_params, 0, sizeof(ucp_context_h));
context_params.field_mask = UCP_PARAM_FIELD_FEATURES |
UCP_PARAM_FIELD_MT_WORKERS_SHARED |
UCP_PARAM_FIELD_ESTIMATED_NUM_EPS |
UCP_PARAM_FIELD_REQUEST_INIT |
UCP_PARAM_FIELD_REQUEST_SIZE;
context_params.features = UCP_FEATURE_RMA | UCP_FEATURE_AMO32 | UCP_FEATURE_AMO64;
context_params.mt_workers_shared = 0;
context_params.estimated_num_eps = ompi_proc_world_size();
context_params.request_init = internal_req_init;
context_params.request_size = sizeof(ompi_osc_ucx_internal_request_t);
status = ucp_init(&context_params, config, &mca_osc_ucx_component.ucp_context);
ucp_config_release(config);
if (UCS_OK != status) {
opal_output_verbose(1, ompi_osc_base_framework.framework_output,
"%s:%d: ucp_init failed: %d\n",
__FILE__, __LINE__, status);
ret = OMPI_ERROR;
goto error;
}
return ret;
error:
if (progress_registered) opal_progress_unregister(progress_callback);
if (requests_created) OBJ_DESTRUCT(&mca_osc_ucx_component.requests);
if (mca_osc_ucx_component.ucp_context) ucp_cleanup(mca_osc_ucx_component.ucp_context);
return ret;
}
int mca_pml_ob1_enable(bool enable)
{
if( false == enable ) {
return OMPI_SUCCESS;
}
OBJ_CONSTRUCT(&mca_pml_ob1.lock, opal_mutex_t);
/* fragments */
OBJ_CONSTRUCT(&mca_pml_ob1.rdma_frags, opal_free_list_t);
opal_free_list_init ( &mca_pml_ob1.rdma_frags,
sizeof(mca_pml_ob1_rdma_frag_t),
opal_cache_line_size,
OBJ_CLASS(mca_pml_ob1_rdma_frag_t),
0,opal_cache_line_size,
mca_pml_ob1.free_list_num,
mca_pml_ob1.free_list_max,
mca_pml_ob1.free_list_inc,
NULL, 0, NULL, NULL, NULL);
OBJ_CONSTRUCT(&mca_pml_ob1.recv_frags, opal_free_list_t);
opal_free_list_init ( &mca_pml_ob1.recv_frags,
sizeof(mca_pml_ob1_recv_frag_t) + mca_pml_ob1.unexpected_limit,
opal_cache_line_size,
OBJ_CLASS(mca_pml_ob1_recv_frag_t),
0,opal_cache_line_size,
mca_pml_ob1.free_list_num,
mca_pml_ob1.free_list_max,
mca_pml_ob1.free_list_inc,
NULL, 0, NULL, NULL, NULL);
OBJ_CONSTRUCT(&mca_pml_ob1.pending_pckts, opal_free_list_t);
opal_free_list_init ( &mca_pml_ob1.pending_pckts,
sizeof(mca_pml_ob1_pckt_pending_t),
opal_cache_line_size,
OBJ_CLASS(mca_pml_ob1_pckt_pending_t),
0,opal_cache_line_size,
mca_pml_ob1.free_list_num,
mca_pml_ob1.free_list_max,
mca_pml_ob1.free_list_inc,
NULL, 0, NULL, NULL, NULL);
OBJ_CONSTRUCT(&mca_pml_ob1.buffers, opal_free_list_t);
OBJ_CONSTRUCT(&mca_pml_ob1.send_ranges, opal_free_list_t);
opal_free_list_init ( &mca_pml_ob1.send_ranges,
sizeof(mca_pml_ob1_send_range_t) +
(mca_pml_ob1.max_send_per_range - 1) * sizeof(mca_pml_ob1_com_btl_t),
opal_cache_line_size,
OBJ_CLASS(mca_pml_ob1_send_range_t),
0,opal_cache_line_size,
mca_pml_ob1.free_list_num,
mca_pml_ob1.free_list_max,
mca_pml_ob1.free_list_inc,
NULL, 0, NULL, NULL, NULL);
/* pending operations */
OBJ_CONSTRUCT(&mca_pml_ob1.send_pending, opal_list_t);
OBJ_CONSTRUCT(&mca_pml_ob1.recv_pending, opal_list_t);
OBJ_CONSTRUCT(&mca_pml_ob1.pckt_pending, opal_list_t);
OBJ_CONSTRUCT(&mca_pml_ob1.rdma_pending, opal_list_t);
/* missing communicator pending list */
OBJ_CONSTRUCT(&mca_pml_ob1.non_existing_communicator_pending, opal_list_t);
/**
* If we get here this is the PML who get selected for the run. We
* should get ownership for the send and receive requests list, and
* initialize them with the size of our own requests.
*/
opal_free_list_init ( &mca_pml_base_send_requests,
sizeof(mca_pml_ob1_send_request_t) +
(mca_pml_ob1.max_rdma_per_request - 1) *
sizeof(mca_pml_ob1_com_btl_t),
opal_cache_line_size,
OBJ_CLASS(mca_pml_ob1_send_request_t),
0,opal_cache_line_size,
mca_pml_ob1.free_list_num,
mca_pml_ob1.free_list_max,
mca_pml_ob1.free_list_inc,
NULL, 0, NULL, NULL, NULL);
opal_free_list_init ( &mca_pml_base_recv_requests,
sizeof(mca_pml_ob1_recv_request_t) +
(mca_pml_ob1.max_rdma_per_request - 1) *
sizeof(mca_pml_ob1_com_btl_t),
opal_cache_line_size,
OBJ_CLASS(mca_pml_ob1_recv_request_t),
0,opal_cache_line_size,
mca_pml_ob1.free_list_num,
mca_pml_ob1.free_list_max,
mca_pml_ob1.free_list_inc,
NULL, 0, NULL, NULL, NULL);
mca_pml_ob1.enabled = true;
return OMPI_SUCCESS;
}
int init_context_freelists(mca_btl_ofi_context_t *context)
{
int rc;
OBJ_CONSTRUCT(&context->rdma_comp_list, opal_free_list_t);
rc = opal_free_list_init(&context->rdma_comp_list,
sizeof(mca_btl_ofi_rdma_completion_t),
opal_cache_line_size,
OBJ_CLASS(mca_btl_ofi_rdma_completion_t),
0,
0,
512,
-1,
512,
NULL,
0,
NULL,
NULL,
NULL);
if (rc != OPAL_SUCCESS) {
BTL_VERBOSE(("cannot allocate completion freelist"));
return rc;
}
if (TWO_SIDED_ENABLED) {
OBJ_CONSTRUCT(&context->frag_comp_list, opal_free_list_t);
rc = opal_free_list_init(&context->frag_comp_list,
sizeof(mca_btl_ofi_frag_completion_t),
opal_cache_line_size,
OBJ_CLASS(mca_btl_ofi_frag_completion_t),
0,
0,
512,
-1,
512,
NULL,
0,
NULL,
NULL,
NULL);
if (rc != OPAL_SUCCESS) {
BTL_VERBOSE(("cannot allocate completion freelist"));
return rc;
}
/* Initialize frag pool */
OBJ_CONSTRUCT(&context->frag_list, opal_free_list_t);
rc = opal_free_list_init(&context->frag_list,
sizeof(mca_btl_ofi_base_frag_t) +
MCA_BTL_OFI_FRAG_SIZE,
opal_cache_line_size,
OBJ_CLASS(mca_btl_ofi_base_frag_t),
0,
0,
1024,
-1,
1024,
NULL,
0,
NULL,
NULL,
NULL);
if (OPAL_SUCCESS != rc) {
BTL_VERBOSE(("failed to init frag pool (free_list)"));
}
}
return rc;
}
int
ompi_mtl_portals4_flowctl_init(void)
{
ptl_me_t me;
int ret;
ompi_mtl_portals4.flowctl.flowctl_active = false;
OBJ_CONSTRUCT(&ompi_mtl_portals4.flowctl.pending_sends, opal_list_t);
OBJ_CONSTRUCT(&ompi_mtl_portals4.flowctl.pending_fl, opal_free_list_t);
opal_free_list_init(&ompi_mtl_portals4.flowctl.pending_fl,
sizeof(ompi_mtl_portals4_pending_request_t),
opal_cache_line_size,
OBJ_CLASS(ompi_mtl_portals4_pending_request_t),
0, 0, 1, -1, 1, NULL, 0, NULL, NULL, NULL);
ompi_mtl_portals4.flowctl.max_send_slots = (ompi_mtl_portals4.send_queue_size - 3) / 3;
ompi_mtl_portals4.flowctl.send_slots = ompi_mtl_portals4.flowctl.max_send_slots;
ompi_mtl_portals4.flowctl.alert_req.type = portals4_req_flowctl;
ompi_mtl_portals4.flowctl.alert_req.event_callback = flowctl_alert_callback;
ompi_mtl_portals4.flowctl.fanout_req.type = portals4_req_flowctl;
ompi_mtl_portals4.flowctl.fanout_req.event_callback = flowctl_fanout_callback;
ompi_mtl_portals4.flowctl.epoch_counter = -1;
ret = PtlPTAlloc(ompi_mtl_portals4.ni_h,
PTL_PT_ONLY_TRUNCATE,
ompi_mtl_portals4.send_eq_h,
REQ_FLOWCTL_TABLE_ID,
&ompi_mtl_portals4.flowctl_idx);
if (OPAL_UNLIKELY(PTL_OK != ret)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: PtlPTAlloc failed: %d\n",
__FILE__, __LINE__, ret);
goto error;
}
ret = PtlCTAlloc(ompi_mtl_portals4.ni_h,
&ompi_mtl_portals4.flowctl.trigger_ct_h);
if (OPAL_UNLIKELY(PTL_OK != ret)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: PtlCTAlloc failed: %d\n",
__FILE__, __LINE__, ret);
goto error;
}
/* everyone creates the trigger ME, even if the root may be the
only to use it */
me.start = NULL;
me.length = 0;
me.min_free = 0;
me.uid = ompi_mtl_portals4.uid;
if (ompi_mtl_portals4.use_logical) {
me.match_id.rank = PTL_RANK_ANY;
} else {
me.match_id.phys.nid = PTL_NID_ANY;
me.match_id.phys.pid = PTL_PID_ANY;
}
me.ignore_bits = 0;
me.options = PTL_ME_OP_PUT |
PTL_ME_ACK_DISABLE |
PTL_ME_EVENT_LINK_DISABLE |
PTL_ME_EVENT_UNLINK_DISABLE |
PTL_ME_EVENT_COMM_DISABLE |
PTL_ME_EVENT_CT_COMM;
me.ct_handle = ompi_mtl_portals4.flowctl.trigger_ct_h;
me.match_bits = MTL_PORTALS4_FLOWCTL_TRIGGER;
ret = PtlMEAppend(ompi_mtl_portals4.ni_h,
ompi_mtl_portals4.flowctl_idx,
&me,
PTL_PRIORITY_LIST,
NULL,
&ompi_mtl_portals4.flowctl.trigger_me_h);
if (OPAL_UNLIKELY(PTL_OK != ret)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: PtlMEAppend failed: %d\n",
__FILE__, __LINE__, ret);
goto error;
}
/* Alert CT/ME for broadcasting out alert when root receives a
trigger */
ret = PtlCTAlloc(ompi_mtl_portals4.ni_h,
&ompi_mtl_portals4.flowctl.alert_ct_h);
if (OPAL_UNLIKELY(PTL_OK != ret)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: PtlCTAlloc failed: %d\n",
__FILE__, __LINE__, ret);
goto error;
}
me.options = PTL_ME_OP_PUT |
PTL_ME_ACK_DISABLE |
PTL_ME_EVENT_LINK_DISABLE |
PTL_ME_EVENT_UNLINK_DISABLE |
PTL_ME_EVENT_CT_COMM;
me.ct_handle = ompi_mtl_portals4.flowctl.alert_ct_h;
me.match_bits = MTL_PORTALS4_FLOWCTL_ALERT;
ret = PtlMEAppend(ompi_mtl_portals4.ni_h,
ompi_mtl_portals4.flowctl_idx,
&me,
PTL_PRIORITY_LIST,
&ompi_mtl_portals4.flowctl.alert_req,
&ompi_mtl_portals4.flowctl.alert_me_h);
if (OPAL_UNLIKELY(PTL_OK != ret)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: PtlMEAppend failed: %d\n",
__FILE__, __LINE__, ret);
goto error;
}
/* Fanin CT/ME for receiving fan-in for restart */
ret = PtlCTAlloc(ompi_mtl_portals4.ni_h,
&ompi_mtl_portals4.flowctl.fanin_ct_h);
if (OPAL_UNLIKELY(PTL_OK != ret)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: PtlCTAlloc failed: %d\n",
__FILE__, __LINE__, ret);
goto error;
}
me.options = PTL_ME_OP_PUT |
PTL_ME_ACK_DISABLE |
PTL_ME_EVENT_COMM_DISABLE |
PTL_ME_EVENT_LINK_DISABLE |
PTL_ME_EVENT_UNLINK_DISABLE |
PTL_ME_EVENT_CT_COMM;
me.ct_handle = ompi_mtl_portals4.flowctl.fanin_ct_h;
me.match_bits = MTL_PORTALS4_FLOWCTL_FANIN;
ret = PtlMEAppend(ompi_mtl_portals4.ni_h,
ompi_mtl_portals4.flowctl_idx,
&me,
PTL_PRIORITY_LIST,
NULL,
&ompi_mtl_portals4.flowctl.fanin_me_h);
if (OPAL_UNLIKELY(PTL_OK != ret)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: PtlMEAppend failed: %d\n",
__FILE__, __LINE__, ret);
goto error;
}
/* Fan-out CT/ME for sending restart messages after fan-in */
ret = PtlCTAlloc(ompi_mtl_portals4.ni_h,
&ompi_mtl_portals4.flowctl.fanout_ct_h);
if (OPAL_UNLIKELY(PTL_OK != ret)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: PtlCTAlloc failed: %d\n",
__FILE__, __LINE__, ret);
goto error;
}
me.options = PTL_ME_OP_PUT |
PTL_ME_ACK_DISABLE |
PTL_ME_EVENT_LINK_DISABLE |
PTL_ME_EVENT_UNLINK_DISABLE |
PTL_ME_EVENT_CT_COMM;
me.ct_handle = ompi_mtl_portals4.flowctl.fanout_ct_h;
me.match_bits = MTL_PORTALS4_FLOWCTL_FANOUT;
ret = PtlMEAppend(ompi_mtl_portals4.ni_h,
ompi_mtl_portals4.flowctl_idx,
&me,
PTL_PRIORITY_LIST,
&ompi_mtl_portals4.flowctl.fanout_req,
&ompi_mtl_portals4.flowctl.fanout_me_h);
if (OPAL_UNLIKELY(PTL_OK != ret)) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: PtlMEAppend failed: %d\n",
__FILE__, __LINE__, ret);
goto error;
}
ompi_mtl_portals4.flowctl.num_children = 0;
gettimeofday(&ompi_mtl_portals4.flowctl.tv, NULL);
ompi_mtl_portals4.flowctl.backoff_count = 0;
ret = OMPI_SUCCESS;
error:
return ret;
}
