static int
ompi_coll_tuned_forced_getvalues( enum COLLTYPE type,
coll_tuned_force_algorithm_params_t *forced_values )
{
coll_tuned_force_algorithm_mca_param_indices_t* mca_params;
const int *tmp;
mca_params = &(ompi_coll_tuned_forced_params[type]);
/**
* Set the selected algorithm to 0 by default. Later on we can check this against 0
* to see if it was setted explicitly (if we suppose that setting it to 0 enable the
* default behavior) or not.
*/
mca_base_var_get_value(mca_params->algorithm_param_index, &tmp, NULL, NULL);
forced_values->algorithm = tmp ? tmp[0] : 0;
if( BARRIER != type ) {
mca_base_var_get_value(mca_params->segsize_param_index, &tmp, NULL, NULL);
if (tmp) forced_values->segsize = tmp[0];
mca_base_var_get_value(mca_params->tree_fanout_param_index, &tmp, NULL, NULL);
if (tmp) forced_values->tree_fanout = tmp[0];
mca_base_var_get_value(mca_params->chain_fanout_param_index, &tmp, NULL, NULL);
if (tmp) forced_values->chain_fanout = tmp[0];
mca_base_var_get_value(mca_params->max_requests_param_index, &tmp, NULL, NULL);
if (tmp) forced_values->max_requests = tmp[0];
}
return (MPI_SUCCESS);
}
int MPI_T_cvar_read (MPI_T_cvar_handle handle, void *buf)
{
const mca_base_var_storage_t *value = NULL;
int rc = MPI_SUCCESS;
if (!mpit_is_initialized ()) {
return MPI_T_ERR_NOT_INITIALIZED;
}
if (MPI_PARAM_CHECK && NULL == buf) {
return MPI_ERR_ARG;
}
ompi_mpit_lock ();
do {
rc = mca_base_var_get_value(handle->var->mbv_index, &value, NULL, NULL);
if (OPAL_SUCCESS != rc || NULL == value) {
/* shouldn't happen */
rc = MPI_ERR_OTHER;
break;
}
switch (handle->var->mbv_type) {
case MCA_BASE_VAR_TYPE_INT:
case MCA_BASE_VAR_TYPE_UNSIGNED_INT:
((int *) buf)[0] = value->intval;
break;
case MCA_BASE_VAR_TYPE_UNSIGNED_LONG:
((unsigned long *) buf)[0] = value->ulval;
break;
case MCA_BASE_VAR_TYPE_UNSIGNED_LONG_LONG:
((unsigned long long *) buf)[0] = value->ullval;
break;
case MCA_BASE_VAR_TYPE_SIZE_T:
((size_t *) buf)[0] = value->sizetval;
break;
case MCA_BASE_VAR_TYPE_BOOL:
((int *) buf)[0] = value->boolval;
break;
case MCA_BASE_VAR_TYPE_DOUBLE:
((double *) buf)[0] = value->lfval;
break;
case MCA_BASE_VAR_TYPE_STRING:
if (NULL == value->stringval) {
((char *)buf)[0] = '\0';
} else {
strcpy ((char *) buf, value->stringval);
}
break;
default:
rc = MPI_ERR_OTHER;
}
} while (0);
ompi_mpit_unlock ();
return rc;
}
/**
* Function for finding and opening either all MCA components, or the one
* that was specifically requested via a MCA parameter.
*/
static int mca_spml_base_open(mca_base_open_flag_t flags)
{
/**
* Construct the send and receive request queues. There are 2 reasons to do it
* here. First, as they are globals it's better to construct them in one common
* place. Second, in order to be able to allow the external debuggers to show
* their content, they should get constructed as soon as possible once the MPI
* process is started.
*/
OBJ_CONSTRUCT(&mca_spml_base_put_requests, ompi_free_list_t);
OBJ_CONSTRUCT(&mca_spml_base_get_requests, ompi_free_list_t);
OBJ_CONSTRUCT(&mca_spml_base_spml, opal_pointer_array_t);
/* Open up all available components */
if (OPAL_SUCCESS !=
mca_base_framework_components_open(&oshmem_spml_base_framework, flags)) {
return OSHMEM_ERROR;
}
/* Set a sentinel in case we don't select any components (e.g.,
ompi_info) */
mca_spml_base_selected_component.spmlm_finalize = NULL;
/**
* Right now our selection of BTLs is completely broken. If we have
* multiple SPMLs that use BTLs than we will open all BTLs several times, leading to
* undefined behaviors. The simplest solution, at least until we
* figure out the correct way to do it, is to force a default SPML that
* uses BTLs and any other SPMLs that do not in the mca_spml_base_spml array.
*/
#if MCA_ompi_pml_DIRECT_CALL
opal_pointer_array_add(&mca_spml_base_spml,
strdup(stringify(MCA_oshmem_spml_DIRECT_CALL_COMPONENT)));
#else
{
const char **default_spml = NULL;
int var_id;
var_id = mca_base_var_find("oshmem", "spml", NULL, NULL);
mca_base_var_get_value(var_id, &default_spml, NULL, NULL);
if( (NULL == default_spml || NULL == default_spml[0] ||
0 == strlen(default_spml[0])) || (default_spml[0][0] == '^') ) {
#ifdef OSHMEM_HAS_IKRIT
opal_pointer_array_add(&mca_spml_base_spml, strdup("ikrit"));
#endif
opal_pointer_array_add(&mca_spml_base_spml, strdup("yoda"));
} else {
opal_pointer_array_add(&mca_spml_base_spml, strdup(default_spml[0]));
}
}
#endif
return OSHMEM_SUCCESS;
}
int orte_snapc_base_select(bool seed, bool app)
{
int exit_status = OPAL_SUCCESS;
orte_snapc_base_component_t *best_component = NULL;
orte_snapc_base_module_t *best_module = NULL;
const char **include_list = NULL;
int var_id;
/*
* Register the framework MCA param and look up include list
*/
/* XXX -- TODO -- framework_subsytem -- this shouldn't be necessary once the framework system is in place */
var_id = mca_base_var_find(NULL, "snapc", NULL, NULL);
mca_base_var_get_value(var_id, &include_list, NULL, NULL);
if(NULL != include_list && NULL != include_list[0] &&
0 == strncmp(include_list[0], "none", strlen("none")) ){
opal_output_verbose(10, orte_snapc_base_framework.framework_output,
"snapc:select: Using %s component",
include_list[0]);
best_module = &none_module;
/* Close all components since none will be used */
mca_base_components_close(0, /* Pass 0 to keep this from closing the output handle */
&orte_snapc_base_framework.framework_components,
NULL);
/* JJH: Todo: Check if none is in the list */
goto skip_select;
}
/*
* Select the best component
*/
if( OPAL_SUCCESS != mca_base_select("snapc", orte_snapc_base_framework.framework_output,
&orte_snapc_base_framework.framework_components,
(mca_base_module_t **) &best_module,
(mca_base_component_t **) &best_component) ) {
/* This will only happen if no component was selected */
exit_status = ORTE_ERROR;
goto cleanup;
}
skip_select:
/* Save the winner */
orte_snapc = *best_module;
/* Initialize the winner */
if (NULL != best_module) {
if (OPAL_SUCCESS != orte_snapc.snapc_init(seed, app)) {
exit_status = OPAL_ERROR;
goto cleanup;
}
}
cleanup:
return exit_status;
}
int
ompi_init_preconnect_mpi(void)
{
int comm_size = ompi_comm_size(MPI_COMM_WORLD);
int comm_rank = ompi_comm_rank(MPI_COMM_WORLD);
int param, next, prev, i, ret = OMPI_SUCCESS;
struct ompi_request_t * requests[2];
char inbuf[1], outbuf[1];
const bool *value;
param = mca_base_var_find("ompi", "mpi", NULL, "preconnect_mpi");
if (0 > param) return OMPI_SUCCESS;
ret = mca_base_var_get_value(param, &value, NULL, NULL);
if (OMPI_SUCCESS != ret || 0 == value[0]) {
return OMPI_SUCCESS;
}
inbuf[0] = outbuf[0] = '\0';
/* Each iteration, every process sends to its neighbor i hops to
the right and receives from its neighbor i hops to the left.
Because send_complete is used, there will only ever be one
outstanding send and one outstanding receive in the network at
a time for any given process. This limits any "flooding"
effect that can occur with other connection algorithms. While
the flooding algorithms may be a more efficient use of
resources, they can overwhelm the out-of-band connection system
used to wire up some networks, leading to poor performance and
hangs. */
for (i = 1 ; i <= comm_size / 2 ; ++i) {
next = (comm_rank + i) % comm_size;
prev = (comm_rank - i + comm_size) % comm_size;
ret = MCA_PML_CALL(isend(outbuf, 1, MPI_CHAR,
next, 1,
MCA_PML_BASE_SEND_COMPLETE,
MPI_COMM_WORLD,
&requests[1]));
if (OMPI_SUCCESS != ret) return ret;
ret = MCA_PML_CALL(irecv(inbuf, 1, MPI_CHAR,
prev, 1,
MPI_COMM_WORLD,
&requests[0]));
if(OMPI_SUCCESS != ret) return ret;
ret = ompi_request_wait_all(2, requests, MPI_STATUSES_IGNORE);
if (OMPI_SUCCESS != ret) return ret;
}
return ret;
}
int mca_io_ompio_get_fcoll_dynamic_num_io_procs (int *num_procs)
{
int param;
param = mca_base_var_find("ompi", "fcoll", "dynamic", "num_io_procs");
if (param >= 0){
const int *value = NULL;
mca_base_var_get_value(param, &value, NULL, NULL);
*num_procs = value[0];
/* printf("num procs : %d\n", num_procs);*/
return OMPI_SUCCESS;
}
else
return -1;
}
static int tuned_open(void)
{
int rc;
#if OPAL_ENABLE_DEBUG
{
int param;
param = mca_base_var_find("ompi", "coll", "base", "verbose");
if (param >= 0) {
const int *verbose = NULL;
mca_base_var_get_value(param, &verbose, NULL, NULL);
if (verbose && verbose[0] > 0) {
ompi_coll_tuned_stream = opal_output_open(NULL);
}
}
}
#endif /* OPAL_ENABLE_DEBUG */
/* now check that the user hasn't overrode any of the decision functions if dynamic rules are enabled */
/* the user can redo this before every comm dup/create if they like */
/* this is useful for benchmarking and user knows best tuning */
/* as this is the component we only lookup the indicies of the mca params */
/* the actual values are looked up during comm create via module init */
/* intra functions first */
/* if dynamic rules allowed then look up dynamic rules config filename, else we leave it an empty filename (NULL) */
/* by default DISABLE dynamic rules and instead use fixed [if based] rules */
if (ompi_coll_tuned_use_dynamic_rules) {
if( ompi_coll_tuned_dynamic_rules_filename ) {
OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:component_open Reading collective rules file [%s]",
ompi_coll_tuned_dynamic_rules_filename));
rc = ompi_coll_tuned_read_rules_config_file( ompi_coll_tuned_dynamic_rules_filename,
&(mca_coll_tuned_component.all_base_rules), COLLCOUNT);
if( rc >= 0 ) {
OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:module_open Read %d valid rules\n", rc));
} else {
OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:module_open Reading collective rules file failed\n"));
mca_coll_tuned_component.all_base_rules = NULL;
}
}
}
OPAL_OUTPUT((ompi_coll_tuned_stream, "coll:tuned:component_open: done!"));
return OMPI_SUCCESS;
}
int mca_io_ompio_get_fcoll_dynamic_cycle_buffer_size (int *cycle_buffer_size)
{
int param;
param = mca_base_var_find("ompi", "fcoll", "dynamic", "cycle_buffer_size");
if (param >= 0){
const int *value = NULL;
mca_base_var_get_value(param, &value, NULL, NULL);
*cycle_buffer_size = value[0];
/* printf ("cycle_buffer_size : %d\n", *cycle_buffer_size);*/
return OMPI_SUCCESS;
}
else
return -1;
}
int mca_io_ompio_get_fcoll_dynamic_constant_cbs (int *constant_cbs)
{
int param;
param = mca_base_var_find("ompi", "fcoll", "dynamic", "constant_cbs");
if (param >= 0){
const int *value = NULL;
mca_base_var_get_value(param, &value, NULL, NULL);
*constant_cbs = value[0];
/* printf ("constant_cbs: %d\n", constant_cbs);*/
return OMPI_SUCCESS;
}
else{
constant_cbs[0] = -1;
return OMPI_SUCCESS;
}
}
static bool
check_config_value_equal(char *key, ompi_info_t *info, char *value)
{
char *value_string;
int value_len, ret, flag, param;
const bool *flag_value;
bool result = false;
ret = ompi_info_get_valuelen(info, key, &value_len, &flag);
if (OMPI_SUCCESS != ret) goto info_not_found;
if (flag == 0) goto info_not_found;
value_len++;
value_string = (char*)malloc(sizeof(char) * value_len + 1); /* Should malloc 1 char for NUL-termination */
if (NULL == value_string) goto info_not_found;
ret = ompi_info_get(info, key, value_len, value_string, &flag);
if (OMPI_SUCCESS != ret) {
free(value_string);
goto info_not_found;
}
assert(flag != 0);
if (0 == strcmp(value_string, value)) result = true;
free(value_string);
return result;
info_not_found:
param = mca_base_var_find("ompi", "osc", "portals4", key);
if (0 > param) return false;
ret = mca_base_var_get_value(param, &flag_value, NULL, NULL);
if (OMPI_SUCCESS != ret) return false;
if (0 == strcmp(value_string, value)) result = true;
return result;
}
int mca_bml_r2_ft_event(int state)
{
#if OPAL_ENABLE_FT_CR == 1
static bool first_continue_pass = false;
ompi_proc_t** procs = NULL;
size_t num_procs;
size_t btl_idx;
int ret, p;
int loc_state;
int param_type = -1;
const char **btl_list;
if(OPAL_CRS_CHECKPOINT == state) {
/* Do nothing for now */
}
else if(OPAL_CRS_CONTINUE == state) {
first_continue_pass = !first_continue_pass;
/* Since nothing in Checkpoint, we are fine here (unless required by BTL) */
if (opal_cr_continue_like_restart && !first_continue_pass) {
procs = ompi_proc_all(&num_procs);
if(NULL == procs) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
}
}
else if(OPAL_CRS_RESTART_PRE == state ) {
/* Nothing here */
}
else if(OPAL_CRS_RESTART == state ) {
procs = ompi_proc_all(&num_procs);
if(NULL == procs) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
}
else if(OPAL_CRS_TERM == state ) {
;
}
else {
;
}
/* Never call the ft_event functions attached to the BTLs on the second
* pass of RESTART since on the first pass they were unloaded and therefore
* no longer exist.
*/
if( OPAL_CRS_RESTART != state ) {
if( OPAL_CRS_CONTINUE == state && !first_continue_pass ) {
;
} else {
/* Since we only ever call into the BTLs once during the first restart
* pass, just lie to them on this pass for a bit of local clarity.
*/
if( OPAL_CRS_RESTART_PRE == state ) {
loc_state = OPAL_CRS_RESTART;
} else {
loc_state = state;
}
/*
* Call ft_event in:
* - BTL modules
* - MPool modules
*
* These should be cleaning out stale state, and memory references in
* preparation for being shut down.
*/
for(btl_idx = 0; btl_idx < mca_bml_r2.num_btl_modules; btl_idx++) {
/*
* Notify Mpool
*/
if( NULL != (mca_bml_r2.btl_modules[btl_idx])->btl_mpool &&
NULL != (mca_bml_r2.btl_modules[btl_idx])->btl_mpool->mpool_ft_event ) {
opal_output_verbose(10, ompi_cr_output,
"bml:r2: ft_event: Notify the %s MPool.\n",
(mca_bml_r2.btl_modules[btl_idx])->btl_mpool->mpool_component->mpool_version.mca_component_name);
if(OMPI_SUCCESS != (ret = (mca_bml_r2.btl_modules[btl_idx])->btl_mpool->mpool_ft_event(loc_state) ) ) {
continue;
}
}
/*
* Notify BTL
*/
if( NULL != (mca_bml_r2.btl_modules[btl_idx])->btl_ft_event) {
opal_output_verbose(10, ompi_cr_output,
"bml:r2: ft_event: Notify the %s BTL.\n",
(mca_bml_r2.btl_modules[btl_idx])->btl_component->btl_version.mca_component_name);
if(OMPI_SUCCESS != (ret = (mca_bml_r2.btl_modules[btl_idx])->btl_ft_event(loc_state) ) ) {
continue;
}
}
}
} /* OPAL_CRS_CONTINUE == state && !first_continue_pass */
}
if(OPAL_CRS_CHECKPOINT == state) {
;
}
else if(OPAL_CRS_CONTINUE == state) {
/* Matches OPAL_CRS_RESTART_PRE */
if (opal_cr_continue_like_restart && first_continue_pass) {
if( OMPI_SUCCESS != (ret = mca_bml_r2_finalize()) ) {
opal_output(0, "bml:r2: ft_event(Restart): Failed to finalize BML framework\n");
return ret;
}
if( OMPI_SUCCESS != (ret = mca_base_framework_close(&opal_btl_base_framework)) ) {
opal_output(0, "bml:r2: ft_event(Restart): Failed to close BTL framework\n");
return ret;
}
}
/* Matches OPAL_CRS_RESTART */
else if (opal_cr_continue_like_restart && !first_continue_pass) {
/*
* Barrier to make all processes have been successfully restarted before
* we try to remove some restart only files.
*/
opal_pmix.fence(NULL, 0);
/*
* Re-open the BTL framework to get the full list of components.
*/
if( OMPI_SUCCESS != (ret = mca_base_framework_open(&opal_btl_base_framework, 0)) ) {
opal_output(0, "bml:r2: ft_event(Restart): Failed to open BTL framework\n");
return ret;
}
/*
* Re-select the BTL components/modules
* This will cause the BTL components to discover the available
* network options on this machine, and post proper modex informaiton.
*/
if( OMPI_SUCCESS != (ret = mca_btl_base_select(OPAL_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_THREAD_MULTIPLE) ) ) {
opal_output(0, "bml:r2: ft_event(Restart): Failed to select in BTL framework\n");
return ret;
}
/*
* Clear some structures so we can properly repopulate them
*/
mca_bml_r2.btls_added = false;
for(p = 0; p < (int)num_procs; ++p) {
if( NULL != procs[p]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]) {
OBJ_RELEASE(procs[p]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]);
procs[p]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML] = NULL;
}
OBJ_RELEASE(procs[p]);
}
if( NULL != procs ) {
free(procs);
procs = NULL;
}
}
}
else if(OPAL_CRS_RESTART_PRE == state ) {
opal_output_verbose(10, ompi_cr_output,
"bml:r2: ft_event(Restart): Finalize BML\n");
/*
* Finalize the BML
* - Flush progress functions
* - Flush module references
* - mca_btl_base_close()
* Need to do this because we may have BTL components that were
* unloaded in the first selection that may be available now.
* Conversely we may have BTL components loaded now that
* are not available now.
*/
if( OMPI_SUCCESS != (ret = mca_bml_r2_finalize()) ) {
opal_output(0, "bml:r2: ft_event(Restart): Failed to finalize BML framework\n");
return ret;
}
if( OMPI_SUCCESS != (ret = mca_base_framework_close(&opal_btl_base_framework)) ) {
opal_output(0, "bml:r2: ft_event(Restart): Failed to close BTL framework\n");
return ret;
}
}
else if(OPAL_CRS_RESTART == state ) {
/*
* Barrier to make all processes have been successfully restarted before
* we try to remove some restart only files.
*/
opal_pmix.fence(NULL, 0);
/*
* Re-open the BTL framework to get the full list of components.
* - but first clear the MCA value that was there
*/
param_type = mca_base_var_find("ompi", "btl", NULL, NULL);
btl_list = NULL;
mca_base_var_get_value(param_type, &btl_list, NULL, NULL);
opal_output_verbose(11, ompi_cr_output,
"Restart (Previous BTL MCA): <%s>\n", btl_list ? btl_list[0] : "");
if( OMPI_SUCCESS != (ret = mca_base_framework_open(&opal_btl_base_framework, 0)) ) {
opal_output(0, "bml:r2: ft_event(Restart): Failed to open BTL framework\n");
return ret;
}
/* The reregistered paramter is guaranteed to have the same index */
btl_list = NULL;
mca_base_var_get_value(param_type, &btl_list, NULL, NULL);
opal_output_verbose(11, ompi_cr_output,
"Restart (New BTL MCA): <%s>\n", btl_list ? btl_list[0] : "");
if( NULL != btl_list ) {
free(btl_list);
btl_list = NULL;
}
/*
* Re-select the BTL components/modules
* This will cause the BTL components to discover the available
* network options on this machine, and post proper modex informaiton.
*/
if( OMPI_SUCCESS != (ret = mca_btl_base_select(OPAL_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_THREAD_MULTIPLE) ) ) {
opal_output(0, "bml:r2: ft_event(Restart): Failed to select in BTL framework\n");
return ret;
}
/*
* Clear some structures so we can properly repopulate them
*/
mca_bml_r2.btls_added = false;
for(p = 0; p < (int)num_procs; ++p) {
if( NULL != procs[p]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]) {
OBJ_RELEASE(procs[p]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML]);
procs[p]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML] = NULL;
}
OBJ_RELEASE(procs[p]);
}
if( NULL != procs ) {
free(procs);
procs = NULL;
}
}
else if(OPAL_CRS_TERM == state ) {
;
}
else {
;
}
#endif
return OMPI_SUCCESS;
}
