/*
* This function is called at the initialization time of every
* file. It is used to select which io component will be
* active for a given file.
*/
int mca_io_base_file_select(ompi_file_t *file,
mca_base_component_t *preferred)
{
int err;
char *str;
opal_list_t *selectable;
opal_list_item_t *item;
avail_io_t *avail, selected;
/* Announce */
opal_output_verbose(10, ompi_io_base_framework.framework_output,
"io:base:file_select: new file: %s",
file->f_filename);
/* Initialize all the relevant pointers, since they're used as
sentinel values */
file->f_io_version = MCA_IO_BASE_V_NONE;
file->f_io_selected_data = NULL;
/* Compute the intersection of all of my available components with
the components from all the other processes in this file */
/* JMS CONTINUE HERE */
/* See if a preferred component was provided. If so, try to
select it. If we don't succeed, fall through and do a normal
selection. */
err = OMPI_ERROR;
if (NULL != preferred) {
str = &(preferred->mca_component_name[0]);
opal_output_verbose(10, ompi_io_base_framework.framework_output,
"io:base:file_select: Checking preferred module: %s",
str);
selectable = check_components(&ompi_io_base_framework.framework_components,
file, &str, 1);
/* If we didn't get a preferred module, then call again
without a preferred module. This makes the logic below
dramatically simpler. */
if (NULL == selectable) {
return mca_io_base_file_select(file, NULL);
}
/* We only fall through here if we were able to select one of
the preferred modules */
}
/* Nope -- a specific [set of] component[s] was not requested. Go
check them all. */
else {
opal_output_verbose(10, ompi_io_base_framework.framework_output,
"io:base:file_select: Checking all available modules");
selectable = check_components(&ompi_io_base_framework.framework_components,
file, NULL, 0);
}
/* Upon return from the above, the modules list will contain the
list of modules that returned (priority >= 0). If we have no
io modules available, it's an error */
if (NULL == selectable) {
/* There's no modules available. Doh! */
/* show_help */
return OMPI_ERROR;
}
/* Do some kind of collective operation to find a module that
everyone has available */
#if 1
/* For the moment, just take the top module off the list */
/* MSC actually take the buttom */
item = opal_list_remove_last(selectable);
avail = (avail_io_t *) item;
selected = *avail;
OBJ_RELEASE(avail);
#else
/* JMS CONTINUE HERE */
#endif
/* Everything left in the selectable list is therefore unwanted,
and we call their unquery() method (because they all had
query() invoked, but will never have init() invoked in this
scope). */
for (item = opal_list_remove_first(selectable); item != NULL;
item = opal_list_remove_first(selectable)) {
avail = (avail_io_t *) item;
unquery(avail, file);
OBJ_RELEASE(item);
}
OBJ_RELEASE(selectable);
/* Save the pointers of the selected module on the ompi_file_t */
file->f_io_version = selected.ai_version;
file->f_io_selected_component = selected.ai_component;
file->f_io_selected_module = selected.ai_module;
file->f_io_selected_data = selected.ai_module_data;
if (!strcmp (selected.ai_component.v2_0_0.io_version.mca_component_name,
"ompio")) {
int ret;
if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_fs_base_framework, 0))) {
return err;
}
if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_fcoll_base_framework, 0))) {
return err;
}
if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_fbtl_base_framework, 0))) {
return err;
}
if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_sharedfp_base_framework, 0))) {
return err;
}
if (OMPI_SUCCESS !=
(ret = mca_fs_base_find_available(OPAL_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_THREAD_MULTIPLE))) {
return err;
}
if (OMPI_SUCCESS !=
(ret = mca_fcoll_base_find_available(OPAL_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_THREAD_MULTIPLE))) {
return err;
}
if (OMPI_SUCCESS !=
(ret = mca_fbtl_base_find_available(OPAL_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_THREAD_MULTIPLE))) {
return err;
}
if (OMPI_SUCCESS !=
(ret = mca_sharedfp_base_find_available(OPAL_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_THREAD_MULTIPLE))) {
return err;
}
}
/* Finally -- intialize the selected module. */
if (OMPI_SUCCESS != (err = module_init(file))) {
return err;
}
/* Announce the winner */
opal_output_verbose(10, ompi_io_base_framework.framework_output,
"io:base:file_select: Selected io module %s",
selected.ai_component.v2_0_0.io_version.mca_component_name);
return OMPI_SUCCESS;
}
int mca_bml_r2_ft_event(int state)
{
static bool first_continue_pass = false;
ompi_proc_t** procs = NULL;
size_t num_procs;
size_t btl_idx;
int ret, p;
int loc_state;
int param_type = -1;
char *param_list = NULL;
if(OPAL_CRS_CHECKPOINT == state) {
/* Do nothing for now */
}
else if(OPAL_CRS_CONTINUE == state) {
first_continue_pass = !first_continue_pass;
/* Since nothing in Checkpoint, we are fine here (unless required by BTL) */
if( ompi_cr_continue_like_restart && !first_continue_pass) {
procs = ompi_proc_all(&num_procs);
if(NULL == procs) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
}
}
else if(OPAL_CRS_RESTART_PRE == state ) {
/* Nothing here */
}
else if(OPAL_CRS_RESTART == state ) {
procs = ompi_proc_all(&num_procs);
if(NULL == procs) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
}
else if(OPAL_CRS_TERM == state ) {
;
}
else {
;
}
/* Never call the ft_event functions attached to the BTLs on the second
* pass of RESTART since on the first pass they were unloaded and therefore
* no longer exist.
*/
if( OPAL_CRS_RESTART != state ) {
if( OPAL_CRS_CONTINUE == state && !first_continue_pass ) {
;
} else {
/* Since we only ever call into the BTLs once during the first restart
* pass, just lie to them on this pass for a bit of local clarity.
*/
if( OPAL_CRS_RESTART_PRE == state ) {
loc_state = OPAL_CRS_RESTART;
} else {
loc_state = state;
}
/*
* Call ft_event in:
* - BTL modules
* - MPool modules
*
* These should be cleaning out stale state, and memory references in
* preparation for being shut down.
*/
for(btl_idx = 0; btl_idx < mca_bml_r2.num_btl_modules; btl_idx++) {
/*
* Notify Mpool
*/
if( NULL != (mca_bml_r2.btl_modules[btl_idx])->btl_mpool &&
NULL != (mca_bml_r2.btl_modules[btl_idx])->btl_mpool->mpool_ft_event ) {
opal_output_verbose(10, ompi_cr_output,
"bml:r2: ft_event: Notify the %s MPool.\n",
(mca_bml_r2.btl_modules[btl_idx])->btl_mpool->mpool_component->mpool_version.mca_component_name);
if(OMPI_SUCCESS != (ret = (mca_bml_r2.btl_modules[btl_idx])->btl_mpool->mpool_ft_event(loc_state) ) ) {
continue;
}
}
/*
* Notify BTL
*/
if( NULL != (mca_bml_r2.btl_modules[btl_idx])->btl_ft_event) {
opal_output_verbose(10, ompi_cr_output,
"bml:r2: ft_event: Notify the %s BTL.\n",
(mca_bml_r2.btl_modules[btl_idx])->btl_component->btl_version.mca_component_name);
if(OMPI_SUCCESS != (ret = (mca_bml_r2.btl_modules[btl_idx])->btl_ft_event(loc_state) ) ) {
continue;
}
}
}
} /* OPAL_CRS_CONTINUE == state && !first_continue_pass */
}
if(OPAL_CRS_CHECKPOINT == state) {
;
}
else if(OPAL_CRS_CONTINUE == state) {
/* Matches OPAL_CRS_RESTART_PRE */
if( ompi_cr_continue_like_restart && first_continue_pass) {
if( OMPI_SUCCESS != (ret = mca_bml_r2_finalize()) ) {
opal_output(0, "bml:r2: ft_event(Restart): Failed to finalize BML framework\n");
return ret;
}
}
/* Matches OPAL_CRS_RESTART */
else if( ompi_cr_continue_like_restart && !first_continue_pass ) {
/*
* Barrier to make all processes have been successfully restarted before
* we try to remove some restart only files.
*/
if (OMPI_SUCCESS != (ret = orte_grpcomm.barrier())) {
opal_output(0, "bml:r2: ft_event(Restart): Failed in orte_grpcomm.barrier (%d)", ret);
return ret;
}
opal_output_verbose(10, ompi_cr_output,
"bml:r2: ft_event(Restart): Cleanup restart files\n");
opal_crs_base_cleanup_flush();
/*
* Re-open the BTL framework to get the full list of components.
*/
if( OMPI_SUCCESS != (ret = mca_btl_base_open()) ) {
opal_output(0, "bml:r2: ft_event(Restart): Failed to open BTL framework\n");
return ret;
}
/*
* Re-select the BTL components/modules
* This will cause the BTL components to discover the available
* network options on this machine, and post proper modex informaiton.
*/
if( OMPI_SUCCESS != (ret = mca_btl_base_select(OMPI_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_MPI_THREADS) ) ) {
opal_output(0, "bml:r2: ft_event(Restart): Failed to select in BTL framework\n");
return ret;
}
/*
* Clear some structures so we can properly repopulate them
*/
mca_bml_r2.btls_added = false;
for(p = 0; p < (int)num_procs; ++p) {
if( NULL != procs[p]->proc_bml) {
OBJ_RELEASE(procs[p]->proc_bml);
procs[p]->proc_bml = NULL;
}
OBJ_RELEASE(procs[p]);
}
if( NULL != procs ) {
free(procs);
procs = NULL;
}
}
}
else if(OPAL_CRS_RESTART_PRE == state ) {
opal_output_verbose(10, ompi_cr_output,
"bml:r2: ft_event(Restart): Finalize BML\n");
/*
* Finalize the BML
* - Flush progress functions
* - Flush module references
* - mca_btl_base_close()
* Need to do this because we may have BTL components that were
* unloaded in the first selection that may be available now.
* Conversely we may have BTL components loaded now that
* are not available now.
*/
if( OMPI_SUCCESS != (ret = mca_bml_r2_finalize()) ) {
opal_output(0, "bml:r2: ft_event(Restart): Failed to finalize BML framework\n");
return ret;
}
}
else if(OPAL_CRS_RESTART == state ) {
/*
* Barrier to make all processes have been successfully restarted before
* we try to remove some restart only files.
*/
if (OMPI_SUCCESS != (ret = orte_grpcomm.barrier())) {
opal_output(0, "bml:r2: ft_event(Restart): Failed in orte_grpcomm.barrier (%d)", ret);
return ret;
}
opal_output_verbose(10, ompi_cr_output,
"bml:r2: ft_event(Restart): Cleanup restart files\n");
opal_crs_base_cleanup_flush();
/*
* Re-open the BTL framework to get the full list of components.
* - but first clear the MCA value that was there
*/
param_type = mca_base_param_find("btl", NULL, NULL);
mca_base_param_lookup_string(param_type, ¶m_list);
opal_output_verbose(11, ompi_cr_output,
"Restart (Previous BTL MCA): <%s>\n", param_list);
if( NULL != param_list ) {
free(param_list);
param_list = NULL;
}
/* Deregister the old value, and refresh the file cache to grab any updates */
mca_base_param_deregister(param_type);
mca_base_param_recache_files(false);
if( OMPI_SUCCESS != (ret = mca_btl_base_open()) ) {
opal_output(0, "bml:r2: ft_event(Restart): Failed to open BTL framework\n");
return ret;
}
param_type = mca_base_param_find("btl", NULL, NULL);
mca_base_param_lookup_string(param_type, ¶m_list);
opal_output_verbose(11, ompi_cr_output,
"Restart (New BTL MCA): <%s>\n", param_list);
if( NULL != param_list ) {
free(param_list);
param_list = NULL;
}
/*
* Re-select the BTL components/modules
* This will cause the BTL components to discover the available
* network options on this machine, and post proper modex informaiton.
*/
if( OMPI_SUCCESS != (ret = mca_btl_base_select(OMPI_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_MPI_THREADS) ) ) {
opal_output(0, "bml:r2: ft_event(Restart): Failed to select in BTL framework\n");
return ret;
}
/*
* Clear some structures so we can properly repopulate them
*/
mca_bml_r2.btls_added = false;
for(p = 0; p < (int)num_procs; ++p) {
if( NULL != procs[p]->proc_bml) {
OBJ_RELEASE(procs[p]->proc_bml);
procs[p]->proc_bml = NULL;
}
OBJ_RELEASE(procs[p]);
}
if( NULL != procs ) {
free(procs);
procs = NULL;
}
}
else if(OPAL_CRS_TERM == state ) {
;
}
else {
;
}
return OMPI_SUCCESS;
}
static void recv_data(int fd, short args, void *cbdata)
{
bool found;
int i, rc;
orte_node_t *nd, *nd2;
opal_list_t nds, ndtmp;
opal_list_item_t *item, *itm;
char recv_msg[8192];
int nbytes, idx, sjob;
char **alloc, *nodelist, *tpn;
local_jobtracker_t *ptr, *jtrk;
local_apptracker_t *aptrk;
orte_app_context_t *app;
orte_jobid_t jobid;
orte_job_t *jdata;
opal_output_verbose(2, orte_ras_base_framework.framework_output,
"%s ras:slurm: dynamic allocation - data recvd",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
/* read the data from the socket and put it in the
* nodes field of op
*/
memset(recv_msg, 0, sizeof(recv_msg));
nbytes = read(fd, recv_msg, sizeof(recv_msg) - 1);
opal_output_verbose(2, orte_ras_base_framework.framework_output,
"%s ras:slurm: dynamic allocation msg: %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), recv_msg);
/* check if we got something */
if (0 == nbytes || 0 == strlen(recv_msg) || strstr(recv_msg, "failure") != NULL) {
/* show an error here - basically, a "nothing was available"
* message
*/
orte_show_help("help-ras-slurm.txt", "slurm-dyn-alloc-failed", true,
(0 == strlen(recv_msg)) ? "NO MSG" : recv_msg);
ORTE_ACTIVATE_JOB_STATE(NULL, ORTE_JOB_STATE_ALLOC_FAILED);
return;
}
/* break the message into its component parts, separated by colons */
alloc = opal_argv_split(recv_msg, ':');
/* the first section contains the ORTE jobid for this allocation */
tpn = strchr(alloc[0], '=');
orte_util_convert_string_to_jobid(&jobid, tpn+1);
/* get the corresponding job object */
jdata = orte_get_job_data_object(jobid);
jtrk = NULL;
/* find the associated tracking object */
for (item = opal_list_get_first(&jobs);
item != opal_list_get_end(&jobs);
item = opal_list_get_next(item)) {
ptr = (local_jobtracker_t*)item;
if (ptr->jobid == jobid) {
jtrk = ptr;
break;
}
}
if (NULL == jtrk) {
orte_show_help("help-ras-slurm.txt", "slurm-dyn-alloc-failed", true, "NO JOB TRACKER");
ORTE_ACTIVATE_JOB_STATE(NULL, ORTE_JOB_STATE_ALLOC_FAILED);
opal_argv_free(alloc);
return;
}
/* stop the timeout event */
opal_event_del(&jtrk->timeout_ev);
/* cycle across all the remaining parts - each is the allocation for
* an app in this job
*/
OBJ_CONSTRUCT(&nds, opal_list_t);
OBJ_CONSTRUCT(&ndtmp, opal_list_t);
idx = -1;
sjob = -1;
nodelist = NULL;
for (i=1; NULL != alloc[i]; i++) {
if (ORTE_SUCCESS != parse_alloc_msg(alloc[i], &idx, &sjob, &nodelist, &tpn)) {
orte_show_help("help-ras-slurm.txt", "slurm-dyn-alloc-failed", true, jtrk->cmd);
ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_ALLOC_FAILED);
return;
}
if (idx < 0 || NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, idx))) {
orte_show_help("help-ras-slurm.txt", "slurm-dyn-alloc-failed", true, jtrk->cmd);
ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_ALLOC_FAILED);
opal_argv_free(alloc);
return;
}
/* track the Slurm jobid */
if (NULL == (aptrk = (local_apptracker_t*)opal_pointer_array_get_item(&jtrk->apps, idx))) {
aptrk = OBJ_NEW(local_apptracker_t);
opal_pointer_array_set_item(&jtrk->apps, idx, aptrk);
}
aptrk->sjob = sjob;
/* release the current dash_host as that contained the *desired* allocation */
opal_argv_free(app->dash_host);
app->dash_host = NULL;
/* since the nodelist/tpn may contain regular expressions, parse them */
if (ORTE_SUCCESS != (rc = orte_ras_slurm_discover(nodelist, tpn, &ndtmp))) {
ORTE_ERROR_LOG(rc);
ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_ALLOC_FAILED);
opal_argv_free(alloc);
return;
}
/* transfer the discovered nodes to our node list, and construct
* the new dash_host entry to match what was allocated
*/
while (NULL != (item = opal_list_remove_first(&ndtmp))) {
nd = (orte_node_t*)item;
opal_argv_append_nosize(&app->dash_host, nd->name);
/* check for duplicates */
found = false;
for (itm = opal_list_get_first(&nds);
itm != opal_list_get_end(&nds);
itm = opal_list_get_next(itm)) {
nd2 = (orte_node_t*)itm;
if (0 == strcmp(nd->name, nd2->name)) {
found = true;
nd2->slots += nd->slots;
OBJ_RELEASE(item);
break;
}
}
if (!found) {
/* append the new node to our list */
opal_list_append(&nds, item);
}
}
/* cleanup */
free(nodelist);
free(tpn);
}
/* cleanup */
opal_argv_free(alloc);
OBJ_DESTRUCT(&ndtmp);
if (opal_list_is_empty(&nds)) {
/* if we get here, then we were able to contact slurm,
* which means we are in an actively managed cluster.
* However, slurm indicated that nothing is currently
* available that meets our requirements. This is a fatal
* situation - we do NOT have the option of running on
* user-specified hosts as the cluster is managed.
*/
OBJ_DESTRUCT(&nds);
orte_show_help("help-ras-base.txt", "ras-base:no-allocation", true);
ORTE_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
}
/* store the found nodes */
if (ORTE_SUCCESS != (rc = orte_ras_base_node_insert(&nds, jdata))) {
ORTE_ERROR_LOG(rc);
OBJ_DESTRUCT(&nds);
ORTE_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
return;
}
OBJ_DESTRUCT(&nds);
/* default to no-oversubscribe-allowed for managed systems */
if (!(ORTE_MAPPING_SUBSCRIBE_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping))) {
ORTE_SET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping, ORTE_MAPPING_NO_OVERSUBSCRIBE);
}
/* flag that the allocation is managed */
orte_managed_allocation = true;
/* move the job along */
ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_ALLOCATION_COMPLETE);
/* all done */
return;
}
static void process_close(int fd, short args, void *cbdata)
{
orte_dfs_request_t *close_dfs = (orte_dfs_request_t*)cbdata;
orte_dfs_tracker_t *tptr, *trk;
opal_list_item_t *item;
opal_buffer_t *buffer;
int rc;
opal_output_verbose(1, orte_dfs_base_framework.framework_output,
"%s closing fd %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
close_dfs->local_fd);
/* look in our local records for this fd */
trk = NULL;
for (item = opal_list_get_first(&active_files);
item != opal_list_get_end(&active_files);
item = opal_list_get_next(item)) {
tptr = (orte_dfs_tracker_t*)item;
if (tptr->local_fd == close_dfs->local_fd) {
trk = tptr;
break;
}
}
if (NULL == trk) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
if (NULL != close_dfs->close_cbfunc) {
close_dfs->close_cbfunc(close_dfs->local_fd, close_dfs->cbdata);
}
OBJ_RELEASE(close_dfs);
return;
}
/* setup a message for the daemon telling
* them what file to close
*/
buffer = OBJ_NEW(opal_buffer_t);
if (OPAL_SUCCESS != (rc = opal_dss.pack(buffer, &close_dfs->cmd, 1, ORTE_DFS_CMD_T))) {
ORTE_ERROR_LOG(rc);
goto complete;
}
if (OPAL_SUCCESS != (rc = opal_dss.pack(buffer, &trk->remote_fd, 1, OPAL_INT))) {
ORTE_ERROR_LOG(rc);
goto complete;
}
opal_output_verbose(1, orte_dfs_base_framework.framework_output,
"%s sending close file request to %s for fd %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&trk->host_daemon),
trk->local_fd);
/* send it */
if (0 > (rc = orte_rml.send_buffer_nb(&trk->host_daemon, buffer,
ORTE_RML_TAG_DFS_CMD,
orte_rml_send_callback, NULL))) {
ORTE_ERROR_LOG(rc);
OBJ_RELEASE(buffer);
goto complete;
}
complete:
opal_list_remove_item(&active_files, &trk->super);
OBJ_RELEASE(trk);
if (NULL != close_dfs->close_cbfunc) {
close_dfs->close_cbfunc(close_dfs->local_fd, close_dfs->cbdata);
}
OBJ_RELEASE(close_dfs);
}
static void process_reads(int fd, short args, void *cbdata)
{
orte_dfs_request_t *read_dfs = (orte_dfs_request_t*)cbdata;
orte_dfs_tracker_t *tptr, *trk;
opal_list_item_t *item;
opal_buffer_t *buffer;
int64_t i64;
int rc;
/* look in our local records for this fd */
trk = NULL;
for (item = opal_list_get_first(&active_files);
item != opal_list_get_end(&active_files);
item = opal_list_get_next(item)) {
tptr = (orte_dfs_tracker_t*)item;
if (tptr->local_fd == read_dfs->local_fd) {
trk = tptr;
break;
}
}
if (NULL == trk) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
OBJ_RELEASE(read_dfs);
return;
}
/* add this request to our pending list */
read_dfs->id = req_id++;
opal_list_append(&requests, &read_dfs->super);
/* setup a message for the daemon telling
* them what file to read
*/
buffer = OBJ_NEW(opal_buffer_t);
if (OPAL_SUCCESS != (rc = opal_dss.pack(buffer, &read_dfs->cmd, 1, ORTE_DFS_CMD_T))) {
ORTE_ERROR_LOG(rc);
goto complete;
}
/* include the request id */
if (OPAL_SUCCESS != (rc = opal_dss.pack(buffer, &read_dfs->id, 1, OPAL_UINT64))) {
ORTE_ERROR_LOG(rc);
goto complete;
}
if (OPAL_SUCCESS != (rc = opal_dss.pack(buffer, &trk->remote_fd, 1, OPAL_INT))) {
ORTE_ERROR_LOG(rc);
goto complete;
}
i64 = (int64_t)read_dfs->read_length;
if (OPAL_SUCCESS != (rc = opal_dss.pack(buffer, &i64, 1, OPAL_INT64))) {
ORTE_ERROR_LOG(rc);
goto complete;
}
opal_output_verbose(1, orte_dfs_base_framework.framework_output,
"%s sending read file request to %s for fd %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&trk->host_daemon),
trk->local_fd);
/* send it */
if (0 > (rc = orte_rml.send_buffer_nb(&trk->host_daemon, buffer,
ORTE_RML_TAG_DFS_CMD,
orte_rml_send_callback, NULL))) {
ORTE_ERROR_LOG(rc);
OBJ_RELEASE(buffer);
}
/* don't release the request */
return;
complete:
/* don't need to hang on to this request */
opal_list_remove_item(&requests, &read_dfs->super);
OBJ_RELEASE(read_dfs);
}
/*
* Open up all directories in a given path and search for components of
* the specified type (and possibly of a given name).
*
* Note that we use our own path iteration functionality (vs. ltdl's
* lt_dladdsearchdir() functionality) because we need to look at
* companion .ompi_info files in the same directory as the library to
* generate dependencies, etc. If we use the plain lt_dlopen()
* functionality, we would not get the directory name of the file
* finally opened in recursive dependency traversals.
*/
static void find_dyn_components(const char *path, const char *type_name,
const char **names, bool include_mode,
opal_list_t *found_components)
{
int i, len;
char *path_to_use = NULL, *dir, *end;
component_file_item_t *file;
opal_list_item_t *cur;
char prefix[32 + MCA_BASE_MAX_TYPE_NAME_LEN], *basename;
/* If path is NULL, iterate over the set of directories specified by
the MCA param mca_base_component_path. If path is not NULL, then
use that as the path. */
if (NULL == path) {
if (NULL != mca_base_component_path) {
path_to_use = strdup (mca_base_component_path);
} else {
/* If there's no path, then there's nothing to search -- we're
done */
return;
}
if (NULL == path_to_use) {
/* out of memory */
return;
}
} else {
path_to_use = strdup(path);
}
/* If we haven't done so already, iterate over all the files in
the directories in the path and make a master array of all the
matching filenames that we find. Save the filenames in an
argv-style array. Re-scan do this if the mca_component_path
has changed. */
if (NULL == found_filenames ||
(NULL != last_path_to_use &&
0 != strcmp(path_to_use, last_path_to_use))) {
if (NULL != found_filenames) {
opal_argv_free(found_filenames);
found_filenames = NULL;
free(last_path_to_use);
last_path_to_use = NULL;
}
if (NULL == last_path_to_use) {
last_path_to_use = strdup(path_to_use);
}
dir = path_to_use;
if (NULL != dir) {
do {
end = strchr(dir, OPAL_ENV_SEP);
if (NULL != end) {
*end = '\0';
}
if ((0 == strcmp(dir, "USER_DEFAULT") ||
0 == strcmp(dir, "USR_DEFAULT"))
&& NULL != mca_base_user_default_path) {
if (0 != lt_dlforeachfile(mca_base_user_default_path,
save_filename, NULL)) {
break;
}
} else if (0 == strcmp(dir, "SYS_DEFAULT") ||
0 == strcmp(dir, "SYSTEM_DEFAULT")) {
if (0 != lt_dlforeachfile(mca_base_system_default_path,
save_filename, NULL)) {
break;
}
} else {
if (0 != lt_dlforeachfile(dir, save_filename, NULL)) {
break;
}
}
dir = end + 1;
} while (NULL != end);
}
}
/* Look through the list of found files and find those that match
the desired framework name */
snprintf(prefix, sizeof(prefix) - 1, component_template, type_name);
len = strlen(prefix);
OBJ_CONSTRUCT(&found_files, opal_list_t);
for (i = 0; NULL != found_filenames && NULL != found_filenames[i]; ++i) {
basename = strrchr(found_filenames[i], '/');
if (NULL == basename) {
basename = found_filenames[i];
} else {
basename += 1;
}
if (0 != strncmp(basename, prefix, len)) {
continue;
}
/* We found a match; save all the relevant details in the
found_files list */
file = OBJ_NEW(component_file_item_t);
if (NULL == file) {
return;
}
strncpy(file->type, type_name, MCA_BASE_MAX_TYPE_NAME_LEN);
file->type[MCA_BASE_MAX_TYPE_NAME_LEN] = '\0';
strncpy(file->name, basename + len, MCA_BASE_MAX_COMPONENT_NAME_LEN);
file->name[MCA_BASE_MAX_COMPONENT_NAME_LEN] = '\0';
strncpy(file->basename, basename, OPAL_PATH_MAX);
file->basename[OPAL_PATH_MAX] = '\0';
strncpy(file->filename, found_filenames[i], OPAL_PATH_MAX);
file->filename[OPAL_PATH_MAX] = '\0';
file->status = UNVISITED;
opal_list_append(&found_files, (opal_list_item_t *)
file);
}
/* Iterate through all the filenames that we found that matched
the framework we were looking for. Since one component may
[try to] call another to be loaded, only try to load the
UNVISITED files. Also, ignore the return code -- basically,
give every file one chance to try to load. If they load,
great. If not, great. */
for (cur = opal_list_get_first(&found_files);
opal_list_get_end(&found_files) != cur;
cur = opal_list_get_next(cur)) {
file = (component_file_item_t *) cur;
if( UNVISITED == file->status ) {
bool op = true;
file->status = CHECKING_CYCLE;
op = use_component(include_mode, names, file->name);
if( true == op ) {
open_component(file, found_components);
}
}
}
/* So now we have a final list of loaded components. We can free all
the file information. */
for (cur = opal_list_remove_first(&found_files);
NULL != cur;
cur = opal_list_remove_first(&found_files)) {
OBJ_RELEASE(cur);
}
OBJ_DESTRUCT(&found_files);
/* All done, now let's cleanup */
free(path_to_use);
}
/* receives take place in an event, so we are free to process
* the request list without fear of getting things out-of-order
*/
static void recv_dfs(int status, orte_process_name_t* sender,
opal_buffer_t* buffer, orte_rml_tag_t tag,
void* cbdata)
{
orte_dfs_cmd_t cmd;
int32_t cnt;
orte_dfs_request_t *dfs, *dptr;
opal_list_item_t *item;
int remote_fd, rc;
int64_t i64;
uint64_t rid;
orte_dfs_tracker_t *trk;
/* unpack the command this message is responding to */
cnt = 1;
if (OPAL_SUCCESS != (rc = opal_dss.unpack(buffer, &cmd, &cnt, ORTE_DFS_CMD_T))) {
ORTE_ERROR_LOG(rc);
return;
}
opal_output_verbose(1, orte_dfs_base_framework.framework_output,
"%s recvd cmd %d from sender %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), (int)cmd,
ORTE_NAME_PRINT(sender));
switch (cmd) {
case ORTE_DFS_OPEN_CMD:
/* unpack the request id */
cnt = 1;
if (OPAL_SUCCESS != (rc = opal_dss.unpack(buffer, &rid, &cnt, OPAL_UINT64))) {
ORTE_ERROR_LOG(rc);
return;
}
/* unpack the remote fd */
cnt = 1;
if (OPAL_SUCCESS != (rc = opal_dss.unpack(buffer, &remote_fd, &cnt, OPAL_INT))) {
ORTE_ERROR_LOG(rc);
return;
}
/* search our list of requests to find the matching one */
dfs = NULL;
for (item = opal_list_get_first(&requests);
item != opal_list_get_end(&requests);
item = opal_list_get_next(item)) {
dptr = (orte_dfs_request_t*)item;
if (dptr->id == rid) {
/* as the request has been fulfilled, remove it */
opal_list_remove_item(&requests, item);
dfs = dptr;
break;
}
}
if (NULL == dfs) {
opal_output_verbose(1, orte_dfs_base_framework.framework_output,
"%s recvd open file - no corresponding request found for local fd %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), local_fd);
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return;
}
/* if the remote_fd < 0, then we had an error, so return
* the error value to the caller
*/
if (remote_fd < 0) {
opal_output_verbose(1, orte_dfs_base_framework.framework_output,
"%s recvd open file response error file %s [error: %d]",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
dfs->uri, remote_fd);
if (NULL != dfs->open_cbfunc) {
dfs->open_cbfunc(remote_fd, dfs->cbdata);
}
/* release the request */
OBJ_RELEASE(dfs);
return;
}
/* otherwise, create a tracker for this file */
trk = OBJ_NEW(orte_dfs_tracker_t);
trk->requestor.jobid = ORTE_PROC_MY_NAME->jobid;
trk->requestor.vpid = ORTE_PROC_MY_NAME->vpid;
trk->host_daemon.jobid = sender->jobid;
trk->host_daemon.vpid = sender->vpid;
trk->filename = strdup(dfs->uri);
/* define the local fd */
trk->local_fd = local_fd++;
/* record the remote file descriptor */
trk->remote_fd = remote_fd;
/* add it to our list of active files */
opal_list_append(&active_files, &trk->super);
/* return the local_fd to the caller for
* subsequent operations
*/
opal_output_verbose(1, orte_dfs_base_framework.framework_output,
"%s recvd open file completed for file %s [local fd: %d remote fd: %d]",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
dfs->uri, trk->local_fd, remote_fd);
if (NULL != dfs->open_cbfunc) {
dfs->open_cbfunc(trk->local_fd, dfs->cbdata);
}
/* release the request */
OBJ_RELEASE(dfs);
break;
case ORTE_DFS_SIZE_CMD:
/* unpack the request id for this request */
cnt = 1;
if (OPAL_SUCCESS != (rc = opal_dss.unpack(buffer, &rid, &cnt, OPAL_UINT64))) {
ORTE_ERROR_LOG(rc);
return;
}
/* search our list of requests to find the matching one */
dfs = NULL;
for (item = opal_list_get_first(&requests);
item != opal_list_get_end(&requests);
item = opal_list_get_next(item)) {
dptr = (orte_dfs_request_t*)item;
if (dptr->id == rid) {
/* request was fulfilled, so remove it */
opal_list_remove_item(&requests, item);
dfs = dptr;
break;
}
}
if (NULL == dfs) {
opal_output_verbose(1, orte_dfs_base_framework.framework_output,
"%s recvd size - no corresponding request found for local fd %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), local_fd);
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return;
}
/* get the size */
cnt = 1;
if (OPAL_SUCCESS != (rc = opal_dss.unpack(buffer, &i64, &cnt, OPAL_INT64))) {
ORTE_ERROR_LOG(rc);
OBJ_RELEASE(dfs);
return;
}
/* pass it back to the original caller */
if (NULL != dfs->size_cbfunc) {
dfs->size_cbfunc(i64, dfs->cbdata);
}
/* release the request */
OBJ_RELEASE(dfs);
break;
case ORTE_DFS_SEEK_CMD:
/* unpack the request id for this read */
cnt = 1;
if (OPAL_SUCCESS != (rc = opal_dss.unpack(buffer, &rid, &cnt, OPAL_UINT64))) {
ORTE_ERROR_LOG(rc);
return;
}
/* search our list of requests to find the matching one */
dfs = NULL;
for (item = opal_list_get_first(&requests);
item != opal_list_get_end(&requests);
item = opal_list_get_next(item)) {
dptr = (orte_dfs_request_t*)item;
if (dptr->id == rid) {
/* request was fulfilled, so remove it */
opal_list_remove_item(&requests, item);
dfs = dptr;
break;
}
}
if (NULL == dfs) {
opal_output_verbose(1, orte_dfs_base_framework.framework_output,
"%s recvd seek - no corresponding request found for local fd %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), local_fd);
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return;
}
/* get the returned offset/status */
cnt = 1;
if (OPAL_SUCCESS != (rc = opal_dss.unpack(buffer, &i64, &cnt, OPAL_INT64))) {
ORTE_ERROR_LOG(rc);
OBJ_RELEASE(dfs);
return;
}
/* pass it back to the original caller */
if (NULL != dfs->seek_cbfunc) {
dfs->seek_cbfunc(i64, dfs->cbdata);
}
/* release the request */
OBJ_RELEASE(dfs);
break;
case ORTE_DFS_READ_CMD:
/* unpack the request id for this read */
cnt = 1;
if (OPAL_SUCCESS != (rc = opal_dss.unpack(buffer, &rid, &cnt, OPAL_UINT64))) {
ORTE_ERROR_LOG(rc);
return;
}
/* search our list of requests to find the matching one */
dfs = NULL;
for (item = opal_list_get_first(&requests);
item != opal_list_get_end(&requests);
item = opal_list_get_next(item)) {
dptr = (orte_dfs_request_t*)item;
if (dptr->id == rid) {
/* request was fulfilled, so remove it */
opal_list_remove_item(&requests, item);
dfs = dptr;
break;
}
}
if (NULL == dfs) {
opal_output_verbose(1, orte_dfs_base_framework.framework_output,
"%s recvd read - no corresponding request found for local fd %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), local_fd);
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return;
}
/* get the bytes read */
cnt = 1;
if (OPAL_SUCCESS != (rc = opal_dss.unpack(buffer, &i64, &cnt, OPAL_INT64))) {
ORTE_ERROR_LOG(rc);
OBJ_RELEASE(dfs);
return;
}
if (0 < i64) {
cnt = i64;
if (OPAL_SUCCESS != (rc = opal_dss.unpack(buffer, dfs->read_buffer, &cnt, OPAL_UINT8))) {
ORTE_ERROR_LOG(rc);
OBJ_RELEASE(dfs);
return;
}
}
/* pass them back to the original caller */
if (NULL != dfs->read_cbfunc) {
dfs->read_cbfunc(i64, dfs->read_buffer, dfs->cbdata);
}
/* release the request */
OBJ_RELEASE(dfs);
break;
case ORTE_DFS_POST_CMD:
/* unpack the request id for this read */
cnt = 1;
if (OPAL_SUCCESS != (rc = opal_dss.unpack(buffer, &rid, &cnt, OPAL_UINT64))) {
ORTE_ERROR_LOG(rc);
return;
}
/* search our list of requests to find the matching one */
dfs = NULL;
for (item = opal_list_get_first(&requests);
item != opal_list_get_end(&requests);
item = opal_list_get_next(item)) {
dptr = (orte_dfs_request_t*)item;
if (dptr->id == rid) {
/* request was fulfilled, so remove it */
opal_list_remove_item(&requests, item);
dfs = dptr;
break;
}
}
if (NULL == dfs) {
opal_output_verbose(1, orte_dfs_base_framework.framework_output,
"%s recvd post - no corresponding request found",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return;
}
if (NULL != dfs->post_cbfunc) {
dfs->post_cbfunc(dfs->cbdata);
}
OBJ_RELEASE(dfs);
break;
case ORTE_DFS_GETFM_CMD:
/* unpack the request id for this read */
cnt = 1;
if (OPAL_SUCCESS != (rc = opal_dss.unpack(buffer, &rid, &cnt, OPAL_UINT64))) {
ORTE_ERROR_LOG(rc);
return;
}
/* search our list of requests to find the matching one */
dfs = NULL;
for (item = opal_list_get_first(&requests);
item != opal_list_get_end(&requests);
item = opal_list_get_next(item)) {
dptr = (orte_dfs_request_t*)item;
if (dptr->id == rid) {
/* request was fulfilled, so remove it */
opal_list_remove_item(&requests, item);
dfs = dptr;
break;
}
}
if (NULL == dfs) {
opal_output_verbose(1, orte_dfs_base_framework.framework_output,
"%s recvd getfm - no corresponding request found",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return;
}
/* return it to caller */
if (NULL != dfs->fm_cbfunc) {
dfs->fm_cbfunc(buffer, dfs->cbdata);
}
OBJ_RELEASE(dfs);
break;
default:
opal_output(0, "TEST:DFS:RECV WTF");
break;
}
}
void orte_iof_orted_read_handler(int fd, short event, void *cbdata)
{
orte_iof_read_event_t *rev = (orte_iof_read_event_t*)cbdata;
unsigned char data[ORTE_IOF_BASE_MSG_MAX];
opal_buffer_t *buf=NULL;
int rc;
int32_t numbytes;
orte_iof_proc_t *proct = (orte_iof_proc_t*)rev->proc;
ORTE_ACQUIRE_OBJECT(rev);
/* As we may use timer events, fd can be bogus (-1)
* use the right one here
*/
fd = rev->fd;
/* read up to the fragment size */
#if !defined(__WINDOWS__)
numbytes = read(fd, data, sizeof(data));
#else
{
DWORD readed;
HANDLE handle = (HANDLE)_get_osfhandle(fd);
ReadFile(handle, data, sizeof(data), &readed, NULL);
numbytes = (int)readed;
}
#endif /* !defined(__WINDOWS__) */
if (NULL == proct) {
/* nothing we can do */
ORTE_ERROR_LOG(ORTE_ERR_ADDRESSEE_UNKNOWN);
return;
}
OPAL_OUTPUT_VERBOSE((1, orte_iof_base_framework.framework_output,
"%s iof:orted:read handler read %d bytes from %s, fd %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
numbytes, ORTE_NAME_PRINT(&proct->name), fd));
if (numbytes <= 0) {
if (0 > numbytes) {
/* either we have a connection error or it was a non-blocking read */
if (EAGAIN == errno || EINTR == errno) {
/* non-blocking, retry */
ORTE_IOF_READ_ACTIVATE(rev);
return;
}
OPAL_OUTPUT_VERBOSE((1, orte_iof_base_framework.framework_output,
"%s iof:orted:read handler %s Error on connection:%d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&proct->name), fd));
}
/* numbytes must have been zero, so go down and close the fd etc */
goto CLEAN_RETURN;
}
/* see if the user wanted the output directed to files */
if (NULL != rev->sink) {
/* output to the corresponding file */
orte_iof_base_write_output(&proct->name, rev->tag, data, numbytes, rev->sink->wev);
}
if (!proct->copy) {
/* re-add the event */
ORTE_IOF_READ_ACTIVATE(rev);
return;
}
/* prep the buffer */
buf = OBJ_NEW(opal_buffer_t);
/* pack the stream first - we do this so that flow control messages can
* consist solely of the tag
*/
if (ORTE_SUCCESS != (rc = opal_dss.pack(buf, &rev->tag, 1, ORTE_IOF_TAG))) {
ORTE_ERROR_LOG(rc);
goto CLEAN_RETURN;
}
/* pack name of process that gave us this data */
if (ORTE_SUCCESS != (rc = opal_dss.pack(buf, &proct->name, 1, ORTE_NAME))) {
ORTE_ERROR_LOG(rc);
goto CLEAN_RETURN;
}
/* pack the data - only pack the #bytes we read! */
if (ORTE_SUCCESS != (rc = opal_dss.pack(buf, &data, numbytes, OPAL_BYTE))) {
ORTE_ERROR_LOG(rc);
goto CLEAN_RETURN;
}
/* start non-blocking RML call to forward received data */
OPAL_OUTPUT_VERBOSE((1, orte_iof_base_framework.framework_output,
"%s iof:orted:read handler sending %d bytes to HNP",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), numbytes));
orte_rml.send_buffer_nb(orte_mgmt_conduit,
ORTE_PROC_MY_HNP, buf, ORTE_RML_TAG_IOF_HNP,
orte_rml_send_callback, NULL);
/* re-add the event */
ORTE_IOF_READ_ACTIVATE(rev);
return;
CLEAN_RETURN:
/* must be an error, or zero bytes were read indicating that the
* proc terminated this IOF channel - either way, release the
* corresponding event. This deletes the read event and closes
* the file descriptor */
if (rev->tag & ORTE_IOF_STDOUT) {
if( NULL != proct->revstdout ) {
orte_iof_base_static_dump_output(proct->revstdout);
OBJ_RELEASE(proct->revstdout);
}
} else if (rev->tag & ORTE_IOF_STDERR) {
if( NULL != proct->revstderr ) {
orte_iof_base_static_dump_output(proct->revstderr);
OBJ_RELEASE(proct->revstderr);
}
} else if (rev->tag & ORTE_IOF_STDDIAG) {
if( NULL != proct->revstddiag ) {
orte_iof_base_static_dump_output(proct->revstddiag);
OBJ_RELEASE(proct->revstddiag);
}
}
/* check to see if they are all done */
if (NULL == proct->revstdout &&
NULL == proct->revstderr &&
NULL == proct->revstddiag) {
/* this proc's iof is complete */
ORTE_ACTIVATE_PROC_STATE(&proct->name, ORTE_PROC_STATE_IOF_COMPLETE);
}
if (NULL != buf) {
OBJ_RELEASE(buf);
}
return;
}
/* look up the remote pointer in the peer rcache and attach if
* necessary */
mca_mpool_base_registration_t *vader_get_registation (struct mca_btl_base_endpoint_t *ep, void *rem_ptr,
size_t size, int flags, void **local_ptr)
{
struct mca_rcache_base_module_t *rcache = ep->segment_data.xpmem.rcache;
mca_mpool_base_registration_t *regs[10], *reg = NULL;
xpmem_addr_t xpmem_addr;
uintptr_t base, bound;
int rc, i;
/* protect rcache access */
OPAL_THREAD_LOCK(&ep->lock);
/* use btl/self for self communication */
assert (ep->peer_smp_rank != MCA_BTL_VADER_LOCAL_RANK);
base = (uintptr_t) down_align_addr(rem_ptr, mca_btl_vader_component.log_attach_align);
bound = (uintptr_t) up_align_addr((void *)((uintptr_t) rem_ptr + size - 1),
mca_btl_vader_component.log_attach_align) + 1;
if (OPAL_UNLIKELY(bound > VADER_MAX_ADDRESS)) {
bound = VADER_MAX_ADDRESS;
}
/* several segments may match the base pointer */
rc = rcache->rcache_find_all (rcache, (void *) base, bound - base, regs, 10);
for (i = 0 ; i < rc ; ++i) {
if (bound <= (uintptr_t)regs[i]->bound && base >= (uintptr_t)regs[i]->base) {
(void)opal_atomic_add (®s[i]->ref_count, 1);
reg = regs[i];
goto reg_found;
}
if (regs[i]->flags & MCA_MPOOL_FLAGS_PERSIST) {
continue;
}
/* remove this pointer from the rcache and decrement its reference count
(so it is detached later) */
rc = rcache->rcache_delete (rcache, regs[i]);
if (OPAL_UNLIKELY(0 != rc)) {
/* someone beat us to it? */
break;
}
/* start the new segment from the lower of the two bases */
base = (uintptr_t) regs[i]->base < base ? (uintptr_t) regs[i]->base : base;
(void)opal_atomic_add (®s[i]->ref_count, -1);
if (OPAL_LIKELY(0 == regs[i]->ref_count)) {
/* this pointer is not in use */
(void) xpmem_detach (regs[i]->alloc_base);
OBJ_RELEASE(regs[i]);
}
break;
}
reg = OBJ_NEW(mca_mpool_base_registration_t);
if (OPAL_LIKELY(NULL != reg)) {
/* stick around for awhile */
reg->ref_count = 2;
reg->base = (unsigned char *) base;
reg->bound = (unsigned char *) bound;
reg->flags = flags;
#if defined(HAVE_SN_XPMEM_H)
xpmem_addr.id = ep->segment_data.xpmem.apid;
#else
xpmem_addr.apid = ep->segment_data.xpmem.apid;
#endif
xpmem_addr.offset = base;
reg->alloc_base = xpmem_attach (xpmem_addr, bound - base, NULL);
if (OPAL_UNLIKELY((void *)-1 == reg->alloc_base)) {
OPAL_THREAD_UNLOCK(&ep->lock);
OBJ_RELEASE(reg);
return NULL;
}
opal_memchecker_base_mem_defined (reg->alloc_base, bound - base);
rcache->rcache_insert (rcache, reg, 0);
}
reg_found:
opal_atomic_wmb ();
*local_ptr = (void *) ((uintptr_t) reg->alloc_base +
(ptrdiff_t)((uintptr_t) rem_ptr - (uintptr_t) reg->base));
OPAL_THREAD_UNLOCK(&ep->lock);
return reg;
}
int mca_btl_tcp_add_procs( struct mca_btl_base_module_t* btl,
size_t nprocs,
struct opal_proc_t **procs,
struct mca_btl_base_endpoint_t** peers,
opal_bitmap_t* reachable )
{
mca_btl_tcp_module_t* tcp_btl = (mca_btl_tcp_module_t*)btl;
const opal_proc_t* my_proc; /* pointer to caller's proc structure */
int i, rc;
/* get pointer to my proc structure */
if( NULL == (my_proc = opal_proc_local_get()) )
return OPAL_ERR_OUT_OF_RESOURCE;
for(i = 0; i < (int) nprocs; i++) {
struct opal_proc_t* opal_proc = procs[i];
mca_btl_tcp_proc_t* tcp_proc;
mca_btl_base_endpoint_t* tcp_endpoint;
bool existing_found = false;
/* Do not create loopback TCP connections */
if( my_proc == opal_proc ) {
continue;
}
if(NULL == (tcp_proc = mca_btl_tcp_proc_create(opal_proc))) {
continue;
}
/*
* Check to make sure that the peer has at least as many interface
* addresses exported as we are trying to use. If not, then
* don't bind this BTL instance to the proc.
*/
OPAL_THREAD_LOCK(&tcp_proc->proc_lock);
for (uint32_t j = 0 ; j < (uint32_t)tcp_proc->proc_endpoint_count ; ++j) {
tcp_endpoint = tcp_proc->proc_endpoints[j];
if (tcp_endpoint->endpoint_btl == tcp_btl) {
existing_found = true;
break;
}
}
if (!existing_found) {
/* The btl_proc datastructure is shared by all TCP BTL
* instances that are trying to reach this destination.
* Cache the peer instance on the btl_proc.
*/
tcp_endpoint = OBJ_NEW(mca_btl_tcp_endpoint_t);
if(NULL == tcp_endpoint) {
OPAL_THREAD_UNLOCK(&tcp_proc->proc_lock);
return OPAL_ERR_OUT_OF_RESOURCE;
}
tcp_endpoint->endpoint_btl = tcp_btl;
rc = mca_btl_tcp_proc_insert(tcp_proc, tcp_endpoint);
if(rc != OPAL_SUCCESS) {
OPAL_THREAD_UNLOCK(&tcp_proc->proc_lock);
OBJ_RELEASE(tcp_endpoint);
continue;
}
OPAL_THREAD_LOCK(&tcp_btl->tcp_endpoints_mutex);
opal_list_append(&tcp_btl->tcp_endpoints, (opal_list_item_t*)tcp_endpoint);
OPAL_THREAD_UNLOCK(&tcp_btl->tcp_endpoints_mutex);
}
OPAL_THREAD_UNLOCK(&tcp_proc->proc_lock);
if (NULL != reachable) {
opal_bitmap_set_bit(reachable, i);
}
peers[i] = tcp_endpoint;
}
return OPAL_SUCCESS;
}
void mca_oob_ud_peer_release (mca_oob_ud_peer_t *peer)
{
OBJ_RELEASE(peer);
}
int orte_grpcomm_base_allgather_list(opal_list_t *names, opal_buffer_t *sbuf, opal_buffer_t *rbuf)
{
opal_list_item_t *item;
orte_namelist_t *peer, *root;
orte_std_cntr_t num_peers;
int rc;
OPAL_OUTPUT_VERBOSE((1, orte_grpcomm_base_output,
"%s grpcomm: entering allgather_list",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
/* the first entry on the list is the "root" that collects
* all the data - everyone else just sends and gets back
* the results
*/
root = (orte_namelist_t*)opal_list_get_first(names);
/*** NON-ROOT ***/
if (OPAL_EQUAL != opal_dss.compare(&root->name, ORTE_PROC_MY_NAME, ORTE_NAME)) {
/* everyone but root sends data */
OPAL_OUTPUT_VERBOSE((2, orte_grpcomm_base_output,
"%s allgather_list: sending my data to %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&root->name)));
if (0 > orte_rml.send_buffer(&root->name, sbuf, ORTE_RML_TAG_ALLGATHER_LIST, 0)) {
ORTE_ERROR_LOG(ORTE_ERR_COMM_FAILURE);
return ORTE_ERR_COMM_FAILURE;
}
OPAL_OUTPUT_VERBOSE((2, orte_grpcomm_base_output,
"%s allgather_list: buffer sent",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
/* setup the buffer that will recv the results */
allgather_buf = OBJ_NEW(opal_buffer_t);
/* now receive the final result from rank=0. Be sure to do this in
* a manner that allows us to return without being in a recv!
*/
allgather_num_recvd = 0;
allgather_failed = false;
rc = orte_rml.recv_buffer_nb(ORTE_NAME_WILDCARD, ORTE_RML_TAG_ALLGATHER_LIST,
ORTE_RML_NON_PERSISTENT, allgather_client_recv, NULL);
if (rc != ORTE_SUCCESS && rc != ORTE_ERR_NOT_IMPLEMENTED) {
ORTE_ERROR_LOG(rc);
return rc;
}
ORTE_PROGRESSED_WAIT(allgather_failed, allgather_num_recvd, 1);
/* if the allgather failed, return an error */
if (allgather_failed) {
ORTE_ERROR_LOG(ORTE_ERR_COMM_FAILURE);
OBJ_RELEASE(allgather_buf);
return ORTE_ERR_COMM_FAILURE;
}
/* copy payload to the caller's buffer */
if (ORTE_SUCCESS != (rc = opal_dss.copy_payload(rbuf, allgather_buf))) {
ORTE_ERROR_LOG(rc);
OBJ_RELEASE(allgather_buf);
return rc;
}
OBJ_RELEASE(allgather_buf);
OPAL_OUTPUT_VERBOSE((2, orte_grpcomm_base_output,
"%s allgather_list: buffer received",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
return ORTE_SUCCESS;
}
/*** ROOT ***/
/* count how many peers are participating, including myself */
num_peers = (orte_std_cntr_t)opal_list_get_size(names);
/* seed the outgoing buffer with the num_procs so it can be unpacked */
if (ORTE_SUCCESS != (rc = opal_dss.pack(rbuf, &num_peers, 1, ORTE_STD_CNTR))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* put my own information into the outgoing buffer */
if (ORTE_SUCCESS != (rc = opal_dss.copy_payload(rbuf, sbuf))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* setup the recv conditions */
allgather_failed = false;
allgather_num_recvd = 0;
/* setup the buffer that will recv the results */
allgather_buf = OBJ_NEW(opal_buffer_t);
OPAL_OUTPUT_VERBOSE((2, orte_grpcomm_base_output,
"%s allgather_list: waiting to recv %ld inputs",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
(long)num_peers-1));
/* post the non-blocking recv */
rc = orte_rml.recv_buffer_nb(ORTE_NAME_WILDCARD, ORTE_RML_TAG_ALLGATHER_LIST,
ORTE_RML_NON_PERSISTENT, allgather_server_recv, NULL);
if (rc != ORTE_SUCCESS && rc != ORTE_ERR_NOT_IMPLEMENTED) {
ORTE_ERROR_LOG(rc);
return rc;
}
ORTE_PROGRESSED_WAIT(allgather_failed, allgather_num_recvd, num_peers-1);
/* cancel the lingering recv */
if (ORTE_SUCCESS != (rc = orte_rml.recv_cancel(ORTE_NAME_WILDCARD, ORTE_RML_TAG_ALLGATHER_LIST))) {
ORTE_ERROR_LOG(rc);
OBJ_RELEASE(allgather_buf);
return rc;
}
OPAL_OUTPUT_VERBOSE((2, orte_grpcomm_base_output,
"%s allgather_list: received all data",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
/* copy the received info to the caller's buffer */
if (ORTE_SUCCESS != (rc = opal_dss.copy_payload(rbuf, allgather_buf))) {
ORTE_ERROR_LOG(rc);
OBJ_RELEASE(allgather_buf);
return rc;
}
OBJ_RELEASE(allgather_buf);
/* broadcast the results */
allgather_num_sent = 0;
for (item = opal_list_get_first(names);
item != opal_list_get_end(names);
item = opal_list_get_next(item)) {
peer = (orte_namelist_t*)item;
/* skip myself */
if (OPAL_EQUAL == opal_dss.compare(&root->name, &peer->name, ORTE_NAME)) {
continue;
}
/* transmit the buffer to this process */
if (0 > orte_rml.send_buffer_nb(&peer->name, rbuf, ORTE_RML_TAG_ALLGATHER_LIST,
0, allgather_send_cb, 0)) {
ORTE_ERROR_LOG(ORTE_ERR_COMM_FAILURE);
return ORTE_ERR_COMM_FAILURE;
}
}
ORTE_PROGRESSED_WAIT(false, allgather_num_sent, num_peers-1);
OPAL_OUTPUT_VERBOSE((1, orte_grpcomm_base_output,
"%s grpcomm: allgather_list completed",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
return ORTE_SUCCESS;
}
/* dispatch for callback on message completion */
static int
component_fragment_cb(ompi_request_t *request)
{
int ret;
ompi_osc_pt2pt_buffer_t *buffer;
ompi_osc_pt2pt_module_t *module;
if (request->req_status._cancelled) {
opal_output_verbose(5, ompi_osc_base_framework.framework_output,
"pt2pt request was canceled");
return OMPI_ERR_NOT_AVAILABLE;
}
buffer = (ompi_osc_pt2pt_buffer_t*) request->req_complete_cb_data;
module = (ompi_osc_pt2pt_module_t*) buffer->data;
assert(request->req_status._ucount >= (int) sizeof(ompi_osc_pt2pt_base_header_t));
/* handle message */
switch (((ompi_osc_pt2pt_base_header_t*) buffer->payload)->hdr_type) {
case OMPI_OSC_PT2PT_HDR_PUT:
{
/* get our header and payload */
ompi_osc_pt2pt_send_header_t *header =
(ompi_osc_pt2pt_send_header_t*) buffer->payload;
void *payload = (void*) (header + 1);
#if !defined(WORDS_BIGENDIAN) && OPAL_ENABLE_HETEROGENEOUS_SUPPORT
if (header->hdr_base.hdr_flags & OMPI_OSC_PT2PT_HDR_FLAG_NBO) {
OMPI_OSC_PT2PT_SEND_HDR_NTOH(*header);
}
#endif
if (!ompi_win_exposure_epoch(module->p2p_win)) {
if (OMPI_WIN_FENCE & ompi_win_get_mode(module->p2p_win)) {
ompi_win_set_mode(module->p2p_win,
OMPI_WIN_FENCE |
OMPI_WIN_ACCESS_EPOCH |
OMPI_WIN_EXPOSE_EPOCH);
}
}
ret = ompi_osc_pt2pt_sendreq_recv_put(module, header, payload);
}
break;
case OMPI_OSC_PT2PT_HDR_ACC:
{
/* get our header and payload */
ompi_osc_pt2pt_send_header_t *header =
(ompi_osc_pt2pt_send_header_t*) buffer->payload;
void *payload = (void*) (header + 1);
#if !defined(WORDS_BIGENDIAN) && OPAL_ENABLE_HETEROGENEOUS_SUPPORT
if (header->hdr_base.hdr_flags & OMPI_OSC_PT2PT_HDR_FLAG_NBO) {
OMPI_OSC_PT2PT_SEND_HDR_NTOH(*header);
}
#endif
if (!ompi_win_exposure_epoch(module->p2p_win)) {
if (OMPI_WIN_FENCE & ompi_win_get_mode(module->p2p_win)) {
ompi_win_set_mode(module->p2p_win,
OMPI_WIN_FENCE |
OMPI_WIN_ACCESS_EPOCH |
OMPI_WIN_EXPOSE_EPOCH);
}
}
/* receive into temporary buffer */
ret = ompi_osc_pt2pt_sendreq_recv_accum(module, header, payload);
}
break;
case OMPI_OSC_PT2PT_HDR_GET:
{
/* get our header and payload */
ompi_osc_pt2pt_send_header_t *header =
(ompi_osc_pt2pt_send_header_t*) buffer->payload;
void *payload = (void*) (header + 1);
ompi_datatype_t *datatype;
ompi_osc_pt2pt_replyreq_t *replyreq;
ompi_proc_t *proc;
#if !defined(WORDS_BIGENDIAN) && OPAL_ENABLE_HETEROGENEOUS_SUPPORT
if (header->hdr_base.hdr_flags & OMPI_OSC_PT2PT_HDR_FLAG_NBO) {
OMPI_OSC_PT2PT_SEND_HDR_NTOH(*header);
}
#endif
if (!ompi_win_exposure_epoch(module->p2p_win)) {
if (OMPI_WIN_FENCE & ompi_win_get_mode(module->p2p_win)) {
ompi_win_set_mode(module->p2p_win,
OMPI_WIN_FENCE |
OMPI_WIN_ACCESS_EPOCH |
OMPI_WIN_EXPOSE_EPOCH);
}
}
/* create or get a pointer to our datatype */
proc = ompi_comm_peer_lookup( module->p2p_comm, header->hdr_origin );
datatype = ompi_osc_base_datatype_create(proc, &payload);
if (NULL == datatype) {
opal_output(ompi_osc_base_framework.framework_output,
"Error recreating datatype. Aborting.");
ompi_mpi_abort(module->p2p_comm, 1, false);
}
/* create replyreq sendreq */
ret = ompi_osc_pt2pt_replyreq_alloc_init(module,
header->hdr_origin,
header->hdr_origin_sendreq,
header->hdr_target_disp,
header->hdr_target_count,
datatype,
&replyreq);
/* send replyreq */
ompi_osc_pt2pt_replyreq_send(module, replyreq);
/* sendreq does the right retain, so we can release safely */
OBJ_RELEASE(datatype);
}
break;
case OMPI_OSC_PT2PT_HDR_REPLY:
{
ompi_osc_pt2pt_reply_header_t *header =
(ompi_osc_pt2pt_reply_header_t*) buffer->payload;
void *payload = (void*) (header + 1);
ompi_osc_pt2pt_sendreq_t *sendreq;
#if !defined(WORDS_BIGENDIAN) && OPAL_ENABLE_HETEROGENEOUS_SUPPORT
if (header->hdr_base.hdr_flags & OMPI_OSC_PT2PT_HDR_FLAG_NBO) {
OMPI_OSC_PT2PT_REPLY_HDR_NTOH(*header);
}
#endif
/* get original sendreq pointer */
sendreq = (ompi_osc_pt2pt_sendreq_t*) header->hdr_origin_sendreq.pval;
module = sendreq->req_module;
/* receive data */
ompi_osc_pt2pt_replyreq_recv(module, sendreq, header, payload);
}
break;
case OMPI_OSC_PT2PT_HDR_POST:
{
int32_t count;
OPAL_THREAD_LOCK(&module->p2p_lock);
count = (module->p2p_num_post_msgs -= 1);
OPAL_THREAD_UNLOCK(&module->p2p_lock);
if (count == 0) opal_condition_broadcast(&module->p2p_cond);
}
break;
case OMPI_OSC_PT2PT_HDR_COMPLETE:
{
ompi_osc_pt2pt_control_header_t *header =
(ompi_osc_pt2pt_control_header_t*) buffer->payload;
int32_t count;
#if !defined(WORDS_BIGENDIAN) && OPAL_ENABLE_HETEROGENEOUS_SUPPORT
if (header->hdr_base.hdr_flags & OMPI_OSC_PT2PT_HDR_FLAG_NBO) {
OMPI_OSC_PT2PT_CONTROL_HDR_NTOH(*header);
}
#endif
/* we've heard from one more place, and have value reqs to
process */
OPAL_THREAD_LOCK(&module->p2p_lock);
count = (module->p2p_num_complete_msgs -= 1);
count += (module->p2p_num_pending_in += header->hdr_value[0]);
OPAL_THREAD_UNLOCK(&module->p2p_lock);
if (count == 0) opal_condition_broadcast(&module->p2p_cond);
}
break;
case OMPI_OSC_PT2PT_HDR_LOCK_REQ:
{
ompi_osc_pt2pt_control_header_t *header =
(ompi_osc_pt2pt_control_header_t*) buffer->payload;
int32_t count;
#if !defined(WORDS_BIGENDIAN) && OPAL_ENABLE_HETEROGENEOUS_SUPPORT
if (header->hdr_base.hdr_flags & OMPI_OSC_PT2PT_HDR_FLAG_NBO) {
OMPI_OSC_PT2PT_CONTROL_HDR_NTOH(*header);
}
#endif
if (header->hdr_value[1] > 0) {
ompi_osc_pt2pt_passive_lock(module, header->hdr_value[0],
header->hdr_value[1]);
} else {
OPAL_THREAD_LOCK(&module->p2p_lock);
count = (module->p2p_lock_received_ack += 1);
OPAL_THREAD_UNLOCK(&module->p2p_lock);
if (count != 0) opal_condition_broadcast(&module->p2p_cond);
}
}
break;
case OMPI_OSC_PT2PT_HDR_UNLOCK_REQ:
{
ompi_osc_pt2pt_control_header_t *header =
(ompi_osc_pt2pt_control_header_t*) buffer->payload;
#if !defined(WORDS_BIGENDIAN) && OPAL_ENABLE_HETEROGENEOUS_SUPPORT
if (header->hdr_base.hdr_flags & OMPI_OSC_PT2PT_HDR_FLAG_NBO) {
OMPI_OSC_PT2PT_CONTROL_HDR_NTOH(*header);
}
#endif
ompi_osc_pt2pt_passive_unlock(module, header->hdr_value[0],
header->hdr_value[1]);
}
break;
case OMPI_OSC_PT2PT_HDR_UNLOCK_REPLY:
{
int32_t count;
OPAL_THREAD_LOCK(&module->p2p_lock);
count = (module->p2p_num_pending_out -= 1);
OPAL_THREAD_UNLOCK(&module->p2p_lock);
if (count == 0) opal_condition_broadcast(&module->p2p_cond);
}
break;
default:
opal_output_verbose(5, ompi_osc_base_framework.framework_output,
"received one-sided packet for with unknown type");
}
ompi_request_free(&request);
ret = ompi_osc_pt2pt_component_irecv(buffer->payload,
mca_osc_pt2pt_component.p2p_c_eager_size,
MPI_BYTE,
MPI_ANY_SOURCE,
CONTROL_MSG_TAG,
module->p2p_comm,
&buffer->request,
component_fragment_cb,
buffer);
return ret;
}
void GameManager::FreeObject(CBaseObject** ppObj)
{
if (NULL == ppObj)
{
return;
}
if (NULL == *ppObj)
{
return;
}
CBaseObject *pObj = *ppObj;
const CGUID& guid = pObj->GetExID();
switch(pObj->GetType())
{
case TYPE_GOODS: // 物品
{
m_GoodsList.erase(guid);
/*((CGoods*)pObj)->~CGoods();
m_MultiMemPool.Multi_Free((void*)pObj, sizeof(CGoods));*/
OBJ_RELEASE(CGoods, pObj);
}
break;
case TYPE_COLLECTION:
{
m_CollectionList.erase(guid);
/*((CCollection*)pObj)->~CCollection();
m_MultiMemPool.Multi_Free((void*)pObj, sizeof(CCollection));*/
OBJ_RELEASE(CCollection, pObj);
}
break;
case TYPE_MONSTER: // 怪物
{
m_MonsterList.erase(guid);
/*((CMonster*)pObj)->~CMonster();
m_MultiMemPool.Multi_Free((void*)pObj, sizeof(CMonster));*/
OBJ_RELEASE(CMonster, pObj);
}
break;
case TYPE_PLAYER: // 玩家
{
m_PlayerList.erase(guid);
// ((CPlayer*)pObj)->~CPlayer();
// m_MultiMemPool.Multi_Free((void*)pObj, sizeof(CPlayer));
OBJ_RELEASE(CPlayer, pObj);
}
break;
case TYPE_REGION: // 地图
{
m_RegionList.erase(guid);
// ((CRegion*)pObj)->~CRegion();
// m_MultiMemPool.Multi_Free((void*)pObj, sizeof(CRegion));
OBJ_RELEASE(CServerRegion, pObj);
}
break;
case TYPE_BUILD:
{
m_BuildList.erase(guid);
// ((CBuild*)pObj)->~CBuild();
// m_MultiMemPool.Multi_Free(pObj, sizeof(CBuild));
OBJ_RELEASE(CBuild, pObj);
}
break;
case TYPE_CITYGATE:
{
m_GateList.erase(guid);
// ((CGate*)pObj)->~CGate();
// m_MultiMemPool.Multi_Free(pObj, sizeof(CGate));
OBJ_RELEASE(CGate, pObj);
}
break;
default:
assert(false);
}
*ppObj = NULL;
}
static int test_upper( unsigned int length )
{
double *mat1, *mat2, *inbuf;
opal_datatype_t *pdt;
opal_convertor_t * pConv;
char *ptr;
int rc;
unsigned int i, j, iov_count, split_chunk, total_length;
size_t max_data;
struct iovec a;
TIMER_DATA_TYPE start, end;
long total_time;
/*printf( "test upper matrix\n" );*/
pdt = upper_matrix( length );
/*opal_datatype_dump( pdt );*/
mat1 = malloc( length * length * sizeof(double) );
init_random_upper_matrix( length, mat1 );
mat2 = calloc( length * length, sizeof(double) );
total_length = length * (length + 1) * ( sizeof(double) / 2);
inbuf = (double*)malloc( total_length );
ptr = (char*)inbuf;
/* copy upper matrix in the array simulating the input buffer */
for( i = 0; i < length; i++ ) {
uint32_t pos = i * length + i;
for( j = i; j < length; j++, pos++ ) {
*inbuf = mat1[pos];
inbuf++;
}
}
inbuf = (double*)ptr;
pConv = opal_convertor_create( remote_arch, 0 );
if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( pConv, pdt, 1, mat2 ) ) {
printf( "Cannot attach the datatype to a convertor\n" );
return OPAL_ERROR;
}
GET_TIME( start );
split_chunk = (length + 1) * sizeof(double);
/* split_chunk = (total_length + 1) * sizeof(double); */
for( i = total_length; i > 0; ) {
if( i <= split_chunk ) { /* equal test just to be able to set a breakpoint */
split_chunk = i;
}
a.iov_base = ptr;
a.iov_len = split_chunk;
iov_count = 1;
max_data = split_chunk;
opal_convertor_unpack( pConv, &a, &iov_count, &max_data );
ptr += max_data;
i -= max_data;
if( mat2[0] != inbuf[0] ) assert(0);
}
GET_TIME( end );
total_time = ELAPSED_TIME( start, end );
printf( "complete unpacking in %ld microsec\n", total_time );
free( inbuf );
rc = check_diag_matrix( length, mat1, mat2 );
free( mat1 );
free( mat2 );
/* test the automatic destruction pf the data */
opal_datatype_destroy( &pdt );
assert( pdt == NULL );
OBJ_RELEASE( pConv );
return rc;
}
static void mca_coll_hcoll_module_destruct(mca_coll_hcoll_module_t *hcoll_module)
{
opal_list_item_t *item, *item_next;
opal_list_t *am;
mca_coll_hcoll_module_t *module;
ompi_communicator_t *comm;
int context_destroyed;
am = &mca_coll_hcoll_component.active_modules;
if (hcoll_module->comm == &ompi_mpi_comm_world.comm){
/* If we get here then we are detroying MPI_COMM_WORLD now. So,
* it is safe to destory all the other communicators and corresponding
* hcoll contexts that could still be on the "active_modules" list.
*/
item = opal_list_get_first(am);
while (item != opal_list_get_end(am)){
item_next = opal_list_get_next(item);
module = ((mca_coll_hcoll_module_list_item_wrapper_t *)item)->module;
comm = module->comm;
context_destroyed = 0;
while(!context_destroyed){
hcoll_destroy_context(module->hcoll_context,
(rte_grp_handle_t)comm,
&context_destroyed);
}
module->hcoll_context = NULL;
OBJ_RELEASE(comm);
opal_list_remove_item(am,item);
OBJ_RELEASE(item);
item = item_next;
}
/* Now destory the comm_world hcoll context as well */
context_destroyed = 0;
while(!context_destroyed){
hcoll_destroy_context(hcoll_module->hcoll_context,
(rte_grp_handle_t)hcoll_module->comm,
&context_destroyed);
}
}
OBJ_RELEASE(hcoll_module->previous_barrier_module);
OBJ_RELEASE(hcoll_module->previous_bcast_module);
OBJ_RELEASE(hcoll_module->previous_reduce_module);
OBJ_RELEASE(hcoll_module->previous_allreduce_module);
OBJ_RELEASE(hcoll_module->previous_allgather_module);
OBJ_RELEASE(hcoll_module->previous_allgatherv_module);
OBJ_RELEASE(hcoll_module->previous_gather_module);
OBJ_RELEASE(hcoll_module->previous_gatherv_module);
OBJ_RELEASE(hcoll_module->previous_alltoall_module);
OBJ_RELEASE(hcoll_module->previous_alltoallv_module);
OBJ_RELEASE(hcoll_module->previous_alltoallw_module);
OBJ_RELEASE(hcoll_module->previous_reduce_scatter_module);
OBJ_RELEASE(hcoll_module->previous_ibarrier_module);
OBJ_RELEASE(hcoll_module->previous_ibcast_module);
OBJ_RELEASE(hcoll_module->previous_iallreduce_module);
OBJ_RELEASE(hcoll_module->previous_iallgather_module);
mca_coll_hcoll_module_clear(hcoll_module);
}
int mca_base_components_filter (const char *framework_name, opal_list_t *components, int output_id,
const char *filter_names, uint32_t filter_flags)
{
mca_base_component_list_item_t *cli, *next;
char **requested_component_names = NULL;
bool include_mode, can_use;
int ret;
assert (NULL != components);
if (0 == filter_flags && NULL == filter_names) {
return OPAL_SUCCESS;
}
ret = mca_base_component_parse_requested (filter_names, &include_mode,
&requested_component_names);
if (OPAL_SUCCESS != ret) {
return ret;
}
OPAL_LIST_FOREACH_SAFE(cli, next, components, mca_base_component_list_item_t) {
const mca_base_component_t *component = cli->cli_component;
mca_base_open_only_dummy_component_t *dummy =
(mca_base_open_only_dummy_component_t *) cli->cli_component;
can_use = use_component (include_mode, (const char **) requested_component_names,
cli->cli_component->mca_component_name);
if (!can_use || (filter_flags & dummy->data.param_field) != filter_flags) {
if (can_use && (filter_flags & MCA_BASE_METADATA_PARAM_CHECKPOINT) &&
!(MCA_BASE_METADATA_PARAM_CHECKPOINT & dummy->data.param_field)) {
opal_output_verbose(10, output_id,
"mca: base: components_filter: "
"(%s) Component %s is *NOT* Checkpointable - Disabled",
component->reserved,
component->mca_component_name);
}
opal_list_remove_item (components, &cli->super);
mca_base_component_unload (component, output_id);
OBJ_RELEASE(cli);
} else if (filter_flags & MCA_BASE_METADATA_PARAM_CHECKPOINT) {
opal_output_verbose(10, output_id,
"mca: base: components_filter: "
"(%s) Component %s is Checkpointable",
component->reserved,
component->mca_component_name);
}
}
if (include_mode) {
ret = component_find_check (framework_name, requested_component_names, components);
} else {
ret = OPAL_SUCCESS;
}
if (NULL != requested_component_names) {
opal_argv_free (requested_component_names);
}
return ret;
}
static bool test1(void) /* verify dss_copy_payload */
{
opal_buffer_t *bufA, *bufB;
int rc;
int32_t i;
int16_t src[NUM_ELEMS];
int16_t dst[NUM_ELEMS];
int32_t src32[NUM_ELEMS];
int32_t dst32[NUM_ELEMS];
/* init src arrays */
for (i=0; i < NUM_ELEMS; i++) {
src[i] = i;
src32[i] = 132 * i;
}
/* init A */
bufA = OBJ_NEW(opal_buffer_t);
if (NULL == bufA) {
fprintf(test_out, "orte_buffer failed init in OBJ_NEW\n");
return false;
}
opal_dss.set_buffer_type(bufA, OPAL_DSS_BUFFER_NON_DESC);
/* pack something in A */
for (i=0;i<NUM_ITERS;i++) {
rc = opal_dss.pack(bufA, src, NUM_ELEMS, OPAL_INT16);
if (OPAL_SUCCESS != rc) {
fprintf(test_out, "opal_dss.pack failed with return code %d\n", rc);
return(false);
}
}
/* setup bufB */
bufB = OBJ_NEW(opal_buffer_t);
if (NULL == bufB) {
fprintf(test_out, "orte_buffer failed init in OBJ_NEW\n");
return false;
}
opal_dss.set_buffer_type(bufB, OPAL_DSS_BUFFER_NON_DESC);
/* pack something in B */
for (i=0;i<NUM_ITERS;i++) {
rc = opal_dss.pack(bufB, src32, NUM_ELEMS, OPAL_INT32);
if (OPAL_SUCCESS != rc) {
fprintf(test_out, "opal_dss.pack failed with return code %d\n", rc);
return(false);
}
}
/* copy payload to bufB */
if (OPAL_SUCCESS != (rc = opal_dss.copy_payload(bufB, bufA))) {
fprintf(test_out, "opal_dss.copy_payload failed with return code %d\n", rc);
return(false);
}
/* pack some more stuff in B */
for (i=0;i<NUM_ITERS;i++) {
rc = opal_dss.pack(bufB, src32, NUM_ELEMS, OPAL_INT32);
if (OPAL_SUCCESS != rc) {
fprintf(test_out, "opal_dss.pack failed with return code %d\n", rc);
return(false);
}
}
/* validate the results */
for (i=0; i<NUM_ITERS; i++) {
int j;
int32_t count;
for(j=0; j<NUM_ELEMS; j++)
dst32[j] = -1;
count = NUM_ELEMS;
rc = opal_dss.unpack(bufB, dst32, &count, OPAL_INT32);
if (OPAL_SUCCESS != rc || count != NUM_ELEMS) {
fprintf(test_out, "opal_dss.unpack of dest buffer failed with return code %d\n", rc);
return(false);
}
for(j=0; j<NUM_ELEMS; j++) {
if(src32[j] != dst32[j]) {
fprintf(test_out, "test2: invalid results from unpack of dest buffer\n");
return(false);
}
}
}
for (i=0; i<NUM_ITERS; i++) {
int j;
int32_t count;
for(j=0; j<NUM_ELEMS; j++)
dst[j] = -1;
count = NUM_ELEMS;
rc = opal_dss.unpack(bufB, dst, &count, OPAL_INT16);
if (OPAL_SUCCESS != rc || count != NUM_ELEMS) {
fprintf(test_out, "opal_dss.unpack of dest buffer failed with return code %d\n", rc);
return(false);
}
for(j=0; j<NUM_ELEMS; j++) {
if(src[j] != dst[j]) {
fprintf(test_out, "test2: invalid results from unpack of dest buffer\n");
return(false);
}
}
}
for (i=0; i<NUM_ITERS; i++) {
int j;
int32_t count;
for(j=0; j<NUM_ELEMS; j++)
dst32[j] = -1;
count = NUM_ELEMS;
rc = opal_dss.unpack(bufB, dst32, &count, OPAL_INT32);
if (OPAL_SUCCESS != rc || count != NUM_ELEMS) {
fprintf(test_out, "opal_dss.unpack of dest buffer failed with return code %d\n", rc);
return(false);
}
for(j=0; j<NUM_ELEMS; j++) {
if(src32[j] != dst32[j]) {
fprintf(test_out, "test2: invalid results from unpack of dest buffer\n");
return(false);
}
}
}
/* check that A is still okay */
for (i=0; i<NUM_ITERS; i++) {
int j;
int32_t count;
for(j=0; j<NUM_ELEMS; j++)
dst[j] = -1;
count = NUM_ELEMS;
rc = opal_dss.unpack(bufA, dst, &count, OPAL_INT16);
if (OPAL_SUCCESS != rc || count != NUM_ELEMS) {
fprintf(test_out, "opal_dss.unpack of src buffer failed with return code %d\n", rc);
return(false);
}
for(j=0; j<NUM_ELEMS; j++) {
if(src[j] != dst[j]) {
fprintf(test_out, "test2: invalid results from unpack of src buffer\n");
return(false);
}
}
}
OBJ_RELEASE(bufA);
OBJ_RELEASE(bufB);
if (NULL != bufA || NULL != bufB) {
fprintf(test_out, "OBJ_RELEASE did not NULL the buffer pointer\n");
return false;
}
return (true);
}
int mca_topo_base_find_available(bool enable_progress_threads,
bool enable_mpi_threads)
{
bool found = false;
mca_base_component_priority_list_item_t *entry;
opal_list_item_t *p;
/* Initialize the list */
OBJ_CONSTRUCT(&mca_topo_base_components_available, opal_list_t);
mca_topo_base_components_available_valid = true;
/* The list of components which we should check is already present
in mca_topo_base_components_opened, which was established in
mca_topo_base_open */
for (found = false,
p = opal_list_remove_first (&mca_topo_base_components_opened);
NULL != p;
p = opal_list_remove_first (&mca_topo_base_components_opened)) {
entry = OBJ_NEW(mca_base_component_priority_list_item_t);
entry->super.cli_component =
((mca_base_component_list_item_t *)p)->cli_component;
/* Now for this entry, we have to determine the thread level. Call
a subroutine to do the job for us */
if (OMPI_SUCCESS == init_query(entry->super.cli_component, entry,
enable_progress_threads,
enable_mpi_threads)) {
/* Save the results in the list. The priority is not relvant at
this point in time. But we save the thread arguments so that
the initial selection algorithm can negotiate overall thread
level for this process */
entry->cpli_priority = 0;
opal_list_append (&mca_topo_base_components_available,
(opal_list_item_t *) entry);
found = true;
} else {
/* The component does not want to run, so close it. Its close()
has already been invoked. Close it out of the DSO repository
(if it is there in the repository) */
mca_base_component_repository_release(entry->super.cli_component);
OBJ_RELEASE(entry);
}
/* Free entry from the "opened" list */
OBJ_RELEASE(p);
}
/* The opened list is no longer necessary, so we can free it */
OBJ_DESTRUCT (&mca_topo_base_components_opened);
mca_topo_base_components_opened_valid = false;
/* There should atleast be one topo component which was available */
if (false == found) {
/* Need to free all items in the list */
OBJ_DESTRUCT(&mca_topo_base_components_available);
mca_topo_base_components_available_valid = false;
opal_output_verbose (10, mca_topo_base_output,
"topo:find_available: no topo components available!");
return OMPI_ERROR;
}
/* All done */
return OMPI_SUCCESS;
}
int orte_list_local_hnps(opal_list_t *hnps, bool connect)
{
int ret;
DIR *cur_dirp = NULL;
struct dirent * dir_entry;
char *contact_filename = NULL;
orte_hnp_contact_t *hnp;
char *headdir;
/*
* Check to make sure we have access to the top-level directory
*/
headdir = opal_os_path(false, orte_process_info.tmpdir_base, orte_process_info.top_session_dir, NULL);
if( ORTE_SUCCESS != (ret = opal_os_dirpath_access(headdir, 0) )) {
/* it is okay not to find this as there may not be any
* HNP's present, and we don't write our own session dir
*/
if (ORTE_ERR_NOT_FOUND != ret) {
ORTE_ERROR_LOG(ret);
}
goto cleanup;
}
/*
* Open up the base directory so we can get a listing
*/
if( NULL == (cur_dirp = opendir(headdir)) ) {
goto cleanup;
}
/*
* For each directory
*/
while( NULL != (dir_entry = readdir(cur_dirp)) ) {
/*
* Skip the obvious
*/
if( 0 == strncmp(dir_entry->d_name, ".", strlen(".")) ||
0 == strncmp(dir_entry->d_name, "..", strlen("..")) ) {
continue;
}
/*
* See if a contact file exists in this directory and read it
*/
contact_filename = opal_os_path( false, headdir,
dir_entry->d_name, "contact.txt", NULL );
hnp = OBJ_NEW(orte_hnp_contact_t);
if (ORTE_SUCCESS == (ret = orte_read_hnp_contact_file(contact_filename, hnp, connect))) {
opal_list_append(hnps, &(hnp->super));
} else {
OBJ_RELEASE(hnp);
}
free(contact_filename);
}
cleanup:
if( NULL != cur_dirp )
closedir(cur_dirp);
free(headdir);
return (opal_list_is_empty(hnps) ? ORTE_ERR_NOT_FOUND : ORTE_SUCCESS);
}
static void process_opens(int fd, short args, void *cbdata)
{
orte_dfs_request_t *dfs = (orte_dfs_request_t*)cbdata;
int rc;
opal_buffer_t *buffer;
char *scheme, *host=NULL, *filename=NULL;
orte_process_name_t daemon;
opal_list_t lt;
opal_namelist_t *nm;
opal_output_verbose(1, orte_dfs_base_framework.framework_output,
"%s PROCESSING OPEN", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
/* get the scheme to determine if we can process locally or not */
if (NULL == (scheme = opal_uri_get_scheme(dfs->uri))) {
ORTE_ERROR_LOG(ORTE_ERR_BAD_PARAM);
goto complete;
}
opal_output_verbose(1, orte_dfs_base_framework.framework_output,
"%s GOT SCHEME", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
if (0 != strcmp(scheme, "file")) {
/* not yet supported */
orte_show_help("orte_dfs_help.txt", "unsupported-filesystem",
true, dfs->uri);
free(scheme);
goto complete;
}
free(scheme);
/* dissect the uri to extract host and filename/path */
if (NULL == (filename = opal_filename_from_uri(dfs->uri, &host))) {
goto complete;
}
opal_output_verbose(1, orte_dfs_base_framework.framework_output,
"%s GOT FILENAME %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), filename);
if (NULL == host) {
host = strdup(orte_process_info.nodename);
}
/* ident the daemon on that host */
daemon.jobid = ORTE_PROC_MY_DAEMON->jobid;
OBJ_CONSTRUCT(<, opal_list_t);
if (ORTE_SUCCESS != (rc = opal_pmix.resolve_peers(host, daemon.jobid, <))) {
ORTE_ERROR_LOG(rc);
OBJ_DESTRUCT(<);
goto complete;
}
nm = (opal_namelist_t*)opal_list_get_first(<);
daemon.vpid = nm->name.vpid;
OPAL_LIST_DESTRUCT(<);
opal_output_verbose(1, orte_dfs_base_framework.framework_output,
"%s file %s on host %s daemon %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
filename, host, ORTE_NAME_PRINT(&daemon));
/* add this request to our local list so we can
* match it with the returned response when it comes
*/
dfs->id = req_id++;
opal_list_append(&requests, &dfs->super);
/* setup a message for the daemon telling
* them what file we want to access
*/
buffer = OBJ_NEW(opal_buffer_t);
if (OPAL_SUCCESS != (rc = opal_dss.pack(buffer, &dfs->cmd, 1, ORTE_DFS_CMD_T))) {
ORTE_ERROR_LOG(rc);
opal_list_remove_item(&requests, &dfs->super);
goto complete;
}
/* pass the request id */
if (OPAL_SUCCESS != (rc = opal_dss.pack(buffer, &dfs->id, 1, OPAL_UINT64))) {
ORTE_ERROR_LOG(rc);
opal_list_remove_item(&requests, &dfs->super);
goto complete;
}
if (OPAL_SUCCESS != (rc = opal_dss.pack(buffer, &filename, 1, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
opal_list_remove_item(&requests, &dfs->super);
goto complete;
}
opal_output_verbose(1, orte_dfs_base_framework.framework_output,
"%s sending open file request to %s file %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&daemon),
filename);
/* send it */
if (0 > (rc = orte_rml.send_buffer_nb(&daemon, buffer,
ORTE_RML_TAG_DFS_CMD,
orte_rml_send_callback, NULL))) {
ORTE_ERROR_LOG(rc);
OBJ_RELEASE(buffer);
opal_list_remove_item(&requests, &dfs->super);
goto complete;
}
/* don't release it */
free(host);
free(filename);
return;
complete:
/* we get here if an error occurred - execute any
* pending callback so the proc doesn't hang
*/
if (NULL != host) {
free(host);
}
if (NULL != filename) {
free(filename);
}
if (NULL != dfs->open_cbfunc) {
dfs->open_cbfunc(-1, dfs->cbdata);
}
OBJ_RELEASE(dfs);
}
static void stdin_write_handler(int fd, short event, void *cbdata)
{
orte_iof_sink_t *sink = (orte_iof_sink_t*)cbdata;
orte_iof_write_event_t *wev = sink->wev;
opal_list_item_t *item;
orte_iof_write_output_t *output;
int num_written;
OPAL_OUTPUT_VERBOSE((1, orte_iof_base.iof_output,
"%s hnp:stdin:write:handler writing data to %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
wev->fd));
/* lock us up to protect global operations */
OPAL_THREAD_LOCK(&mca_iof_hnp_component.lock);
wev->pending = false;
while (NULL != (item = opal_list_remove_first(&wev->outputs))) {
output = (orte_iof_write_output_t*)item;
/* if an abnormal termination has occurred, just dump
* this data as we are aborting
*/
if (orte_abnormal_term_ordered) {
OBJ_RELEASE(output);
continue;
}
if (0 == output->numbytes) {
/* this indicates we are to close the fd - there is
* nothing to write
*/
OPAL_OUTPUT_VERBOSE((20, orte_iof_base.iof_output,
"%s iof:hnp closing fd %d on write event due to zero bytes output",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), wev->fd));
OBJ_RELEASE(wev);
sink->wev = NULL;
/* just leave - we don't want to restart the
* read event!
*/
goto DEPART;
}
num_written = write(wev->fd, output->data, output->numbytes);
OPAL_OUTPUT_VERBOSE((1, orte_iof_base.iof_output,
"%s hnp:stdin:write:handler wrote %d bytes",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
num_written));
if (num_written < 0) {
if (EAGAIN == errno || EINTR == errno) {
/* push this item back on the front of the list */
opal_list_prepend(&wev->outputs, item);
/* leave the write event running so it will call us again
* when the fd is ready.
*/
wev->pending = true;
opal_event_add(wev->ev, 0);
goto CHECK;
}
/* otherwise, something bad happened so all we can do is declare an
* error and abort
*/
OBJ_RELEASE(output);
OPAL_OUTPUT_VERBOSE((20, orte_iof_base.iof_output,
"%s iof:hnp closing fd %d on write event due to negative bytes written",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), wev->fd));
OBJ_RELEASE(wev);
sink->wev = NULL;
goto DEPART;
} else if (num_written < output->numbytes) {
OPAL_OUTPUT_VERBOSE((1, orte_iof_base.iof_output,
"%s hnp:stdin:write:handler incomplete write %d - adjusting data",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), num_written));
/* incomplete write - adjust data to avoid duplicate output */
memmove(output->data, &output->data[num_written], output->numbytes - num_written);
/* push this item back on the front of the list */
opal_list_prepend(&wev->outputs, item);
/* leave the write event running so it will call us again
* when the fd is ready.
*/
wev->pending = true;
opal_event_add(wev->ev, 0);
goto CHECK;
}
OBJ_RELEASE(output);
}
CHECK:
if (NULL != mca_iof_hnp_component.stdinev &&
!orte_abnormal_term_ordered &&
!mca_iof_hnp_component.stdinev->active) {
OPAL_OUTPUT_VERBOSE((1, orte_iof_base.iof_output,
"read event is off - checking if okay to restart"));
/* if we have turned off the read event, check to
* see if the output list has shrunk enough to
* turn it back on
*
* RHC: Note that when multiple procs want stdin, we
* can get into a fight between a proc turnin stdin
* back "on" and other procs turning it "off". There
* is no clear way to resolve this as different procs
* may take input at different rates.
*/
if (opal_list_get_size(&wev->outputs) < ORTE_IOF_MAX_INPUT_BUFFERS) {
/* restart the read */
OPAL_OUTPUT_VERBOSE((1, orte_iof_base.iof_output,
"restarting read event"));
mca_iof_hnp_component.stdinev->active = true;
opal_event_add(mca_iof_hnp_component.stdinev->ev, 0);
}
}
DEPART:
/* unlock and go */
OPAL_THREAD_UNLOCK(&mca_iof_hnp_component.lock);
}
static void process_sizes(int fd, short args, void *cbdata)
{
orte_dfs_request_t *size_dfs = (orte_dfs_request_t*)cbdata;
orte_dfs_tracker_t *tptr, *trk;
opal_list_item_t *item;
opal_buffer_t *buffer;
int rc;
opal_output_verbose(1, orte_dfs_base_framework.framework_output,
"%s processing get_size on fd %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
size_dfs->local_fd);
/* look in our local records for this fd */
trk = NULL;
for (item = opal_list_get_first(&active_files);
item != opal_list_get_end(&active_files);
item = opal_list_get_next(item)) {
tptr = (orte_dfs_tracker_t*)item;
if (tptr->local_fd == size_dfs->local_fd) {
trk = tptr;
break;
}
}
if (NULL == trk) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
OBJ_RELEASE(size_dfs);
return;
}
/* add this request to our local list so we can
* match it with the returned response when it comes
*/
size_dfs->id = req_id++;
opal_list_append(&requests, &size_dfs->super);
/* setup a message for the daemon telling
* them what file we want to access
*/
buffer = OBJ_NEW(opal_buffer_t);
if (OPAL_SUCCESS != (rc = opal_dss.pack(buffer, &size_dfs->cmd, 1, ORTE_DFS_CMD_T))) {
ORTE_ERROR_LOG(rc);
opal_list_remove_item(&requests, &size_dfs->super);
goto complete;
}
/* pass the request id */
if (OPAL_SUCCESS != (rc = opal_dss.pack(buffer, &size_dfs->id, 1, OPAL_UINT64))) {
ORTE_ERROR_LOG(rc);
opal_list_remove_item(&requests, &size_dfs->super);
goto complete;
}
if (OPAL_SUCCESS != (rc = opal_dss.pack(buffer, &trk->remote_fd, 1, OPAL_INT))) {
ORTE_ERROR_LOG(rc);
opal_list_remove_item(&requests, &size_dfs->super);
goto complete;
}
opal_output_verbose(1, orte_dfs_base_framework.framework_output,
"%s sending get_size request to %s for fd %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&trk->host_daemon),
trk->local_fd);
/* send it */
if (0 > (rc = orte_rml.send_buffer_nb(&trk->host_daemon, buffer,
ORTE_RML_TAG_DFS_CMD,
orte_rml_send_callback, NULL))) {
ORTE_ERROR_LOG(rc);
OBJ_RELEASE(buffer);
opal_list_remove_item(&requests, &size_dfs->super);
if (NULL != size_dfs->size_cbfunc) {
size_dfs->size_cbfunc(-1, size_dfs->cbdata);
}
goto complete;
}
/* leave the request there */
return;
complete:
OBJ_RELEASE(size_dfs);
}
static int
local_copy_with_convertor_2datatypes( opal_datatype_t const * const send_type, int send_count,
opal_datatype_t const * const recv_type, int recv_count,
int chunk )
{
ptrdiff_t send_lb, send_extent, recv_lb, recv_extent;
void *pdst = NULL, *psrc = NULL, *ptemp = NULL;
char *odst, *osrc;
opal_convertor_t *send_convertor = NULL, *recv_convertor = NULL;
struct iovec iov;
uint32_t iov_count;
size_t max_data, length = 0, send_malloced_size, recv_malloced_size;;
int32_t done1 = 0, done2 = 0;
TIMER_DATA_TYPE start, end, unpack_start, unpack_end;
long total_time, unpack_time = 0;
send_malloced_size = compute_memory_size(send_type, send_count);
recv_malloced_size = compute_memory_size(recv_type, recv_count);
opal_datatype_get_extent( send_type, &send_lb, &send_extent );
opal_datatype_get_extent( recv_type, &recv_lb, &recv_extent );
odst = (char*)malloc( recv_malloced_size );
osrc = (char*)malloc( send_malloced_size );
ptemp = malloc( chunk );
/* fill up the receiver with ZEROS */
{
for( size_t i = 0; i < send_malloced_size; i++ )
osrc[i] = i % 128 + 32;
}
memset( odst, 0, recv_malloced_size );
pdst = odst - recv_lb;
psrc = osrc - send_lb;
send_convertor = opal_convertor_create( remote_arch, 0 );
if( OPAL_SUCCESS != opal_convertor_prepare_for_send( send_convertor, send_type, send_count, psrc ) ) {
printf( "Unable to create the send convertor. Is the datatype committed ?\n" );
goto clean_and_return;
}
recv_convertor = opal_convertor_create( remote_arch, 0 );
if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( recv_convertor, recv_type, recv_count, pdst ) ) {
printf( "Unable to create the recv convertor. Is the datatype committed ?\n" );
goto clean_and_return;
}
cache_trash(); /* make sure the cache is useless */
GET_TIME( start );
while( (done1 & done2) != 1 ) {
/* They are supposed to finish in exactly the same time. */
if( done1 | done2 ) {
printf( "WRONG !!! the send is %s but the receive is %s in local_copy_with_convertor_2datatypes\n",
(done1 ? "finish" : "not finish"),
(done2 ? "finish" : "not finish") );
}
max_data = chunk;
iov_count = 1;
iov.iov_base = ptemp;
iov.iov_len = chunk;
if( done1 == 0 ) {
done1 = opal_convertor_pack( send_convertor, &iov, &iov_count, &max_data );
}
if( done2 == 0 ) {
GET_TIME( unpack_start );
done2 = opal_convertor_unpack( recv_convertor, &iov, &iov_count, &max_data );
GET_TIME( unpack_end );
unpack_time += ELAPSED_TIME( unpack_start, unpack_end );
}
length += max_data;
if( outputFlags & RESET_CONVERTORS ) {
size_t pos = 0;
opal_convertor_set_position(send_convertor, &pos);
pos = length;
opal_convertor_set_position(send_convertor, &pos);
assert(pos == length);
pos = 0;
opal_convertor_set_position(recv_convertor, &pos);
pos = length;
opal_convertor_set_position(recv_convertor, &pos);
assert(pos == length);
}
}
GET_TIME( end );
total_time = ELAPSED_TIME( start, end );
printf( "copying different data-types using convertors in %ld microsec\n", total_time );
printf( "\t unpack in %ld microsec [pack in %ld microsec]\n", unpack_time,
total_time - unpack_time );
clean_and_return:
if( send_convertor != NULL ) {
OBJ_RELEASE( send_convertor ); assert( send_convertor == NULL );
}
if( recv_convertor != NULL ) {
OBJ_RELEASE( recv_convertor ); assert( recv_convertor == NULL );
}
if( NULL != odst ) free( odst );
if( NULL != osrc ) free( osrc );
if( NULL != ptemp ) free( ptemp );
return OPAL_SUCCESS;
}
int
main(int argc, char *argv[])
{
int ret, exit_status = ORTE_SUCCESS;
pid_t child_pid = 0;
orte_sstore_base_global_snapshot_info_t *snapshot = NULL;
char *basedir = NULL;
char *tmp_str = NULL;
/***************
* Initialize
***************/
if (ORTE_SUCCESS != (ret = initialize(argc, argv))) {
exit_status = ret;
goto cleanup;
}
snapshot = OBJ_NEW(orte_sstore_base_global_snapshot_info_t);
if( opal_path_is_absolute(orte_restart_globals.snapshot_ref) ) {
basedir = opal_dirname(orte_restart_globals.snapshot_ref);
tmp_str = opal_basename(orte_restart_globals.snapshot_ref);
free(orte_restart_globals.snapshot_ref);
orte_restart_globals.snapshot_ref = strdup(tmp_str);
free(tmp_str);
tmp_str = NULL;
} else if( NULL != strchr(orte_restart_globals.snapshot_ref, '/') ) {
basedir = opal_dirname(orte_restart_globals.snapshot_ref);
tmp_str = opal_basename(orte_restart_globals.snapshot_ref);
free(orte_restart_globals.snapshot_ref);
orte_restart_globals.snapshot_ref = strdup(tmp_str);
free(tmp_str);
tmp_str = NULL;
} else {
basedir = NULL; /* Use MCA parameter */
}
/*
* Note: If the seq # passed is -1, then the largest seq # is selected,
* ow the seq # requested is selected if available
* 'basedir': Snapshot Base location to look in. If NULL then MCA parameter is used
*/
if( ORTE_SUCCESS != (ret = orte_sstore.request_restart_handle(&(snapshot->ss_handle),
basedir,
orte_restart_globals.snapshot_ref,
orte_restart_globals.seq_number,
snapshot))) {
opal_show_help("help-orte-restart.txt", "invalid_filename", true,
orte_restart_globals.snapshot_ref);
exit_status = ret;
goto cleanup;
}
orte_restart_globals.seq_number = snapshot->seq_num;
if(orte_restart_globals.info_only ) {
if (ORTE_SUCCESS != (ret = snapshot_info(snapshot))) {
exit_status = ret;
goto cleanup;
}
exit_status = ORTE_SUCCESS;
goto cleanup;
}
/******************************
* Create the app file to use with mpirun/orterun
******************************/
if( ORTE_SUCCESS != (ret = create_appfile(snapshot) ) ) {
exit_status = ret;
goto cleanup;
}
if( orte_restart_globals.app_only ) {
printf("Created Appfile:\n\t%s\n", orte_restart_globals.appfile);
exit_status = ORTE_SUCCESS;
goto cleanup;
}
/******************************
* Restart in this process [mpirun/orterun]
******************************/
if( orte_restart_globals.verbose ) {
opal_output_verbose(10, orte_restart_globals.output,
"Restarting from file (%s)",
orte_restart_globals.snapshot_ref);
if( orte_restart_globals.forked ) {
opal_output_verbose(10, orte_restart_globals.output,
"\t Forking off a child");
} else {
opal_output_verbose(10, orte_restart_globals.output,
"\t Exec in self");
}
}
if( ORTE_SUCCESS != (ret = spawn_children(snapshot, &child_pid)) ) {
opal_show_help("help-orte-restart.txt", "restart_cmd_failure", true,
orte_restart_globals.snapshot_ref, ret);
exit_status = ret;
goto cleanup;
}
/***************
* Cleanup
***************/
cleanup:
if( NULL != basedir ) {
free(basedir);
basedir = NULL;
}
if( NULL != tmp_str ) {
free(tmp_str);
tmp_str = NULL;
}
if( NULL != snapshot ) {
OBJ_RELEASE(snapshot);
snapshot = NULL;
}
if (OPAL_SUCCESS != (ret = finalize())) {
return ret;
}
return exit_status;
}
static int local_copy_with_convertor( opal_datatype_t const * const pdt, int count, int chunk )
{
ptrdiff_t lb, extent;
void *pdst = NULL, *psrc = NULL, *ptemp = NULL;
char *odst, *osrc;
opal_convertor_t *send_convertor = NULL, *recv_convertor = NULL;
struct iovec iov;
uint32_t iov_count;
size_t max_data, length = 0, malloced_size;
int32_t done1 = 0, done2 = 0, errors = 0;
TIMER_DATA_TYPE start, end, unpack_start, unpack_end;
long total_time, unpack_time = 0;
malloced_size = compute_memory_size(pdt, count);
opal_datatype_get_extent( pdt, &lb, &extent );
odst = (char*)malloc( malloced_size );
osrc = (char*)malloc( malloced_size );
ptemp = malloc( chunk );
{
for( size_t i = 0; i < malloced_size; osrc[i] = i % 128 + 32, i++ );
memcpy(odst, osrc, malloced_size);
}
pdst = odst - lb;
psrc = osrc - lb;
send_convertor = opal_convertor_create( remote_arch, 0 );
if( OPAL_SUCCESS != opal_convertor_prepare_for_send( send_convertor, pdt, count, psrc ) ) {
printf( "Unable to create the send convertor. Is the datatype committed ?\n" );
goto clean_and_return;
}
recv_convertor = opal_convertor_create( remote_arch, 0 );
if( OPAL_SUCCESS != opal_convertor_prepare_for_recv( recv_convertor, pdt, count, pdst ) ) {
printf( "Unable to create the recv convertor. Is the datatype committed ?\n" );
goto clean_and_return;
}
cache_trash(); /* make sure the cache is useless */
GET_TIME( start );
while( (done1 & done2) != 1 ) {
/* They are supposed to finish in exactly the same time. */
if( done1 | done2 ) {
printf( "WRONG !!! the send is %s but the receive is %s in local_copy_with_convertor\n",
(done1 ? "finish" : "not finish"),
(done2 ? "finish" : "not finish") );
}
max_data = chunk;
iov_count = 1;
iov.iov_base = ptemp;
iov.iov_len = chunk;
if( done1 == 0 ) {
done1 = opal_convertor_pack( send_convertor, &iov, &iov_count, &max_data );
}
if( done2 == 0 ) {
GET_TIME( unpack_start );
done2 = opal_convertor_unpack( recv_convertor, &iov, &iov_count, &max_data );
GET_TIME( unpack_end );
unpack_time += ELAPSED_TIME( unpack_start, unpack_end );
}
length += max_data;
if( outputFlags & RESET_CONVERTORS ) {
struct dt_stack_t stack[1+send_convertor->stack_pos];
int i, stack_pos = send_convertor->stack_pos;
size_t pos;
if( 0 == done1 ) {
memcpy(stack, send_convertor->pStack, (1+send_convertor->stack_pos) * sizeof(struct dt_stack_t));
pos = 0;
opal_convertor_set_position(send_convertor, &pos);
pos = length;
opal_convertor_set_position(send_convertor, &pos);
assert(pos == length);
for(i = 0; i <= stack_pos; i++ ) {
if( stack[i].index != send_convertor->pStack[i].index )
{errors = 1; printf("send stack[%d].index differs (orig %d != new %d) (completed %lu/%lu)\n",
i, stack[i].index, send_convertor->pStack[i].index,
length, pdt->size * count);}
if( stack[i].count != send_convertor->pStack[i].count ) {
if( stack[i].type == send_convertor->pStack[i].type ) {
{errors = 1; printf("send stack[%d].count differs (orig %lu != new %lu) (completed %lu/%lu)\n",
i, stack[i].count, send_convertor->pStack[i].count,
length, pdt->size * count);}
} else {
if( (OPAL_DATATYPE_MAX_PREDEFINED <= stack[i].type) || (OPAL_DATATYPE_MAX_PREDEFINED <= send_convertor->pStack[i].type) )
{errors = 1; printf("send stack[%d].type wrong (orig %d != new %d) (completed %lu/%lu)\n",
i, (int)stack[i].type, (int)send_convertor->pStack[i].type,
length, pdt->size * count);}
else if( (stack[i].count * opal_datatype_basicDatatypes[stack[i].type]->size) !=
(send_convertor->pStack[i].count * opal_datatype_basicDatatypes[send_convertor->pStack[i].type]->size) )
{errors = 1; printf("send stack[%d].type*count differs (orig (%d,%lu) != new (%d, %lu)) (completed %lu/%lu)\n",
i, (int)stack[i].type, stack[i].count,
(int)send_convertor->pStack[i].type, send_convertor->pStack[i].count,
length, pdt->size * count);}
}
}
if( stack[i].disp != send_convertor->pStack[i].disp )
{errors = 1; printf("send stack[%d].disp differs (orig %p != new %p) (completed %lu/%lu)\n",
i, (void*)stack[i].disp, (void*)send_convertor->pStack[i].disp,
length, pdt->size * count);}
if(0 != errors) {assert(0); exit(-1);}
}
}
if( 0 == done2 ) {
memcpy(stack, recv_convertor->pStack, (1+recv_convertor->stack_pos) * sizeof(struct dt_stack_t));
pos = 0;
opal_convertor_set_position(recv_convertor, &pos);
pos = length;
opal_convertor_set_position(recv_convertor, &pos);
assert(pos == length);
for(i = 0; i <= stack_pos; i++ ) {
if( stack[i].index != recv_convertor->pStack[i].index )
{errors = 1; printf("recv stack[%d].index differs (orig %d != new %d) (completed %lu/%lu)\n",
i, stack[i].index, recv_convertor->pStack[i].index,
length, pdt->size * count);}
if( stack[i].count != recv_convertor->pStack[i].count ) {
if( stack[i].type == recv_convertor->pStack[i].type ) {
{errors = 1; printf("recv stack[%d].count differs (orig %lu != new %lu) (completed %lu/%lu)\n",
i, stack[i].count, recv_convertor->pStack[i].count,
length, pdt->size * count);}
} else {
if( (OPAL_DATATYPE_MAX_PREDEFINED <= stack[i].type) || (OPAL_DATATYPE_MAX_PREDEFINED <= recv_convertor->pStack[i].type) )
{errors = 1; printf("recv stack[%d].type wrong (orig %d != new %d) (completed %lu/%lu)\n",
i, (int)stack[i].type, (int)recv_convertor->pStack[i].type,
length, pdt->size * count);}
else if( (stack[i].count * opal_datatype_basicDatatypes[stack[i].type]->size) !=
(recv_convertor->pStack[i].count * opal_datatype_basicDatatypes[recv_convertor->pStack[i].type]->size) )
{errors = 1; printf("recv stack[%d].type*count differs (orig (%d,%lu) != new (%d, %lu)) (completed %lu/%lu)\n",
i, (int)stack[i].type, stack[i].count,
(int)recv_convertor->pStack[i].type, recv_convertor->pStack[i].count,
length, pdt->size * count);}
}
}
if( stack[i].disp != recv_convertor->pStack[i].disp )
{errors = 1; printf("recv stack[%d].disp differs (orig %p != new %p) (completed %lu/%lu)\n",
i, (void*)stack[i].disp, (void*)recv_convertor->pStack[i].disp,
length, pdt->size * count);}
if(0 != errors) {assert(0); exit(-1);}
}
}
}
}
GET_TIME( end );
total_time = ELAPSED_TIME( start, end );
printf( "copying same data-type using convertors in %ld microsec\n", total_time );
printf( "\t unpack in %ld microsec [pack in %ld microsec]\n", unpack_time,
total_time - unpack_time );
if(outputFlags & VALIDATE_DATA) {
for( size_t i = errors = 0; i < malloced_size; i++ ) {
if( odst[i] != osrc[i] ) {
printf("error at position %lu (%d != %d)\n",
(unsigned long)i, (int)(odst[i]), (int)(osrc[i]));
errors++;
if(outputFlags & QUIT_ON_FIRST_ERROR) {
opal_datatype_dump(pdt);
assert(0); exit(-1);
}
}
}
if( 0 == errors ) {
printf("Validation check successfully passed\n");
} else {
printf("Found %d errors. Giving up!\n", errors);
exit(-1);
}
}
clean_and_return:
if( NULL != send_convertor ) OBJ_RELEASE( send_convertor );
if( NULL != recv_convertor ) OBJ_RELEASE( recv_convertor );
if( NULL != odst ) free( odst );
if( NULL != osrc ) free( osrc );
if( NULL != ptemp ) free( ptemp );
return (0 == errors ? OPAL_SUCCESS : errors);
}
int mca_topo_base_dist_graph_create_adjacent(mca_topo_base_module_t* module,
ompi_communicator_t *comm_old,
int indegree, const int sources[],
const int sourceweights[],
int outdegree,
const int destinations[],
const int destweights[],
opal_info_t *info, int reorder,
ompi_communicator_t **newcomm)
{
mca_topo_base_comm_dist_graph_2_2_0_t *topo = NULL;
int err;
if( OMPI_SUCCESS != (err = ompi_comm_create(comm_old,
comm_old->c_local_group,
newcomm)) ) {
return err;
}
err = OMPI_ERR_OUT_OF_RESOURCE; /* suppose by default something bad will happens */
assert( NULL == (*newcomm)->c_topo );
topo = OBJ_NEW(mca_topo_base_comm_dist_graph_2_2_0_t);
if( NULL == topo ) {
goto bail_out;
}
topo->in = topo->inw = NULL;
topo->out = topo->outw = NULL;
topo->indegree = indegree;
topo->outdegree = outdegree;
topo->weighted = !((MPI_UNWEIGHTED == sourceweights) && (MPI_UNWEIGHTED == destweights));
if (topo->indegree > 0) {
topo->in = (int*)malloc(sizeof(int) * topo->indegree);
if (NULL == topo->in) {
goto bail_out;
}
memcpy(topo->in, sources, sizeof(int) * topo->indegree);
if (MPI_UNWEIGHTED != sourceweights) {
topo->inw = (int*)malloc(sizeof(int) * topo->indegree);
if( NULL == topo->inw ) {
goto bail_out;
}
memcpy( topo->inw, sourceweights, sizeof(int) * topo->indegree );
}
}
if (topo->outdegree > 0) {
topo->out = (int*)malloc(sizeof(int) * topo->outdegree);
if (NULL == topo->out) {
goto bail_out;
}
memcpy(topo->out, destinations, sizeof(int) * topo->outdegree);
topo->outw = NULL;
if (MPI_UNWEIGHTED != destweights) {
if (topo->outdegree > 0) {
topo->outw = (int*)malloc(sizeof(int) * topo->outdegree);
if (NULL == topo->outw) {
goto bail_out;
}
memcpy(topo->outw, destweights, sizeof(int) * topo->outdegree);
}
}
}
(*newcomm)->c_topo = module;
(*newcomm)->c_topo->mtc.dist_graph = topo;
(*newcomm)->c_topo->reorder = reorder;
(*newcomm)->c_flags |= OMPI_COMM_DIST_GRAPH;
return OMPI_SUCCESS;
bail_out:
if (NULL != topo) {
if( NULL != topo->in ) free(topo->in);
if( MPI_UNWEIGHTED != sourceweights ) {
if( NULL != topo->inw ) free(topo->inw);
}
if( NULL != topo->out ) free(topo->out);
if( MPI_UNWEIGHTED != destweights ) {
if( NULL != topo->outw ) free(topo->outw);
}
OBJ_RELEASE(topo);
}
ompi_comm_free(newcomm);
return err;
}
/**
* Main function. Call several tests and print-out the results. It try to stress the convertor
* using difficult data-type constructions as well as strange segment sizes for the conversion.
* Usually, it is able to detect most of the data-type and convertor problems. Any modifications
* on the data-type engine should first pass all the tests from this file, before going into other
* tests.
*/
int main( int argc, char* argv[] )
{
opal_datatype_t *pdt, *pdt1, *pdt2, *pdt3;
int rc, length = 500;
opal_datatype_init();
/**
* By default simulate homogeneous architectures.
*/
remote_arch = opal_local_arch;
printf( "\n\n#\n * TEST CREATE CONTIGUOUS\n#\n\n" );
pdt = create_contiguous_type( &opal_datatype_int1, 10 );
if( outputFlags & CHECK_PACK_UNPACK ) {
local_copy_ddt_count(pdt, 100);
local_copy_with_convertor(pdt, 100, 956);
}
OBJ_RELEASE( pdt ); assert( pdt == NULL );
printf( "\n\n#\n * TEST STRANGE DATATYPE\n#\n\n" );
pdt = create_strange_dt();
if( outputFlags & CHECK_PACK_UNPACK ) {
local_copy_ddt_count(pdt, 1);
local_copy_with_convertor(pdt, 1, 956);
}
OBJ_RELEASE( pdt ); assert( pdt == NULL );
printf( "\n\n#\n * TEST UPPER TRIANGULAR MATRIX (size 100)\n#\n\n" );
pdt = upper_matrix(100);
if( outputFlags & CHECK_PACK_UNPACK ) {
local_copy_ddt_count(pdt, 1);
local_copy_with_convertor(pdt, 1, 48);
}
OBJ_RELEASE( pdt ); assert( pdt == NULL );
mpich_typeub();
mpich_typeub2();
mpich_typeub3();
printf( "\n\n#\n * TEST UPPER MATRIX\n#\n\n" );
rc = test_upper( length );
if( rc == 0 )
printf( "decode [PASSED]\n" );
else
printf( "decode [NOT PASSED]\n" );
printf( "\n\n#\n * TEST MATRIX BORDERS\n#\n\n" );
pdt = test_matrix_borders( length, 100 );
if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
opal_datatype_dump( pdt );
}
OBJ_RELEASE( pdt ); assert( pdt == NULL );
printf( "\n\n#\n * TEST CONTIGUOUS\n#\n\n" );
pdt = test_contiguous();
OBJ_RELEASE( pdt ); assert( pdt == NULL );
printf( "\n\n#\n * TEST STRUCT\n#\n\n" );
pdt = test_struct();
OBJ_RELEASE( pdt ); assert( pdt == NULL );
opal_datatype_create_contiguous(0, &opal_datatype_empty, &pdt1);
opal_datatype_create_contiguous(0, &opal_datatype_empty, &pdt2);
opal_datatype_create_contiguous(0, &opal_datatype_empty, &pdt3);
opal_datatype_add( pdt3, &opal_datatype_int4, 10, 0, -1 );
opal_datatype_add( pdt3, &opal_datatype_float4, 5, 10 * sizeof(int), -1 );
opal_datatype_add( pdt2, &opal_datatype_float4, 1, 0, -1 );
opal_datatype_add( pdt2, pdt3, 3, sizeof(int) * 1, -1 );
opal_datatype_add( pdt1, &opal_datatype_int8, 5, 0, -1 );
opal_datatype_add( pdt1, &opal_datatype_float16, 2, sizeof(long long) * 5, -1 );
printf( ">>--------------------------------------------<<\n" );
if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
opal_datatype_dump( pdt1 );
}
printf( ">>--------------------------------------------<<\n" );
if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
opal_datatype_dump( pdt2 );
}
printf( ">>--------------------------------------------<<\n" );
if( outputFlags & DUMP_DATA_AFTER_COMMIT ) {
opal_datatype_dump( pdt3 );
}
OBJ_RELEASE( pdt1 ); assert( pdt1 == NULL );
OBJ_RELEASE( pdt2 ); assert( pdt2 == NULL );
OBJ_RELEASE( pdt3 ); assert( pdt3 == NULL );
printf( ">>--------------------------------------------<<\n" );
printf( " Contiguous data-type (opal_datatype_float8)\n" );
if( outputFlags & CHECK_PACK_UNPACK ) {
opal_datatype_t const * const ddt = &opal_datatype_float8;
local_copy_ddt_count( ddt, 4500);
local_copy_with_convertor( ddt, 4500, 12 );
local_copy_with_convertor_2datatypes( ddt, 4500, ddt, 4500, 12 );
}
printf( ">>--------------------------------------------<<\n" );
printf( ">>--------------------------------------------<<\n" );
if( outputFlags & CHECK_PACK_UNPACK ) {
printf( "Contiguous multiple data-type (4500*1)\n" );
pdt = create_contiguous_type( &opal_datatype_float8, 4500 );
local_copy_ddt_count(pdt, 1);
local_copy_with_convertor( pdt, 1, 12 );
local_copy_with_convertor_2datatypes( pdt, 1, pdt, 1, 12 );
OBJ_RELEASE( pdt ); assert( pdt == NULL );
printf( "Contiguous multiple data-type (450*10)\n" );
pdt = create_contiguous_type( &opal_datatype_float8, 450 );
local_copy_ddt_count(pdt, 10);
local_copy_with_convertor( pdt, 10, 12 );
local_copy_with_convertor_2datatypes( pdt, 10, pdt, 10, 12 );
OBJ_RELEASE( pdt ); assert( pdt == NULL );
printf( "Contiguous multiple data-type (45*100)\n" );
pdt = create_contiguous_type( &opal_datatype_float8, 45 );
local_copy_ddt_count(pdt, 100);
local_copy_with_convertor( pdt, 100, 12 );
local_copy_with_convertor_2datatypes( pdt, 100, pdt, 100, 12 );
OBJ_RELEASE( pdt ); assert( pdt == NULL );
printf( "Contiguous multiple data-type (100*45)\n" );
pdt = create_contiguous_type( &opal_datatype_float8, 100 );
local_copy_ddt_count(pdt, 45);
local_copy_with_convertor( pdt, 45, 12 );
local_copy_with_convertor_2datatypes( pdt, 45, pdt, 45, 12 );
OBJ_RELEASE( pdt ); assert( pdt == NULL );
printf( "Contiguous multiple data-type (10*450)\n" );
pdt = create_contiguous_type( &opal_datatype_float8, 10 );
local_copy_ddt_count(pdt, 450);
local_copy_with_convertor( pdt, 450, 12 );
local_copy_with_convertor_2datatypes( pdt, 450, pdt, 450, 12 );
OBJ_RELEASE( pdt ); assert( pdt == NULL );
printf( "Contiguous multiple data-type (1*4500)\n" );
pdt = create_contiguous_type( &opal_datatype_float8, 1 );
local_copy_ddt_count(pdt, 4500);
local_copy_with_convertor( pdt, 4500, 12 );
local_copy_with_convertor_2datatypes( pdt, 4500, pdt, 4500, 12 );
OBJ_RELEASE( pdt ); assert( pdt == NULL );
}
printf( ">>--------------------------------------------<<\n" );
printf( ">>--------------------------------------------<<\n" );
printf( "Vector data-type (450 times 10 double stride 11)\n" );
pdt = create_vector_type( &opal_datatype_float8, 450, 10, 11 );
opal_datatype_dump( pdt );
if( outputFlags & CHECK_PACK_UNPACK ) {
local_copy_ddt_count(pdt, 1);
local_copy_with_convertor( pdt, 1, 12 );
local_copy_with_convertor_2datatypes( pdt, 1, pdt, 1, 12 );
local_copy_with_convertor( pdt, 1, 82 );
local_copy_with_convertor_2datatypes( pdt, 1, pdt, 1, 82 );
local_copy_with_convertor( pdt, 1, 6000 );
local_copy_with_convertor_2datatypes( pdt, 1, pdt, 1, 6000 );
local_copy_with_convertor( pdt, 1, 36000 );
local_copy_with_convertor_2datatypes( pdt, 1, pdt, 1, 36000 );
}
printf( ">>--------------------------------------------<<\n" );
OBJ_RELEASE( pdt ); assert( pdt == NULL );
printf( ">>--------------------------------------------<<\n" );
printf( "Struct data-type resized (double unused followed by 2 used doubles)\n" );
pdt = create_struct_constant_gap_resized_ddt( &opal_datatype_float8 );
opal_datatype_dump( pdt );
if( outputFlags & CHECK_PACK_UNPACK ) {
local_copy_ddt_count(pdt, 1);
local_copy_with_convertor( pdt, 100, 11 );
local_copy_with_convertor_2datatypes( pdt, 100, pdt, 100, 11 );
local_copy_with_convertor( pdt, 100, 82 );
local_copy_with_convertor_2datatypes( pdt, 100, pdt, 100, 81 );
local_copy_with_convertor( pdt, 1500, 6000 );
local_copy_with_convertor_2datatypes( pdt, 1500, pdt, 1500, 666 );
local_copy_with_convertor( pdt, 10000, 36000 );
local_copy_with_convertor_2datatypes( pdt, 10000, pdt, 10000, 1111 );
}
printf( ">>--------------------------------------------<<\n" );
OBJ_RELEASE( pdt ); assert( pdt == NULL );
printf( ">>--------------------------------------------<<\n" );
pdt = test_struct_char_double();
if( outputFlags & CHECK_PACK_UNPACK ) {
local_copy_ddt_count(pdt, 4500);
local_copy_with_convertor( pdt, 4500, 12 );
local_copy_with_convertor_2datatypes( pdt, 4500, pdt, 4500, 12 );
}
printf( ">>--------------------------------------------<<\n" );
OBJ_RELEASE( pdt ); assert( pdt == NULL );
printf( ">>--------------------------------------------<<\n" );
pdt = test_create_twice_two_doubles();
if( outputFlags & CHECK_PACK_UNPACK ) {
local_copy_ddt_count(pdt, 4500);
local_copy_with_convertor( pdt, 4500, 12 );
local_copy_with_convertor_2datatypes( pdt, 4500, pdt, 4500, 12 );
}
printf( ">>--------------------------------------------<<\n" );
OBJ_RELEASE( pdt ); assert( pdt == NULL );
printf( ">>--------------------------------------------<<\n" );
pdt = test_create_blacs_type();
if( outputFlags & CHECK_PACK_UNPACK ) {
opal_datatype_dump( pdt );
local_copy_ddt_count(pdt, 4500);
local_copy_with_convertor( pdt, 4500, 956 );
local_copy_with_convertor_2datatypes( pdt, 4500, pdt, 4500, 956 );
local_copy_with_convertor( pdt, 4500, 16*1024 );
local_copy_with_convertor_2datatypes( pdt, 4500, pdt, 4500, 16*1024 );
local_copy_with_convertor( pdt, 4500, 64*1024 );
local_copy_with_convertor_2datatypes( pdt, 4500, pdt, 4500, 64*1024 );
}
printf( ">>--------------------------------------------<<\n" );
OBJ_RELEASE( pdt ); assert( pdt == NULL );
printf( ">>--------------------------------------------<<\n" );
pdt1 = test_create_blacs_type1( &opal_datatype_int4 );
pdt2 = test_create_blacs_type2( &opal_datatype_int4 );
if( outputFlags & CHECK_PACK_UNPACK ) {
local_copy_with_convertor_2datatypes( pdt1, 1, pdt2, 1, 100 );
}
printf( ">>--------------------------------------------<<\n" );
OBJ_RELEASE( pdt1 ); assert( pdt1 == NULL );
OBJ_RELEASE( pdt2 ); assert( pdt2 == NULL );
/* clean-ups all data allocations */
opal_datatype_finalize();
opal_finalize();
return OPAL_SUCCESS;
}
static int
ompi_mtl_ofi_component_register(void)
{
int ret;
mca_base_var_enum_t *new_enum = NULL;
param_priority = 25; /* for now give a lower priority than the psm mtl */
mca_base_component_var_register(&mca_mtl_ofi_component.super.mtl_version,
"priority", "Priority of the OFI MTL component",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
¶m_priority);
prov_include = "psm,psm2,gni";
mca_base_component_var_register(&mca_mtl_ofi_component.super.mtl_version,
"provider_include",
"Comma-delimited list of OFI providers that are considered for use (e.g., \"psm,psm2\"; an empty value means that all providers will be considered). Mutually exclusive with mtl_ofi_provider_exclude.",
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_1,
MCA_BASE_VAR_SCOPE_READONLY,
&prov_include);
prov_exclude = NULL;
mca_base_component_var_register(&mca_mtl_ofi_component.super.mtl_version,
"provider_exclude",
"Comma-delimited list of OFI providers that are not considered for use (default: \"sockets,mxm\"; empty value means that all providers will be considered). Mutually exclusive with mtl_ofi_provider_include.",
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_1,
MCA_BASE_VAR_SCOPE_READONLY,
&prov_exclude);
ret = mca_base_var_enum_create ("control_prog_type", control_prog_type, &new_enum);
if (OPAL_SUCCESS != ret) {
return ret;
}
control_progress = MTL_OFI_PROG_MANUAL;
mca_base_component_var_register (&mca_mtl_ofi_component.super.mtl_version,
"control_progress",
"Specify control progress model (default: manual). Set to auto for auto progress.",
MCA_BASE_VAR_TYPE_INT, new_enum, 0, 0,
OPAL_INFO_LVL_3,
MCA_BASE_VAR_SCOPE_READONLY,
&control_progress);
OBJ_RELEASE(new_enum);
ret = mca_base_var_enum_create ("data_prog_type", data_prog_type, &new_enum);
if (OPAL_SUCCESS != ret) {
return ret;
}
data_progress = MTL_OFI_PROG_AUTO;
mca_base_component_var_register(&mca_mtl_ofi_component.super.mtl_version,
"data_progress",
"Specify data progress model (default: auto). Set to manual for manual progress.",
MCA_BASE_VAR_TYPE_INT, new_enum, 0, 0,
OPAL_INFO_LVL_3,
MCA_BASE_VAR_SCOPE_READONLY,
&data_progress);
OBJ_RELEASE(new_enum);
ret = mca_base_var_enum_create ("av_type", av_table_type, &new_enum);
if (OPAL_SUCCESS != ret) {
return ret;
}
av_type = MTL_OFI_AV_MAP;
mca_base_component_var_register (&mca_mtl_ofi_component.super.mtl_version,
"av",
"Specify AV type to use (default: map). Set to table for FI_AV_TABLE AV type.",
MCA_BASE_VAR_TYPE_INT, new_enum, 0, 0,
OPAL_INFO_LVL_3,
MCA_BASE_VAR_SCOPE_READONLY,
&av_type);
OBJ_RELEASE(new_enum);
return OMPI_SUCCESS;
}
void mca_oob_tcp_recv_handler(int sd, short flags, void *cbdata)
{
mca_oob_tcp_peer_t* peer = (mca_oob_tcp_peer_t*)cbdata;
int rc;
orte_rml_send_t *snd;
if (orte_abnormal_term_ordered) {
return;
}
opal_output_verbose(OOB_TCP_DEBUG_CONNECT, orte_oob_base_framework.framework_output,
"%s:tcp:recv:handler called for peer %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&peer->name));
switch (peer->state) {
case MCA_OOB_TCP_CONNECT_ACK:
if (ORTE_SUCCESS == (rc = mca_oob_tcp_peer_recv_connect_ack(peer, peer->sd, NULL))) {
opal_output_verbose(OOB_TCP_DEBUG_CONNECT, orte_oob_base_framework.framework_output,
"%s:tcp:recv:handler starting send/recv events",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
/* we connected! Start the send/recv events */
if (!peer->recv_ev_active) {
opal_event_add(&peer->recv_event, 0);
peer->recv_ev_active = true;
}
if (peer->timer_ev_active) {
opal_event_del(&peer->timer_event);
peer->timer_ev_active = false;
}
/* if there is a message waiting to be sent, queue it */
if (NULL == peer->send_msg) {
peer->send_msg = (mca_oob_tcp_send_t*)opal_list_remove_first(&peer->send_queue);
}
if (NULL != peer->send_msg && !peer->send_ev_active) {
opal_event_add(&peer->send_event, 0);
peer->send_ev_active = true;
}
/* update our state */
peer->state = MCA_OOB_TCP_CONNECTED;
} else {
opal_output_verbose(OOB_TCP_DEBUG_CONNECT, orte_oob_base_framework.framework_output,
"%s UNABLE TO COMPLETE CONNECT ACK WITH %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&peer->name));
opal_event_del(&peer->recv_event);
ORTE_FORCED_TERMINATE(1);
return;
}
break;
case MCA_OOB_TCP_CONNECTED:
opal_output_verbose(OOB_TCP_DEBUG_CONNECT, orte_oob_base_framework.framework_output,
"%s:tcp:recv:handler CONNECTED",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
/* allocate a new message and setup for recv */
if (NULL == peer->recv_msg) {
opal_output_verbose(OOB_TCP_DEBUG_CONNECT, orte_oob_base_framework.framework_output,
"%s:tcp:recv:handler allocate new recv msg",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
peer->recv_msg = OBJ_NEW(mca_oob_tcp_recv_t);
if (NULL == peer->recv_msg) {
opal_output(0, "%s-%s mca_oob_tcp_peer_recv_handler: unable to allocate recv message\n",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&(peer->name)));
return;
}
/* start by reading the header */
peer->recv_msg->rdptr = (char*)&peer->recv_msg->hdr;
peer->recv_msg->rdbytes = sizeof(mca_oob_tcp_hdr_t);
}
/* if the header hasn't been completely read, read it */
if (!peer->recv_msg->hdr_recvd) {
opal_output_verbose(OOB_TCP_DEBUG_CONNECT, orte_oob_base_framework.framework_output,
"%s:tcp:recv:handler read hdr",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
if (ORTE_SUCCESS == (rc = read_bytes(peer))) {
/* completed reading the header */
peer->recv_msg->hdr_recvd = true;
/* convert the header */
MCA_OOB_TCP_HDR_NTOH(&peer->recv_msg->hdr);
/* if this is a zero-byte message, then we are done */
if (0 == peer->recv_msg->hdr.nbytes) {
opal_output_verbose(OOB_TCP_DEBUG_CONNECT, orte_oob_base_framework.framework_output,
"%s RECVD ZERO-BYTE MESSAGE FROM %s for tag %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&peer->name), peer->recv_msg->hdr.tag);
peer->recv_msg->data = NULL; // make sure
} else {
opal_output_verbose(OOB_TCP_DEBUG_CONNECT, orte_oob_base_framework.framework_output,
"%s:tcp:recv:handler allocate data region of size %lu",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), (unsigned long)peer->recv_msg->hdr.nbytes);
/* allocate the data region */
peer->recv_msg->data = (char*)malloc(peer->recv_msg->hdr.nbytes);
/* point to it */
peer->recv_msg->rdptr = peer->recv_msg->data;
peer->recv_msg->rdbytes = peer->recv_msg->hdr.nbytes;
}
/* fall thru and attempt to read the data */
} else if (ORTE_ERR_RESOURCE_BUSY == rc ||
ORTE_ERR_WOULD_BLOCK == rc) {
/* exit this event and let the event lib progress */
return;
} else {
/* close the connection */
opal_output_verbose(OOB_TCP_DEBUG_CONNECT, orte_oob_base_framework.framework_output,
"%s:tcp:recv:handler error reading bytes - closing connection",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
mca_oob_tcp_peer_close(peer);
return;
}
}
if (peer->recv_msg->hdr_recvd) {
/* continue to read the data block - we start from
* wherever we left off, which could be at the
* beginning or somewhere in the message
*/
if (ORTE_SUCCESS == (rc = read_bytes(peer))) {
/* we recvd all of the message */
opal_output_verbose(OOB_TCP_DEBUG_CONNECT, orte_oob_base_framework.framework_output,
"%s RECVD COMPLETE MESSAGE FROM %s (ORIGIN %s) OF %d BYTES FOR DEST %s TAG %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&peer->name),
ORTE_NAME_PRINT(&peer->recv_msg->hdr.origin),
(int)peer->recv_msg->hdr.nbytes,
ORTE_NAME_PRINT(&peer->recv_msg->hdr.dst),
peer->recv_msg->hdr.tag);
/* am I the intended recipient (header was already converted back to host order)? */
if (peer->recv_msg->hdr.dst.jobid == ORTE_PROC_MY_NAME->jobid &&
peer->recv_msg->hdr.dst.vpid == ORTE_PROC_MY_NAME->vpid) {
/* yes - post it to the RML for delivery */
opal_output_verbose(OOB_TCP_DEBUG_CONNECT, orte_oob_base_framework.framework_output,
"%s DELIVERING TO RML",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
ORTE_RML_POST_MESSAGE(&peer->recv_msg->hdr.origin, peer->recv_msg->hdr.tag,
peer->recv_msg->data,
peer->recv_msg->hdr.nbytes);
OBJ_RELEASE(peer->recv_msg);
} else {
/* promote this to the OOB as some other transport might
* be the next best hop */
opal_output_verbose(OOB_TCP_DEBUG_CONNECT, orte_oob_base_framework.framework_output,
"%s TCP PROMOTING ROUTED MESSAGE FOR %s TO OOB",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&peer->recv_msg->hdr.dst));
snd = OBJ_NEW(orte_rml_send_t);
snd->dst = peer->recv_msg->hdr.dst;
snd->origin = peer->recv_msg->hdr.origin;
snd->tag = peer->recv_msg->hdr.tag;
snd->data = peer->recv_msg->data;
snd->count = peer->recv_msg->hdr.nbytes;
snd->cbfunc.iov = NULL;
snd->cbdata = NULL;
/* activate the OOB send state */
ORTE_OOB_SEND(snd);
/* protect the data */
peer->recv_msg->data = NULL;
/* cleanup */
OBJ_RELEASE(peer->recv_msg);
}
peer->recv_msg = NULL;
return;
} else if (ORTE_ERR_RESOURCE_BUSY == rc ||
ORTE_ERR_WOULD_BLOCK == rc) {
/* exit this event and let the event lib progress */
return;
} else {
// report the error
opal_output(0, "%s-%s mca_oob_tcp_peer_recv_handler: unable to recv message",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&(peer->name)));
/* turn off the recv event */
opal_event_del(&peer->recv_event);
ORTE_FORCED_TERMINATE(1);
return;
}
}
break;
default:
opal_output(0, "%s-%s mca_oob_tcp_peer_recv_handler: invalid socket state(%d)",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&(peer->name)),
peer->state);
// mca_oob_tcp_peer_close(peer);
break;
}
}
