static int ompi_cr_coord_pre_continue(void) {
#if OPAL_ENABLE_FT_CR == 1
int ret, exit_status = OMPI_SUCCESS;
/*
* Can not really do much until ORTE is up and running,
* so defer action until the post_continue function.
*/
opal_output_verbose(10, ompi_cr_output,
"ompi_cr: coord_pre_continue: ompi_cr_coord_pre_continue()");
if( orte_cr_continue_like_restart ) {
/* Mimic ompi_cr_coord_pre_restart(); */
if( OMPI_SUCCESS != (ret = mca_pml.pml_ft_event(OPAL_CRS_CONTINUE))) {
exit_status = ret;
goto cleanup;
}
}
else {
if( opal_cr_timing_barrier_enabled ) {
OPAL_CR_SET_TIMER(OPAL_CR_TIMER_P2PBR1);
}
OPAL_CR_SET_TIMER(OPAL_CR_TIMER_P2P3);
if( opal_cr_timing_barrier_enabled ) {
OPAL_CR_SET_TIMER(OPAL_CR_TIMER_P2PBR2);
}
OPAL_CR_SET_TIMER(OPAL_CR_TIMER_CRCP1);
}
cleanup:
return exit_status;
#else
return OMPI_SUCCESS;
#endif
}
int opal_cr_inc_core_ckpt(pid_t pid,
opal_crs_base_snapshot_t *snapshot,
opal_crs_base_ckpt_options_t *options,
int *state)
{
int ret, exit_status = OPAL_SUCCESS;
OPAL_CR_SET_TIMER(OPAL_CR_TIMER_CORE0);
if(OPAL_SUCCESS != (ret = opal_crs.crs_checkpoint(pid,
snapshot,
options,
(opal_crs_state_type_t *)state))) {
opal_output(opal_cr_output,
"opal_cr: inc_core: Error: The checkpoint failed. %d\n", ret);
exit_status = ret;
}
if(*state == OPAL_CRS_CONTINUE) {
OPAL_CR_SET_TIMER(OPAL_CR_TIMER_CORE1);
if(options->term) {
*state = OPAL_CRS_TERM;
opal_cr_checkpointing_state = OPAL_CR_STATUS_TERM;
} else {
opal_cr_checkpointing_state = OPAL_CR_STATUS_CONTINUE;
}
}
else {
options->term = false;
}
/*
* If restarting read environment stuff that opal-restart left us.
*/
if(*state == OPAL_CRS_RESTART) {
opal_cr_refresh_environ(core_prev_pid);
opal_cr_checkpointing_state = OPAL_CR_STATUS_RESTART_PRE;
}
return exit_status;
}
int mca_pml_ob1_ft_event( int state )
{
static bool first_continue_pass = false;
ompi_proc_t** procs = NULL;
size_t num_procs;
int ret, p;
ompi_rte_collective_t *coll, *modex;
coll = OBJ_NEW(ompi_rte_collective_t);
coll->id = ompi_process_info.peer_init_barrier;
if(OPAL_CRS_CHECKPOINT == state) {
if( opal_cr_timing_barrier_enabled ) {
OPAL_CR_SET_TIMER(OPAL_CR_TIMER_CRCPBR1);
ompi_rte_barrier(coll);
OMPI_WAIT_FOR_COMPLETION(coll->active);
}
OPAL_CR_SET_TIMER(OPAL_CR_TIMER_P2P0);
}
else if(OPAL_CRS_CONTINUE == state) {
first_continue_pass = !first_continue_pass;
if( !first_continue_pass ) {
if( opal_cr_timing_barrier_enabled ) {
OPAL_CR_SET_TIMER(OPAL_CR_TIMER_COREBR0);
ompi_rte_barrier(coll);
OMPI_WAIT_FOR_COMPLETION(coll->active);
}
OPAL_CR_SET_TIMER(OPAL_CR_TIMER_P2P2);
}
if( orte_cr_continue_like_restart && !first_continue_pass ) {
/*
* Get a list of processes
*/
procs = ompi_proc_all(&num_procs);
if(NULL == procs) {
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto clean;
}
/*
* Refresh the proc structure, and publish our proc info in the modex.
* NOTE: Do *not* call ompi_proc_finalize as there are many places in
* the code that point to indv. procs in this strucutre. For our
* needs here we only need to fix up the modex, bml and pml
* references.
*/
if (OMPI_SUCCESS != (ret = ompi_proc_refresh())) {
opal_output(0,
"pml:ob1: ft_event(Restart): proc_refresh Failed %d",
ret);
for(p = 0; p < (int)num_procs; ++p) {
OBJ_RELEASE(procs[p]);
}
free (procs);
goto clean;
}
}
}
else if(OPAL_CRS_RESTART_PRE == state ) {
/* Nothing here */
}
else if(OPAL_CRS_RESTART == state ) {
/*
* Get a list of processes
*/
procs = ompi_proc_all(&num_procs);
if(NULL == procs) {
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto clean;
}
/*
* Clean out the modex information since it is invalid now.
* ompi_rte_purge_proc_attrs();
* This happens at the ORTE level, so doing it again here will cause
* some issues with socket caching.
*/
/*
* Refresh the proc structure, and publish our proc info in the modex.
* NOTE: Do *not* call ompi_proc_finalize as there are many places in
* the code that point to indv. procs in this strucutre. For our
* needs here we only need to fix up the modex, bml and pml
* references.
*/
if (OMPI_SUCCESS != (ret = ompi_proc_refresh())) {
opal_output(0,
"pml:ob1: ft_event(Restart): proc_refresh Failed %d",
ret);
for(p = 0; p < (int)num_procs; ++p) {
OBJ_RELEASE(procs[p]);
}
free (procs);
goto clean;
}
}
else if(OPAL_CRS_TERM == state ) {
;
}
else {
;
}
/* Call the BML
* BML is expected to call ft_event in
* - BTL(s)
* - MPool(s)
*/
if( OMPI_SUCCESS != (ret = mca_bml.bml_ft_event(state))) {
opal_output(0, "pml:base: ft_event: BML ft_event function failed: %d\n",
ret);
}
if(OPAL_CRS_CHECKPOINT == state) {
OPAL_CR_SET_TIMER(OPAL_CR_TIMER_P2P1);
if( opal_cr_timing_barrier_enabled ) {
OPAL_CR_SET_TIMER(OPAL_CR_TIMER_P2PBR0);
/* JJH Cannot barrier here due to progress engine -- ompi_rte_barrier();*/
}
}
else if(OPAL_CRS_CONTINUE == state) {
if( !first_continue_pass ) {
if( opal_cr_timing_barrier_enabled ) {
OPAL_CR_SET_TIMER(OPAL_CR_TIMER_P2PBR1);
ompi_rte_barrier(coll);
OMPI_WAIT_FOR_COMPLETION(coll->active);
}
OPAL_CR_SET_TIMER(OPAL_CR_TIMER_P2P3);
}
if( orte_cr_continue_like_restart && !first_continue_pass ) {
/*
* Exchange the modex information once again.
* BTLs will have republished their modex information.
*/
modex = OBJ_NEW(ompi_rte_collective_t);
modex->id = ompi_process_info.peer_modex;
if (OMPI_SUCCESS != (ret = orte_grpcomm.modex(modex))) {
opal_output(0,
"pml:ob1: ft_event(Restart): Failed orte_grpcomm.modex() = %d",
ret);
OBJ_RELEASE(modex);
goto clean;
}
OMPI_WAIT_FOR_COMPLETION(modex->active);
OBJ_RELEASE(modex);
/*
* Startup the PML stack now that the modex is running again
* Add the new procs (BTLs redo modex recv's)
*/
if( OMPI_SUCCESS != (ret = mca_pml_ob1_add_procs(procs, num_procs) ) ) {
opal_output(0, "pml:ob1: ft_event(Restart): Failed in add_procs (%d)", ret);
goto clean;
}
/* Is this barrier necessary ? JJH */
if (OMPI_SUCCESS != (ret = ompi_rte_barrier(coll))) {
opal_output(0, "pml:ob1: ft_event(Restart): Failed in ompi_rte_barrier (%d)", ret);
goto clean;
}
OMPI_WAIT_FOR_COMPLETION(coll->active);
if( NULL != procs ) {
for(p = 0; p < (int)num_procs; ++p) {
OBJ_RELEASE(procs[p]);
}
free(procs);
procs = NULL;
}
}
if( !first_continue_pass ) {
if( opal_cr_timing_barrier_enabled ) {
OPAL_CR_SET_TIMER(OPAL_CR_TIMER_P2PBR2);
ompi_rte_barrier(coll);
OMPI_WAIT_FOR_COMPLETION(coll->active);
}
OPAL_CR_SET_TIMER(OPAL_CR_TIMER_CRCP1);
}
}
else if(OPAL_CRS_RESTART_PRE == state ) {
/* Nothing here */
}
else if(OPAL_CRS_RESTART == state ) {
/*
* Exchange the modex information once again.
* BTLs will have republished their modex information.
*/
modex = OBJ_NEW(ompi_rte_collective_t);
modex->id = ompi_process_info.peer_modex;
if (OMPI_SUCCESS != (ret = orte_grpcomm.modex(modex))) {
opal_output(0,
"pml:ob1: ft_event(Restart): Failed orte_grpcomm.modex() = %d",
ret);
OBJ_RELEASE(modex);
goto clean;
}
OMPI_WAIT_FOR_COMPLETION(modex->active);
OBJ_RELEASE(modex);
/*
* Startup the PML stack now that the modex is running again
* Add the new procs (BTLs redo modex recv's)
*/
if( OMPI_SUCCESS != (ret = mca_pml_ob1_add_procs(procs, num_procs) ) ) {
opal_output(0, "pml:ob1: ft_event(Restart): Failed in add_procs (%d)", ret);
goto clean;
}
/* Is this barrier necessary ? JJH */
if (OMPI_SUCCESS != (ret = ompi_rte_barrier(coll))) {
opal_output(0, "pml:ob1: ft_event(Restart): Failed in ompi_rte_barrier (%d)", ret);
goto clean;
}
OMPI_WAIT_FOR_COMPLETION(coll->active);
if( NULL != procs ) {
for(p = 0; p < (int)num_procs; ++p) {
OBJ_RELEASE(procs[p]);
}
free(procs);
procs = NULL;
}
}
else if(OPAL_CRS_TERM == state ) {
;
}
else {
;
}
ret = OMPI_SUCCESS;
clean:
OBJ_RELEASE(coll);
return ret;
}
int opal_cr_inc_core_recover(int state)
{
int ret;
opal_cr_user_inc_callback_state_t cb_state;
if( opal_cr_checkpointing_state != OPAL_CR_STATUS_TERM &&
opal_cr_checkpointing_state != OPAL_CR_STATUS_CONTINUE &&
opal_cr_checkpointing_state != OPAL_CR_STATUS_RESTART_PRE &&
opal_cr_checkpointing_state != OPAL_CR_STATUS_RESTART_POST ) {
if(state == OPAL_CRS_CONTINUE) {
OPAL_CR_SET_TIMER(OPAL_CR_TIMER_CORE1);
opal_cr_checkpointing_state = OPAL_CR_STATUS_CONTINUE;
}
/*
* If restarting read environment stuff that opal-restart left us.
*/
else if(state == OPAL_CRS_RESTART) {
opal_cr_refresh_environ(core_prev_pid);
opal_cr_checkpointing_state = OPAL_CR_STATUS_RESTART_PRE;
}
}
/*
* Call User Level INC
*/
if( OPAL_CRS_CONTINUE == state ) {
cb_state = OPAL_CR_INC_STATE_CONTINUE;
}
else if( OPAL_CRS_RESTART == state ) {
cb_state = OPAL_CR_INC_STATE_RESTART;
}
else {
cb_state = OPAL_CR_INC_STATE_ERROR;
}
if(OPAL_SUCCESS != (ret = trigger_user_inc_callback(OPAL_CR_INC_POST_CRS_PRE_MPI,
cb_state)) ) {
return ret;
}
/*
* Use the registered coordination routine
*/
if(OPAL_SUCCESS != (ret = cur_coord_callback(state)) ) {
if ( OPAL_EXISTS != ret ) {
opal_output(opal_cr_output,
"opal_cr: inc_core: Error: cur_coord_callback(%d) failed! %d\n",
state, ret);
}
return ret;
}
if(OPAL_SUCCESS != (ret = trigger_user_inc_callback(OPAL_CR_INC_POST_CRS_POST_MPI,
cb_state)) ) {
return ret;
}
#if OPAL_ENABLE_CRDEBUG == 1
opal_cr_debug_clear_current_ckpt_thread();
#endif
return OPAL_SUCCESS;
}
/*******************************
* Notification Routines
*******************************/
int opal_cr_inc_core(pid_t pid, opal_crs_base_snapshot_t *snapshot, bool term, int *state)
{
int ret, exit_status = OPAL_SUCCESS;
int prev_pid = 0;
prev_pid = getpid();
/*
* Use the registered coordination routine
*/
if(OPAL_SUCCESS != (ret = cur_coord_callback(OPAL_CRS_CHECKPOINT)) ) {
if ( OPAL_EXISTS != ret ) {
opal_output(opal_cr_output,
"opal_cr: inc_core: Error: cur_coord_callback(%d) failed! %d\n",
OPAL_CRS_CHECKPOINT, ret);
}
exit_status = ret;
goto cleanup;
}
/*
* Take the checkpoint
*/
OPAL_CR_SET_TIMER(OPAL_CR_TIMER_CORE0);
if(OPAL_SUCCESS != (ret = opal_crs.crs_checkpoint(pid, snapshot, (opal_crs_state_type_t *)state))) {
opal_output(opal_cr_output,
"opal_cr: inc_core: Error: The checkpoint failed. %d\n", ret);
exit_status = ret;
/* Don't return here since we want to restart the OPAL level stuff */
}
if(*state == OPAL_CRS_CONTINUE) {
OPAL_CR_SET_TIMER(OPAL_CR_TIMER_CORE1);
if(term) {
*state = OPAL_CRS_TERM;
opal_cr_checkpointing_state = OPAL_CR_STATUS_TERM;
} else {
opal_cr_checkpointing_state = OPAL_CR_STATUS_CONTINUE;
}
}
else {
term = false;
}
/*
* If restarting read environment stuff that opal-restart left us.
*/
if(*state == OPAL_CRS_RESTART) {
extract_env_vars(prev_pid);
opal_cr_checkpointing_state = OPAL_CR_STATUS_RESTART_PRE;
}
/*
* Use the registered coordination routine
*/
if(OPAL_SUCCESS != (ret = cur_coord_callback(*state)) ) {
if ( OPAL_EXISTS != ret ) {
opal_output(opal_cr_output,
"opal_cr: inc_core: Error: cur_coord_callback(%d) failed! %d\n",
*state, ret);
}
exit_status = ret;
goto cleanup;
}
cleanup:
return exit_status;
}
