static int rte_ft_event(int state) { int ret, exit_status = ORTE_SUCCESS; orte_proc_type_t svtype; orte_grpcomm_collective_t coll; OBJ_CONSTRUCT(&coll, orte_grpcomm_collective_t); coll.id = orte_process_info.peer_init_barrier; /******** Checkpoint Prep ********/ if(OPAL_CRS_CHECKPOINT == state) { /* * Notify SnapC */ if( ORTE_SUCCESS != (ret = orte_snapc.ft_event(OPAL_CRS_CHECKPOINT))) { ORTE_ERROR_LOG(ret); exit_status = ret; goto cleanup; } /* * Notify Routed */ if( ORTE_SUCCESS != (ret = orte_routed.ft_event(OPAL_CRS_CHECKPOINT))) { ORTE_ERROR_LOG(ret); exit_status = ret; goto cleanup; } /* * Notify RML -> OOB */ if( ORTE_SUCCESS != (ret = orte_rml.ft_event(OPAL_CRS_CHECKPOINT))) { ORTE_ERROR_LOG(ret); exit_status = ret; goto cleanup; } } /******** Continue Recovery ********/ else if (OPAL_CRS_CONTINUE == state ) { OPAL_OUTPUT_VERBOSE((1, orte_ess_base_output, "ess:env ft_event(%2d) - %s is Continuing", state, ORTE_NAME_PRINT(ORTE_PROC_MY_NAME))); /* * Notify RML -> OOB */ if( ORTE_SUCCESS != (ret = orte_rml.ft_event(OPAL_CRS_CONTINUE))) { ORTE_ERROR_LOG(ret); exit_status = ret; goto cleanup; } /* * Notify Routed */ if( ORTE_SUCCESS != (ret = orte_routed.ft_event(OPAL_CRS_CONTINUE))) { ORTE_ERROR_LOG(ret); exit_status = ret; goto cleanup; } /* * Notify SnapC */ if( ORTE_SUCCESS != (ret = orte_snapc.ft_event(OPAL_CRS_CONTINUE))) { ORTE_ERROR_LOG(ret); exit_status = ret; goto cleanup; } if( orte_cr_continue_like_restart ) { /* * Barrier to make all processes have been successfully restarted before * we try to remove some restart only files. */ if (ORTE_SUCCESS != (ret = orte_grpcomm.barrier(&coll))) { opal_output(0, "ess:env: ft_event(%2d): Failed in orte_grpcomm.barrier (%d)", state, ret); exit_status = ret; goto cleanup; } ORTE_WAIT_FOR_COMPLETION(coll.active); if( orte_cr_flush_restart_files ) { OPAL_OUTPUT_VERBOSE((1, orte_ess_base_output, "ess:env ft_event(%2d): %s " "Cleanup restart files...", state, ORTE_NAME_PRINT(ORTE_PROC_MY_NAME))); opal_crs_base_cleanup_flush(); } } } /******** Restart Recovery ********/ else if (OPAL_CRS_RESTART == state ) { OPAL_OUTPUT_VERBOSE((1, orte_ess_base_output, "ess:env ft_event(%2d) - %s is Restarting", state, ORTE_NAME_PRINT(ORTE_PROC_MY_NAME))); /* * This should follow the ess init() function */ /* * Clear nidmap and jmap */ orte_util_nidmap_finalize(); /* * - Reset Contact information */ if( ORTE_SUCCESS != (ret = env_set_name() ) ) { exit_status = ret; } /* * Notify RML -> OOB */ if( ORTE_SUCCESS != (ret = orte_rml.ft_event(OPAL_CRS_RESTART))) { ORTE_ERROR_LOG(ret); exit_status = ret; goto cleanup; } /* * Restart the routed framework * JJH: Lie to the finalize function so it does not try to contact the daemon. */ svtype = orte_process_info.proc_type; orte_process_info.proc_type = ORTE_PROC_TOOL; if (ORTE_SUCCESS != (ret = orte_routed.finalize()) ) { ORTE_ERROR_LOG(ret); exit_status = ret; goto cleanup; } orte_process_info.proc_type = svtype; if (ORTE_SUCCESS != (ret = orte_routed.initialize()) ) { ORTE_ERROR_LOG(ret); exit_status = ret; goto cleanup; } /* * Group Comm - Clean out stale data */ orte_grpcomm.finalize(); if (ORTE_SUCCESS != (ret = orte_grpcomm.init())) { ORTE_ERROR_LOG(ret); exit_status = ret; goto cleanup; } if (ORTE_SUCCESS != (ret = orte_db.remove(NULL, NULL))) { ORTE_ERROR_LOG(ret); exit_status = ret; goto cleanup; } /* * Restart the PLM - Does nothing at the moment, but included for completeness */ if (ORTE_SUCCESS != (ret = orte_plm.finalize())) { ORTE_ERROR_LOG(ret); exit_status = ret; goto cleanup; } if (ORTE_SUCCESS != (ret = orte_plm.init())) { ORTE_ERROR_LOG(ret); exit_status = ret; goto cleanup; } /* * RML - Enable communications */ if (ORTE_SUCCESS != (ret = orte_rml.enable_comm())) { ORTE_ERROR_LOG(ret); exit_status = ret; goto cleanup; } /* * Notify Routed */ if( ORTE_SUCCESS != (ret = orte_routed.ft_event(OPAL_CRS_RESTART))) { ORTE_ERROR_LOG(ret); exit_status = ret; goto cleanup; } /* if one was provided, build my nidmap */ if (ORTE_SUCCESS != (ret = orte_util_nidmap_init(orte_process_info.sync_buf))) { ORTE_ERROR_LOG(ret); exit_status = ret; goto cleanup; } /* * Barrier to make all processes have been successfully restarted before * we try to remove some restart only files. */ if (ORTE_SUCCESS != (ret = orte_grpcomm.barrier(&coll))) { opal_output(0, "ess:env ft_event(%2d): Failed in orte_grpcomm.barrier (%d)", state, ret); exit_status = ret; goto cleanup; } ORTE_WAIT_FOR_COMPLETION(coll.active); if( orte_cr_flush_restart_files ) { OPAL_OUTPUT_VERBOSE((1, orte_ess_base_output, "ess:env ft_event(%2d): %s " "Cleanup restart files...", state, ORTE_NAME_PRINT(ORTE_PROC_MY_NAME))); opal_crs_base_cleanup_flush(); } /* * Session directory re-init */ if (orte_create_session_dirs) { if (ORTE_SUCCESS != (ret = orte_session_dir(true, orte_process_info.tmpdir_base, orte_process_info.nodename, NULL, /* Batch ID -- Not used */ ORTE_PROC_MY_NAME))) { exit_status = ret; } opal_output_set_output_file_info(orte_process_info.proc_session_dir, "output-", NULL, NULL); } /* * Notify SnapC */ if( ORTE_SUCCESS != (ret = orte_snapc.ft_event(OPAL_CRS_RESTART))) { ORTE_ERROR_LOG(ret); exit_status = ret; goto cleanup; } } else if (OPAL_CRS_TERM == state ) { /* Nothing */ } else { /* Error state = Nothing */ } cleanup: OBJ_DESTRUCT(&coll); return exit_status; }
int mca_bml_r2_ft_event(int state) { static bool first_continue_pass = false; ompi_proc_t** procs = NULL; size_t num_procs; size_t btl_idx; int ret, p; int loc_state; int param_type = -1; char *param_list = NULL; if(OPAL_CRS_CHECKPOINT == state) { /* Do nothing for now */ } else if(OPAL_CRS_CONTINUE == state) { first_continue_pass = !first_continue_pass; /* Since nothing in Checkpoint, we are fine here (unless required by BTL) */ if( ompi_cr_continue_like_restart && !first_continue_pass) { procs = ompi_proc_all(&num_procs); if(NULL == procs) { return OMPI_ERR_OUT_OF_RESOURCE; } } } else if(OPAL_CRS_RESTART_PRE == state ) { /* Nothing here */ } else if(OPAL_CRS_RESTART == state ) { procs = ompi_proc_all(&num_procs); if(NULL == procs) { return OMPI_ERR_OUT_OF_RESOURCE; } } else if(OPAL_CRS_TERM == state ) { ; } else { ; } /* Never call the ft_event functions attached to the BTLs on the second * pass of RESTART since on the first pass they were unloaded and therefore * no longer exist. */ if( OPAL_CRS_RESTART != state ) { if( OPAL_CRS_CONTINUE == state && !first_continue_pass ) { ; } else { /* Since we only ever call into the BTLs once during the first restart * pass, just lie to them on this pass for a bit of local clarity. */ if( OPAL_CRS_RESTART_PRE == state ) { loc_state = OPAL_CRS_RESTART; } else { loc_state = state; } /* * Call ft_event in: * - BTL modules * - MPool modules * * These should be cleaning out stale state, and memory references in * preparation for being shut down. */ for(btl_idx = 0; btl_idx < mca_bml_r2.num_btl_modules; btl_idx++) { /* * Notify Mpool */ if( NULL != (mca_bml_r2.btl_modules[btl_idx])->btl_mpool && NULL != (mca_bml_r2.btl_modules[btl_idx])->btl_mpool->mpool_ft_event ) { opal_output_verbose(10, ompi_cr_output, "bml:r2: ft_event: Notify the %s MPool.\n", (mca_bml_r2.btl_modules[btl_idx])->btl_mpool->mpool_component->mpool_version.mca_component_name); if(OMPI_SUCCESS != (ret = (mca_bml_r2.btl_modules[btl_idx])->btl_mpool->mpool_ft_event(loc_state) ) ) { continue; } } /* * Notify BTL */ if( NULL != (mca_bml_r2.btl_modules[btl_idx])->btl_ft_event) { opal_output_verbose(10, ompi_cr_output, "bml:r2: ft_event: Notify the %s BTL.\n", (mca_bml_r2.btl_modules[btl_idx])->btl_component->btl_version.mca_component_name); if(OMPI_SUCCESS != (ret = (mca_bml_r2.btl_modules[btl_idx])->btl_ft_event(loc_state) ) ) { continue; } } } } /* OPAL_CRS_CONTINUE == state && !first_continue_pass */ } if(OPAL_CRS_CHECKPOINT == state) { ; } else if(OPAL_CRS_CONTINUE == state) { /* Matches OPAL_CRS_RESTART_PRE */ if( ompi_cr_continue_like_restart && first_continue_pass) { if( OMPI_SUCCESS != (ret = mca_bml_r2_finalize()) ) { opal_output(0, "bml:r2: ft_event(Restart): Failed to finalize BML framework\n"); return ret; } } /* Matches OPAL_CRS_RESTART */ else if( ompi_cr_continue_like_restart && !first_continue_pass ) { /* * Barrier to make all processes have been successfully restarted before * we try to remove some restart only files. */ if (OMPI_SUCCESS != (ret = orte_grpcomm.barrier())) { opal_output(0, "bml:r2: ft_event(Restart): Failed in orte_grpcomm.barrier (%d)", ret); return ret; } opal_output_verbose(10, ompi_cr_output, "bml:r2: ft_event(Restart): Cleanup restart files\n"); opal_crs_base_cleanup_flush(); /* * Re-open the BTL framework to get the full list of components. */ if( OMPI_SUCCESS != (ret = mca_btl_base_open()) ) { opal_output(0, "bml:r2: ft_event(Restart): Failed to open BTL framework\n"); return ret; } /* * Re-select the BTL components/modules * This will cause the BTL components to discover the available * network options on this machine, and post proper modex informaiton. */ if( OMPI_SUCCESS != (ret = mca_btl_base_select(OMPI_ENABLE_PROGRESS_THREADS, OMPI_ENABLE_MPI_THREADS) ) ) { opal_output(0, "bml:r2: ft_event(Restart): Failed to select in BTL framework\n"); return ret; } /* * Clear some structures so we can properly repopulate them */ mca_bml_r2.btls_added = false; for(p = 0; p < (int)num_procs; ++p) { if( NULL != procs[p]->proc_bml) { OBJ_RELEASE(procs[p]->proc_bml); procs[p]->proc_bml = NULL; } OBJ_RELEASE(procs[p]); } if( NULL != procs ) { free(procs); procs = NULL; } } } else if(OPAL_CRS_RESTART_PRE == state ) { opal_output_verbose(10, ompi_cr_output, "bml:r2: ft_event(Restart): Finalize BML\n"); /* * Finalize the BML * - Flush progress functions * - Flush module references * - mca_btl_base_close() * Need to do this because we may have BTL components that were * unloaded in the first selection that may be available now. * Conversely we may have BTL components loaded now that * are not available now. */ if( OMPI_SUCCESS != (ret = mca_bml_r2_finalize()) ) { opal_output(0, "bml:r2: ft_event(Restart): Failed to finalize BML framework\n"); return ret; } } else if(OPAL_CRS_RESTART == state ) { /* * Barrier to make all processes have been successfully restarted before * we try to remove some restart only files. */ if (OMPI_SUCCESS != (ret = orte_grpcomm.barrier())) { opal_output(0, "bml:r2: ft_event(Restart): Failed in orte_grpcomm.barrier (%d)", ret); return ret; } opal_output_verbose(10, ompi_cr_output, "bml:r2: ft_event(Restart): Cleanup restart files\n"); opal_crs_base_cleanup_flush(); /* * Re-open the BTL framework to get the full list of components. * - but first clear the MCA value that was there */ param_type = mca_base_param_find("btl", NULL, NULL); mca_base_param_lookup_string(param_type, ¶m_list); opal_output_verbose(11, ompi_cr_output, "Restart (Previous BTL MCA): <%s>\n", param_list); if( NULL != param_list ) { free(param_list); param_list = NULL; } /* Deregister the old value, and refresh the file cache to grab any updates */ mca_base_param_deregister(param_type); mca_base_param_recache_files(false); if( OMPI_SUCCESS != (ret = mca_btl_base_open()) ) { opal_output(0, "bml:r2: ft_event(Restart): Failed to open BTL framework\n"); return ret; } param_type = mca_base_param_find("btl", NULL, NULL); mca_base_param_lookup_string(param_type, ¶m_list); opal_output_verbose(11, ompi_cr_output, "Restart (New BTL MCA): <%s>\n", param_list); if( NULL != param_list ) { free(param_list); param_list = NULL; } /* * Re-select the BTL components/modules * This will cause the BTL components to discover the available * network options on this machine, and post proper modex informaiton. */ if( OMPI_SUCCESS != (ret = mca_btl_base_select(OMPI_ENABLE_PROGRESS_THREADS, OMPI_ENABLE_MPI_THREADS) ) ) { opal_output(0, "bml:r2: ft_event(Restart): Failed to select in BTL framework\n"); return ret; } /* * Clear some structures so we can properly repopulate them */ mca_bml_r2.btls_added = false; for(p = 0; p < (int)num_procs; ++p) { if( NULL != procs[p]->proc_bml) { OBJ_RELEASE(procs[p]->proc_bml); procs[p]->proc_bml = NULL; } OBJ_RELEASE(procs[p]); } if( NULL != procs ) { free(procs); procs = NULL; } } else if(OPAL_CRS_TERM == state ) { ; } else { ; } return OMPI_SUCCESS; }