/* broadcast dataset hash from smallest rank we can find that has a copy */ static int scr_distribute_datasets(scr_filemap* map, int id) { int i; /* create a new hash to record dataset descriptor */ scr_hash* send_hash = scr_hash_new(); /* for this dataset, get list of ranks we have data for */ int nranks = 0; int* ranks = NULL; scr_filemap_list_ranks_by_dataset(map, id, &nranks, &ranks); /* for each rank we have files for, * check whether we also have its dataset descriptor */ int invalid_rank_found = 0; int have_dset = 0; for (i=0; i < nranks; i++) { /* get the rank id */ int rank = ranks[i]; /* check that the rank is within range */ if (rank < 0 || rank >= scr_ranks_world) { scr_err("Invalid rank id %d in world of %d @ %s:%d", rank, scr_ranks_world, __FILE__, __LINE__ ); invalid_rank_found = 1; } /* lookup the dataset descriptor hash for this rank */ scr_hash* desc = scr_hash_new(); scr_filemap_get_dataset(map, id, rank, desc); /* if this descriptor has entries, add it to our send hash, * delete the hash otherwise */ if (scr_hash_size(desc) > 0) { have_dset = 1; scr_hash_merge(send_hash, desc); scr_hash_delete(&desc); break; } else { scr_hash_delete(&desc); } } /* free off our list of ranks */ scr_free(&ranks); /* check that we didn't find an invalid rank on any process */ if (! scr_alltrue(invalid_rank_found == 0)) { scr_hash_delete(&send_hash); return SCR_FAILURE; } /* identify the smallest rank that has the dataset */ int source_rank = scr_ranks_world; if (have_dset) { source_rank = scr_my_rank_world; } int min_rank; MPI_Allreduce(&source_rank, &min_rank, 1, MPI_INT, MPI_MIN, scr_comm_world); /* if there is no rank, return with failure */ if (min_rank >= scr_ranks_world) { scr_hash_delete(&send_hash); return SCR_FAILURE; } /* otherwise, bcast the dataset from the minimum rank */ if (scr_my_rank_world != min_rank) { scr_hash_unset_all(send_hash); } scr_hash_bcast(send_hash, min_rank, scr_comm_world); /* record the descriptor in our filemap */ scr_filemap_set_dataset(map, id, scr_my_rank_world, send_hash); scr_filemap_write(scr_map_file, map); /* TODO: at this point, we could delete descriptors for other * ranks for this checkpoint */ /* free off our send hash */ scr_hash_delete(&send_hash); return SCR_SUCCESS; }
/* read contents of summary file */ static int scr_fetch_summary( const char* summary_dir, scr_hash* file_list) { /* assume that we won't succeed in our fetch attempt */ int rc = SCR_SUCCESS; /* check whether summary file exists and is readable */ if (scr_my_rank_world == 0) { /* check that we can access the directory */ if (scr_file_is_readable(summary_dir) != SCR_SUCCESS) { scr_err("Failed to access summary directory %s @ %s:%d", summary_dir, __FILE__, __LINE__ ); rc = SCR_FAILURE; } } /* broadcast success code from rank 0 */ MPI_Bcast(&rc, 1, MPI_INT, 0, scr_comm_world); if (rc != SCR_SUCCESS) { return rc; } /* add path to file list */ scr_hash_util_set_str(file_list, SCR_KEY_PATH, summary_dir); /* build path to summary file */ scr_path* dataset_path = scr_path_from_str(summary_dir); scr_path* meta_path = scr_path_dup(dataset_path); scr_path_append_str(meta_path, ".scr"); scr_path_reduce(meta_path); /* rank 0 reads the summary file */ scr_hash* header = scr_hash_new(); if (scr_my_rank_world == 0) { /* build path to summary file */ scr_path* summary_path = scr_path_dup(meta_path); scr_path_append_str(summary_path, "summary.scr"); const char* summary_file = scr_path_strdup(summary_path); /* open file for reading */ int fd = scr_open(summary_file, O_RDONLY); if (fd >= 0) { /* read summary hash */ ssize_t header_size = scr_hash_read_fd(summary_file, fd, header); if (header_size < 0) { rc = SCR_FAILURE; } /* TODO: check that the version is correct */ /* close the file */ scr_close(summary_file, fd); } else { scr_err("Failed to open summary file %s @ %s:%d", summary_file, __FILE__, __LINE__ ); rc = SCR_FAILURE; } /* free summary path and string */ scr_free(&summary_file); scr_path_delete(&summary_path); } /* broadcast success code from rank 0 */ MPI_Bcast(&rc, 1, MPI_INT, 0, scr_comm_world); if (rc != SCR_SUCCESS) { goto cleanup; } /* broadcast the summary hash */ scr_hash_bcast(header, 0, scr_comm_world); /* extract and record the datast in file list */ scr_hash* dataset_hash = scr_hash_new(); scr_dataset* dataset = scr_hash_get(header, SCR_SUMMARY_6_KEY_DATASET); scr_hash_merge(dataset_hash, dataset); scr_hash_set(file_list, SCR_SUMMARY_6_KEY_DATASET, dataset_hash); /* build path to rank2file map */ scr_path* rank2file_path = scr_path_dup(meta_path); scr_path_append_str(rank2file_path, "rank2file.scr"); /* fetch file names and offsets containing file hash data */ int valid = 0; char* file = NULL; unsigned long offset = 0; if (scr_my_rank_world == 0) { /* rank 0 is only valid reader to start with */ valid = 1; file = scr_path_strdup(rank2file_path); offset = 0; } if (scr_fetch_rank2file_map(dataset_path, 1, &valid, &file, &offset) != SCR_SUCCESS) { rc = SCR_FAILURE; } /* create hashes to exchange data */ scr_hash* send = scr_hash_new(); scr_hash* recv = scr_hash_new(); /* read data from file */ if (valid) { /* open file if necessary */ int fd = scr_open(file, O_RDONLY); if (fd >= 0) { /* create hash to hold file contents */ scr_hash* save = scr_hash_new(); /* read hash from file */ scr_lseek(file, fd, offset, SEEK_SET); ssize_t readsize = scr_hash_read_fd(file, fd, save); if (readsize < 0) { scr_err("Failed to read rank2file map file %s @ %s:%d", file, __FILE__, __LINE__ ); rc = SCR_FAILURE; } /* check that the number of ranks match */ int ranks = 0; scr_hash_util_get_int(save, SCR_SUMMARY_6_KEY_RANKS, &ranks); if (ranks != scr_ranks_world) { scr_err("Invalid number of ranks in %s, got %d expected %d @ %s:%d", file, ranks, scr_ranks_world, __FILE__, __LINE__ ); rc = SCR_FAILURE; } /* delete current send hash, set it to values from file, * delete file hash */ scr_hash_delete(&send); send = scr_hash_extract(save, SCR_SUMMARY_6_KEY_RANK); scr_hash_delete(&save); /* close the file */ scr_close(file, fd); } else { scr_err("Failed to open rank2file map %s @ %s:%d", file, __FILE__, __LINE__ ); rc = SCR_FAILURE; } /* delete file name string */ scr_free(&file); } /* check that everyone read the data ok */ if (! scr_alltrue(rc == SCR_SUCCESS)) { rc = SCR_FAILURE; goto cleanup_hashes; } /* scatter to groups */ scr_hash_exchange_direction(send, recv, scr_comm_world, SCR_HASH_EXCHANGE_RIGHT); /* iterate over the ranks that sent data to us, and set up our * list of files */ scr_hash_elem* elem; for (elem = scr_hash_elem_first(recv); elem != NULL; elem = scr_hash_elem_next(elem)) { /* the key is the source rank, which we don't care about, * the info we need is in the element hash */ scr_hash* elem_hash = scr_hash_elem_hash(elem); /* get pointer to file hash */ scr_hash* file_hash = scr_hash_get(elem_hash, SCR_SUMMARY_6_KEY_FILE); if (file_hash != NULL) { /* TODO: parse summary file format */ scr_hash_merge(file_list, elem_hash); } else { rc = SCR_FAILURE; } } /* fill in file list parameters */ if (rc == SCR_SUCCESS) { /* if we're not using containers, add PATH entry for each of our * files */ scr_hash* files = scr_hash_get(file_list, SCR_KEY_FILE); for (elem = scr_hash_elem_first(files); elem != NULL; elem = scr_hash_elem_next(elem)) { /* get the file name */ char* file = scr_hash_elem_key(elem); /* combine the file name with the summary directory to build a * full path to the file */ scr_path* path_full = scr_path_dup(dataset_path); scr_path_append_str(path_full, file); /* subtract off last component to get just the path */ scr_path_dirname(path_full); char* path = scr_path_strdup(path_full); /* record path in file list */ scr_hash* hash = scr_hash_elem_hash(elem); scr_hash_util_set_str(hash, SCR_KEY_PATH, path); /* free the path and string */ scr_free(&path); scr_path_delete(&path_full); } } /* check that everyone read the data ok */ if (! scr_alltrue(rc == SCR_SUCCESS)) { rc = SCR_FAILURE; goto cleanup_hashes; } cleanup_hashes: /* delete send and receive hashes */ scr_hash_delete(&recv); scr_hash_delete(&send); /* free string and path for rank2file map */ scr_path_delete(&rank2file_path); cleanup: /* free the header hash */ scr_hash_delete(&header); /* free path for dataset directory */ scr_path_delete(&meta_path); scr_path_delete(&dataset_path); return rc; }