int main(int argc, char* argv[]) { int i, j; int index = 1; /* print usage if not enough arguments were given */ if (argc < 2) { printf("Usage: scr_rebuild_xor <size> <root> <missing_xor_filename> <ordered_remaining_xor_filenames>\n"); return 1; } /* TODO: want to pass this on command line? */ /* get current working directory */ char dsetdir[SCR_MAX_FILENAME]; scr_getcwd(dsetdir, sizeof(dsetdir)); /* create and reduce path for dataset */ scr_path* path_dset = scr_path_from_str(dsetdir); scr_path_reduce(path_dset); /* allocate buffers */ char* buffer_A = malloc(buffer_size * sizeof(char)); char* buffer_B = malloc(buffer_size * sizeof(char)); if (buffer_A == NULL || buffer_B == NULL) { scr_err("Failed to allocate buffer memory @ %s:%d", __FILE__, __LINE__ ); return 1; } /* read in the size of the XOR set */ int xor_set_size = (int) strtol(argv[index++], (char **)NULL, 10); if (xor_set_size <= 0) { scr_err("Invalid XOR set size argument %s @ %s:%d", argv[index-1], __FILE__, __LINE__ ); return 1; } /* allocate memory for data structures based on the XOR set size */ int* num_files = malloc(xor_set_size * sizeof(int)); int* offsets = malloc(xor_set_size * sizeof(int)); char** xor_files = malloc(xor_set_size * sizeof(char*)); int* xor_fds = malloc(xor_set_size * sizeof(int)); scr_hash** xor_headers = malloc(xor_set_size * sizeof(scr_hash*)); if (num_files == NULL || offsets == NULL || xor_files == NULL || xor_fds == NULL || xor_headers == NULL) { scr_err("Failed to allocate buffer memory @ %s:%d", __FILE__, __LINE__ ); return 1; } /* read in the rank of the missing process (the root) */ int root = (int) strtol(argv[index++], (char **)NULL, 10); if (root < 0 || root >= xor_set_size) { scr_err("Invalid root argument %s @ %s:%d", argv[index-1], __FILE__, __LINE__ ); return 1; } /* read in the missing xor filename */ xor_files[0] = strdup(argv[index++]); if (xor_files[0] == NULL) { scr_err("Failed to dup XOR filename @ %s:%d", __FILE__, __LINE__ ); return 1; } /* read in the xor filenames (expected to be in order of XOR segment number) */ /* we order ranks so that root is index 0, the rank to the right of root is index 1, and so on */ for (i=0; i < xor_set_size; i++) { xor_headers[i] = scr_hash_new(); /* we'll get the XOR file name for root from the header stored in the XOR file of the partner */ if (i == root) { continue; } /* adjust the index relative to root */ j = i - root; if (j < 0) { j += xor_set_size; } /* copy the XOR file name */ xor_files[j] = strdup(argv[index++]); if (xor_files[j] == NULL) { scr_err("Failed to dup XOR filename @ %s:%d", __FILE__, __LINE__ ); return 1; } } /* open each of the xor files and read in the headers */ for (i=1; i < xor_set_size; i++) { /* open each xor file for reading */ xor_fds[i] = scr_open(xor_files[i], O_RDONLY); if (xor_fds[i] < 0) { scr_err("Opening xor segment file: scr_open(%s) errno=%d %s @ %s:%d", xor_files[i], errno, strerror(errno), __FILE__, __LINE__ ); return 1; } /* read the header from this xor file */ if (scr_hash_read_fd(xor_files[i], xor_fds[i], xor_headers[i]) < 0) { scr_err("Failed to read XOR header from %s @ %s:%d", xor_files[i], __FILE__, __LINE__ ); return 1; } } /* build header for missing XOR file */ int partner_rank = -1; if (xor_set_size >= 2) { scr_hash_merge(xor_headers[0], xor_headers[1]); /* fetch our own file list from rank to our right */ scr_hash* rhs_hash = scr_hash_get(xor_headers[1], SCR_KEY_COPY_XOR_PARTNER); scr_hash* current_hash = scr_hash_new(); scr_hash_merge(current_hash, rhs_hash); scr_hash_set(xor_headers[0], SCR_KEY_COPY_XOR_CURRENT, current_hash); /* we are the partner to the rank to our left */ scr_hash* lhs_hash = scr_hash_get(xor_headers[xor_set_size-1], SCR_KEY_COPY_XOR_CURRENT); scr_hash* partner_hash = scr_hash_new(); scr_hash_merge(partner_hash, lhs_hash); scr_hash_set(xor_headers[0], SCR_KEY_COPY_XOR_PARTNER, partner_hash); /* get global rank of partner */ if (scr_hash_util_get_int(lhs_hash, SCR_KEY_COPY_XOR_RANK, &partner_rank) != SCR_SUCCESS) { scr_err("Failed to read partner rank from XOR file header in %s @ %s:%d", xor_files[xor_set_size-1], __FILE__, __LINE__ ); return 1; } } /* get a pointer to the current hash for the missing rank */ scr_hash* missing_current_hash = scr_hash_get(xor_headers[0], SCR_KEY_COPY_XOR_CURRENT); /* read the rank */ int my_rank = -1; if (scr_hash_util_get_int(missing_current_hash, SCR_KEY_COPY_XOR_RANK, &my_rank) != SCR_SUCCESS) { scr_err("Failed to read rank from XOR file header in %s @ %s:%d", xor_files[0], __FILE__, __LINE__ ); return 1; } /* get the dataset */ scr_dataset* dataset = scr_hash_get(xor_headers[0], SCR_KEY_COPY_XOR_DATASET); /* read the dataset id */ int dset_id = -1; if (scr_dataset_get_id(dataset, &dset_id) != SCR_SUCCESS) { scr_err("Failed to read dataset id from XOR file header in %s @ %s:%d", xor_files[0], __FILE__, __LINE__ ); return 1; } /* read the ranks */ int num_ranks = -1; if (scr_hash_util_get_int(xor_headers[0], SCR_KEY_COPY_XOR_RANKS, &num_ranks) != SCR_SUCCESS) { scr_err("Failed to read ranks from XOR file header in %s @ %s:%d", xor_files[0], __FILE__, __LINE__ ); return 1; } /* get name of partner's fmap */ scr_path* path_partner_map = scr_path_from_str(".scr"); scr_path_append_strf(path_partner_map, "fmap.%d.scr", partner_rank); /* extract partner's flush descriptor */ scr_hash* flushdesc = scr_hash_new(); scr_filemap* partner_map = scr_filemap_new(); scr_filemap_read(path_partner_map, partner_map); scr_filemap_get_flushdesc(partner_map, dset_id, partner_rank, flushdesc); scr_filemap_delete(&partner_map); /* delete partner map path */ scr_path_delete(&path_partner_map); /* determine whether we should preserve user directories */ int preserve_dirs = 0; scr_hash_util_get_int(flushdesc, SCR_SCAVENGE_KEY_PRESERVE, &preserve_dirs); /* read the chunk size */ unsigned long chunk_size = 0; if (scr_hash_util_get_unsigned_long(xor_headers[0], SCR_KEY_COPY_XOR_CHUNK, &chunk_size) != SCR_SUCCESS) { scr_err("Failed to read chunk size from XOR file header in %s @ %s:%d", xor_files[0], __FILE__, __LINE__ ); return 1; } /* determine number of files each member wrote in XOR set */ for (i=0; i < xor_set_size; i++) { /* record the number of files for this rank */ scr_hash* current_hash = scr_hash_get(xor_headers[i], SCR_KEY_COPY_XOR_CURRENT); if (scr_hash_util_get_int(current_hash, SCR_KEY_COPY_XOR_FILES, &num_files[i]) != SCR_SUCCESS) { scr_err("Failed to read number of files from %s @ %s:%d", xor_files[i], __FILE__, __LINE__ ); return 1; } } /* count the total number of files and set the offsets array */ int total_num_files = 0; for (i=0; i < xor_set_size; i++) { offsets[i] = total_num_files; total_num_files += num_files[i]; } /* allocate space for a file descriptor, file name pointer, and filesize for each user file */ int* user_fds = (int*) malloc(total_num_files * sizeof(int)); char** user_files = (char**) malloc(total_num_files * sizeof(char*)); char** user_rel_files = (char**) malloc(total_num_files * sizeof(char*)); unsigned long* user_filesizes = (unsigned long*) malloc(total_num_files * sizeof(unsigned long)); if (user_fds == NULL || user_files == NULL || user_rel_files == NULL || user_filesizes == NULL) { scr_err("Failed to allocate buffer memory @ %s:%d", __FILE__, __LINE__ ); return 1; } /* get file name, file size, and open each of the user files that we have */ for (i=0; i < xor_set_size; i++) { scr_hash* current_hash = scr_hash_get(xor_headers[i], SCR_KEY_COPY_XOR_CURRENT); /* for each file belonging to this rank, get filename, filesize, and open file */ for (j=0; j < num_files[i]; j++) { int offset = offsets[i] + j; /* get the meta data for this file */ scr_meta* meta = scr_hash_get_kv_int(current_hash, SCR_KEY_COPY_XOR_FILE, j); if (meta == NULL) { scr_err("Failed to read meta data for file %d in %s @ %s:%d", j, xor_files[i], __FILE__, __LINE__ ); return 1; } /* record the filesize of this file */ if (scr_meta_get_filesize(meta, &user_filesizes[offset]) != SCR_SUCCESS) { scr_err("Failed to read filesize field for file %d in %s @ %s:%d", j, xor_files[i], __FILE__, __LINE__ ); return 1; } /* get filename */ char* origname; if (scr_meta_get_origname(meta, &origname) != SCR_SUCCESS) { scr_err("Failed to read original name for file %d in %s @ %s:%d", j, xor_files[i], __FILE__, __LINE__ ); return 1; } /* construct full path to user file */ scr_path* path_user_full = scr_path_from_str(origname); if (preserve_dirs) { /* get original path of file */ char* origpath; if (scr_meta_get_origpath(meta, &origpath) != SCR_SUCCESS) { scr_err("Failed to read original path for file %d in %s @ %s:%d", j, xor_files[i], __FILE__, __LINE__ ); return 1; } /* construct full path to file */ scr_path_prepend_str(path_user_full, origpath); } else { /* construct full path to file */ scr_path_prepend(path_user_full, path_dset); } /* reduce path to user file */ scr_path_reduce(path_user_full); /* make a copy of the full path */ user_files[offset] = scr_path_strdup(path_user_full); /* make a copy of relative path */ scr_path* path_user_rel = scr_path_relative(path_dset, path_user_full); user_rel_files[offset] = scr_path_strdup(path_user_rel); scr_path_delete(&path_user_rel); /* free the full path */ scr_path_delete(&path_user_full); /* open the file */ if (i == 0) { /* create directory for file */ scr_path* user_dir_path = scr_path_from_str(user_files[offset]); scr_path_reduce(user_dir_path); scr_path_dirname(user_dir_path); if (! scr_path_is_null(user_dir_path)) { char* user_dir = scr_path_strdup(user_dir_path); mode_t mode_dir = scr_getmode(1, 1, 1); if (scr_mkdir(user_dir, mode_dir) != SCR_SUCCESS) { scr_err("Failed to create directory for user file %s @ %s:%d", user_dir, __FILE__, __LINE__ ); return 1; } scr_free(&user_dir); } scr_path_delete(&user_dir_path); /* open missing file for writing */ mode_t mode_file = scr_getmode(1, 1, 0); user_fds[offset] = scr_open(user_files[offset], O_WRONLY | O_CREAT | O_TRUNC, mode_file); if (user_fds[offset] < 0) { scr_err("Opening user file for writing: scr_open(%s) errno=%d %s @ %s:%d", user_files[offset], errno, strerror(errno), __FILE__, __LINE__ ); return 1; } } else { /* open existing file for reading */ user_fds[offset] = scr_open(user_files[offset], O_RDONLY); if (user_fds[offset] < 0) { scr_err("Opening user file for reading: scr_open(%s) errno=%d %s @ %s:%d", user_files[offset], errno, strerror(errno), __FILE__, __LINE__ ); return 1; } } } } /* finally, open the xor file for the missing rank */ mode_t mode_file = scr_getmode(1, 1, 0); xor_fds[0] = scr_open(xor_files[0], O_WRONLY | O_CREAT | O_TRUNC, mode_file); if (xor_fds[0] < 0) { scr_err("Opening xor file to be reconstructed: scr_open(%s) errno=%d %s @ %s:%d", xor_files[0], errno, strerror(errno), __FILE__, __LINE__ ); return 1; } int rc = 0; /* write the header to the XOR file of the missing rank */ if (scr_hash_write_fd(xor_files[0], xor_fds[0], xor_headers[0]) < 0) { rc = 1; } /* this offset array records the current position we are in the logical file for each rank */ unsigned long* offset = malloc(xor_set_size * sizeof(unsigned long)); if (offset == NULL) { scr_err("Failed to allocate buffer memory @ %s:%d", __FILE__, __LINE__ ); return 1; } for (i=0; i < xor_set_size; i++) { offset[i] = 0; } unsigned long write_pos = 0; int chunk_id; for (chunk_id = 0; chunk_id < xor_set_size && rc == 0; chunk_id++) { size_t nread = 0; while (nread < chunk_size && rc == 0) { /* read upto buffer_size bytes at a time */ size_t count = chunk_size - nread; if (count > buffer_size) { count = buffer_size; } /* clear our buffer */ memset(buffer_A, 0, count); /* read a segment from each rank and XOR it into our buffer */ for (i=1; i < xor_set_size; i++) { /* read the next set of bytes for this chunk from my file into send_buf */ if (chunk_id != ((i + root) % xor_set_size)) { /* read chunk from the logical file for this rank */ if (scr_read_pad_n(num_files[i], &user_files[offsets[i]], &user_fds[offsets[i]], buffer_B, count, offset[i], &user_filesizes[offsets[i]]) != SCR_SUCCESS) { /* our read failed, set the return code to an error */ rc = 1; count = 0; } offset[i] += count; } else { /* read chunk from the XOR file for this rank */ if (scr_read_attempt(xor_files[i], xor_fds[i], buffer_B, count) != count) { /* our read failed, set the return code to an error */ rc = 1; count = 0; } } /* TODO: XORing with unsigned long would be faster here (if chunk size is multiple of this size) */ /* merge the blocks via xor operation */ for (j = 0; j < count; j++) { buffer_A[j] ^= buffer_B[j]; } } /* at this point, we have the data from the missing rank, write it out */ if (chunk_id != root) { /* write chunk to logical file for the missing rank */ if (scr_write_pad_n(num_files[0], &user_files[0], &user_fds[0], buffer_A, count, write_pos, &user_filesizes[0]) != SCR_SUCCESS) { /* our write failed, set the return code to an error */ rc = 1; } write_pos += count; } else { /* write chunk to xor file for the missing rank */ if (scr_write_attempt(xor_files[0], xor_fds[0], buffer_A, count) != count) { /* our write failed, set the return code to an error */ rc = 1; } } nread += count; } } /* close each of the user files */ for (i=0; i < total_num_files; i++) { if (scr_close(user_files[i], user_fds[i]) != SCR_SUCCESS) { rc = 1; } } /* close each of the XOR files */ for (i=0; i < xor_set_size; i++) { if (scr_close(xor_files[i], xor_fds[i]) != SCR_SUCCESS) { rc = 1; } } /* if the write failed, delete the files we just wrote, and return an error */ if (rc != 0) { for (j=0; j < num_files[0]; j++) { scr_file_unlink(user_files[j]); } scr_file_unlink(xor_files[0]); return 1; } /* check that filesizes are correct */ unsigned long filesize; for (j=0; j < num_files[0]; j++) { filesize = scr_file_size(user_files[j]); if (filesize != user_filesizes[j]) { /* the filesize check failed, so delete the file */ scr_file_unlink(user_files[j]); /* mark the file as incomplete */ scr_meta* meta = scr_hash_get_kv_int(missing_current_hash, SCR_KEY_COPY_XOR_FILE, j); scr_meta_set_complete(meta, 0); rc = 1; } } /* TODO: we didn't record the filesize of the XOR file for the missing rank anywhere */ /* create a filemap for this rank */ scr_filemap* map = scr_filemap_new(); if (map == NULL) { scr_err("Failed to allocate filemap @ %s:%d", __FILE__, __LINE__ ); return 1; } /* record the dataset information in the filemap */ scr_filemap_set_dataset(map, dset_id, my_rank, dataset); /* write meta data for each of the user files and add each one to the filemap */ for (j=0; j < num_files[0]; j++) { /* add user file to filemap and record meta data */ char* user_file_relative = user_rel_files[j]; scr_filemap_add_file(map, dset_id, my_rank, user_file_relative); scr_meta* meta = scr_hash_get_kv_int(missing_current_hash, SCR_KEY_COPY_XOR_FILE, j); scr_filemap_set_meta(map, dset_id, my_rank, user_file_relative, meta); } /* write meta data for xor file and add it to the filemap */ scr_filemap_add_file(map, dset_id, my_rank, xor_files[0]); unsigned long full_chunk_filesize = scr_file_size(xor_files[0]); int missing_complete = 1; scr_meta* meta_chunk = scr_meta_new(); scr_meta_set_filename(meta_chunk, xor_files[0]); scr_meta_set_filetype(meta_chunk, SCR_META_FILE_XOR); scr_meta_set_filesize(meta_chunk, full_chunk_filesize); /* TODO: remove this from meta file, for now it's needed in scr_index.c */ scr_meta_set_ranks(meta_chunk, num_ranks); scr_meta_set_complete(meta_chunk, missing_complete); scr_filemap_set_meta(map, dset_id, my_rank, xor_files[0], meta_chunk); /* set expected number of files for the missing rank */ int expected_num_files = scr_filemap_num_files(map, dset_id, my_rank); scr_filemap_set_expected_files(map, dset_id, my_rank, expected_num_files); /* compute, check, and store crc values with files */ for (j=0; j < num_files[0]; j++) { /* compute crc on user file */ char* user_file_relative = user_rel_files[j]; if (scr_compute_crc(map, dset_id, my_rank, user_file_relative) != SCR_SUCCESS) { /* the crc check failed, so delete the file */ scr_file_unlink(user_files[j]); rc = 1; } } if (scr_compute_crc(map, dset_id, my_rank, xor_files[0]) != SCR_SUCCESS) { /* the crc check failed, so delete the file */ scr_file_unlink(xor_files[0]); rc = 1; } /* store flush descriptor */ scr_filemap_set_flushdesc(map, dset_id, my_rank, flushdesc); /* write filemap for this rank */ scr_path* path_map = scr_path_from_str(".scr"); scr_path_append_strf(path_map, "fmap.%d.scr", my_rank); if (scr_filemap_write(path_map, map) != SCR_SUCCESS) { rc = 1; } scr_path_delete(&path_map); /* delete the map */ scr_filemap_delete(&map); scr_meta_delete(&meta_chunk); /* delete the flush/scavenge descriptor */ scr_hash_delete(&flushdesc); scr_free(&offset); for (i=0; i < total_num_files; i++) { scr_free(&user_rel_files[i]); scr_free(&user_files[i]); } scr_free(&user_filesizes); scr_free(&user_rel_files); scr_free(&user_files); scr_free(&user_fds); for (i=0; i < xor_set_size; i++) { scr_hash_delete(&xor_headers[i]); } for (i=0; i < xor_set_size; i++) { scr_free(&xor_files[i]); } scr_free(&xor_headers); scr_free(&xor_fds); scr_free(&xor_files); scr_free(&offsets); scr_free(&num_files); scr_free(&buffer_B); scr_free(&buffer_A); scr_path_delete(&path_dset); return rc; }
/* flush files specified in list, and record corresponding entries for summary file */ static int scr_flush_files_list(scr_hash* file_list, scr_hash* summary) { /* assume we will succeed in this flush */ int rc = SCR_SUCCESS; /* flush each of my files and fill in summary data structure */ scr_hash_elem* elem = NULL; scr_hash* files = scr_hash_get(file_list, SCR_KEY_FILE); for (elem = scr_hash_elem_first(files); elem != NULL; elem = scr_hash_elem_next(elem)) { /* get the filename */ char* file = scr_hash_elem_key(elem); /* convert file to path and extract name of file */ scr_path* path_name = scr_path_from_str(file); scr_path_basename(path_name); /* get the hash for this element */ scr_hash* hash = scr_hash_elem_hash(elem); /* get meta data for this file */ scr_meta* meta = scr_hash_get(hash, SCR_KEY_META); /* if segments are defined, we flush the file to its containers, * otherwise we copy the file out as is */ scr_hash* segments = scr_hash_get(hash, SCR_SUMMARY_6_KEY_SEGMENT); if (segments != NULL) { /* TODO: PRESERVE get original filename here */ /* add this file to the summary file */ char* name = scr_path_strdup(path_name); scr_hash* file_hash = scr_hash_set_kv(summary, SCR_SUMMARY_6_KEY_FILE, name); scr_free(&name); // USERDEF fixme! /* flush the file to the containers listed in its segmenets */ if (scr_flush_file_to_containers(file, meta, segments, scr_prefix) == SCR_SUCCESS) { /* successfully flushed this file, record the filesize */ unsigned long filesize = 0; if (scr_meta_get_filesize(meta, &filesize) == SCR_SUCCESS) { scr_hash_util_set_bytecount(file_hash, SCR_SUMMARY_6_KEY_SIZE, filesize); } /* record the crc32 if one was computed */ uLong crc = 0; if (scr_meta_get_crc32(meta, &crc) == SCR_SUCCESS) { scr_hash_util_set_crc32(file_hash, SCR_SUMMARY_6_KEY_CRC, crc); } /* record segment information in summary file */ scr_hash* segments_copy = scr_hash_new(); scr_hash_merge(segments_copy, segments); scr_hash_set(file_hash, SCR_SUMMARY_6_KEY_SEGMENT, segments_copy); } else { /* the flush failed */ rc = SCR_FAILURE; /* explicitly mark file as incomplete */ scr_hash_set_kv_int(file_hash, SCR_SUMMARY_6_KEY_COMPLETE, 0); } } else { /* get directory to flush file to */ char* dir; if (scr_hash_util_get_str(hash, SCR_KEY_PATH, &dir) == SCR_SUCCESS) { /* create full path of destination file */ scr_path* path_full = scr_path_from_str(dir); scr_path_append(path_full, path_name); /* get relative path to flushed file from SCR_PREFIX directory */ scr_path* path_relative = scr_path_relative(scr_prefix_path, path_full); if (! scr_path_is_null(path_relative)) { /* record the name of the file in the summary hash, and get reference to a hash for this file */ char* name = scr_path_strdup(path_relative); scr_hash* file_hash = scr_hash_set_kv(summary, SCR_SUMMARY_6_KEY_FILE, name); scr_free(&name); /* flush the file and fill in the meta data for this file */ if (scr_flush_a_file(file, dir, meta) == SCR_SUCCESS) { /* successfully flushed this file, record the filesize */ unsigned long filesize = 0; if (scr_meta_get_filesize(meta, &filesize) == SCR_SUCCESS) { scr_hash_util_set_bytecount(file_hash, SCR_SUMMARY_6_KEY_SIZE, filesize); } /* record the crc32 if one was computed */ uLong crc = 0; if (scr_meta_get_crc32(meta, &crc) == SCR_SUCCESS) { scr_hash_util_set_crc32(file_hash, SCR_SUMMARY_6_KEY_CRC, crc); } } else { /* the flush failed */ rc = SCR_FAILURE; /* explicitly mark incomplete files */ scr_hash_set_kv_int(file_hash, SCR_SUMMARY_6_KEY_COMPLETE, 0); } } else { scr_abort(-1, "Failed to get relative path to directory %s from %s @ %s:%d", dir, scr_prefix, __FILE__, __LINE__ ); } /* free relative and full paths */ scr_path_delete(&path_relative); scr_path_delete(&path_full); } else { scr_abort(-1, "Failed to read directory to flush file to @ %s:%d", __FILE__, __LINE__ ); } } /* free the file name path */ scr_path_delete(&path_name); } return rc; }
/* attempt to fetch most recent checkpoint from prefix directory into * cache, fills in map if successful and sets fetch_attempted to 1 if * any fetch is attempted, returns SCR_SUCCESS if successful */ int scr_fetch_sync(scr_filemap* map, int* fetch_attempted) { /* we only return success if we successfully fetch a checkpoint */ int rc = SCR_FAILURE; double time_start, time_end, time_diff; /* start timer */ if (scr_my_rank_world == 0) { time_start = MPI_Wtime(); } /* have rank 0 read the index file */ scr_hash* index_hash = NULL; int read_index_file = 0; if (scr_my_rank_world == 0) { /* create an empty hash to store our index */ index_hash = scr_hash_new(); /* read the index file */ if (scr_index_read(scr_prefix_path, index_hash) == SCR_SUCCESS) { read_index_file = 1; } } /* don't enter while loop below if rank 0 failed to read index file */ int continue_fetching = 1; MPI_Bcast(&read_index_file, 1, MPI_INT, 0, scr_comm_world); if (! read_index_file) { continue_fetching = 0; } /* now start fetching, we keep trying until we exhaust all valid * checkpoints */ char target[SCR_MAX_FILENAME]; int current_checkpoint_id = -1; while (continue_fetching) { /* create a new path */ scr_path* fetch_path = scr_path_new(); /* initialize our target directory to empty string */ strcpy(target, ""); /* rank 0 determines the directory to fetch from */ if (scr_my_rank_world == 0) { /* read the current directory if it's set */ char* current_str; if (scr_index_get_current(index_hash, ¤t_str) == SCR_SUCCESS) { size_t current_str_len = strlen(current_str) + 1; if (current_str_len <= sizeof(target)) { strcpy(target, current_str); } else { /* ERROR */ } } /* lookup the checkpoint id */ int next_checkpoint_id = -1; if (strcmp(target, "") != 0) { /* we have a subdirectory name, lookup the checkpoint id * corresponding to this directory */ scr_index_get_id_by_dir(index_hash, target, &next_checkpoint_id); } else { /* otherwise, just get the most recent complete checkpoint * (that's older than the current id) */ scr_index_get_most_recent_complete(index_hash, current_checkpoint_id, &next_checkpoint_id, target); } current_checkpoint_id = next_checkpoint_id; /* TODODSET: need to verify that dataset is really a checkpoint * and keep searching if not */ /* if we have a subdirectory (target) name, build the full fetch * directory */ if (strcmp(target, "") != 0) { /* record that we're attempting a fetch of this checkpoint in * the index file */ *fetch_attempted = 1; if (current_checkpoint_id != -1) { scr_index_mark_fetched(index_hash, current_checkpoint_id, target); scr_index_write(scr_prefix_path, index_hash); } /* we have a subdirectory, now build the full path */ scr_path_append(fetch_path, scr_prefix_path); scr_path_append_str(fetch_path, target); scr_path_reduce(fetch_path); } } /* broadcast fetch path from rank 0 */ scr_path_bcast(fetch_path, 0, scr_comm_world); /* check whether we've got a path */ if (! scr_path_is_null(fetch_path)) { /* got something, attempt to fetch the checkpoint */ int dset_id, ckpt_id; rc = scr_fetch_files(map, fetch_path, &dset_id, &ckpt_id); if (rc == SCR_SUCCESS) { /* set the dataset and checkpoint ids */ scr_dataset_id = dset_id; scr_checkpoint_id = ckpt_id; /* we succeeded in fetching this checkpoint, set current to * point to it, and stop fetching */ if (scr_my_rank_world == 0) { scr_index_set_current(index_hash, target); scr_index_write(scr_prefix_path, index_hash); } continue_fetching = 0; } else { /* we tried to fetch, but we failed, mark it as failed in * the index file so we don't try it again */ if (scr_my_rank_world == 0) { /* unset the current pointer */ scr_index_unset_current(index_hash); if (current_checkpoint_id != -1 && strcmp(target, "") != 0) { scr_index_mark_failed(index_hash, current_checkpoint_id, target); } scr_index_write(scr_prefix_path, index_hash); } } } else { /* we ran out of valid checkpoints in the index file, * bail out of the loop */ continue_fetching = 0; } /* free fetch path */ scr_path_delete(&fetch_path); } /* delete the index hash */ if (scr_my_rank_world == 0) { scr_hash_delete(&index_hash); } /* broadcast whether we actually attempted to fetch anything * (only rank 0 knows) */ MPI_Bcast(fetch_attempted, 1, MPI_INT, 0, scr_comm_world); /* stop timer for fetch */ if (scr_my_rank_world == 0) { time_end = MPI_Wtime(); time_diff = time_end - time_start; scr_dbg(1, "scr_fetch_files: return code %d, %f secs", rc, time_diff); } return rc; }