/* this moves all files of the specified dataset in the cache to * make them accessible to new rank mapping */ static int scr_distribute_files(scr_filemap* map, const scr_reddesc* red, int id) { int i, round; int rc = SCR_SUCCESS; /* TODO: mark dataset as being distributed in filemap, * because if we fail in the middle of a distribute, * we can't trust the contents of the files anymore, * at which point it should be deleted */ /* clean out any incomplete files before we start */ scr_cache_clean(map); /* for this dataset, get list of ranks we have data for */ int nranks = 0; int* ranks = NULL; scr_filemap_list_ranks_by_dataset(map, id, &nranks, &ranks); /* walk backwards through the list of ranks, and set our start index * to the rank which is the first rank that is equal to or higher * than our own rank -- when we assign round ids below, this offsetting * helps distribute the load */ int start_index = 0; int invalid_rank_found = 0; for (i = nranks-1; i >= 0; i--) { int rank = ranks[i]; /* pick the first rank whose rank id is equal to or higher than our own */ if (rank >= scr_my_rank_world) { start_index = i; } /* while we're at it, check that the rank is within range */ if (rank < 0 || rank >= scr_ranks_world) { scr_err("Invalid rank id %d in world of %d @ %s:%d", rank, scr_ranks_world, __FILE__, __LINE__ ); invalid_rank_found = 1; } } /* check that we didn't find an invalid rank on any process */ if (! scr_alltrue(invalid_rank_found == 0)) { scr_free(&ranks); return SCR_FAILURE; } /* allocate array to record the rank we can send to in each round */ int* have_rank_by_round = (int*) SCR_MALLOC(sizeof(int) * nranks); int* send_flag_by_round = (int*) SCR_MALLOC(sizeof(int) * nranks); /* check that we have all of the files for each rank, * and determine the round we can send them */ scr_hash* send_hash = scr_hash_new(); scr_hash* recv_hash = scr_hash_new(); for (round = 0; round < nranks; round++) { /* get the rank id */ int index = (start_index + round) % nranks; int rank = ranks[index]; /* record the rank indexed by the round number */ have_rank_by_round[round] = rank; /* assume we won't be sending to this rank in this round */ send_flag_by_round[round] = 0; /* if we have files for this rank, specify the round we can * send those files in */ if (scr_bool_have_files(map, id, rank)) { scr_hash_setf(send_hash, NULL, "%d %d", rank, round); } } scr_hash_exchange(send_hash, recv_hash, scr_comm_world); /* search for the minimum round we can get our files */ int retrieve_rank = -1; int retrieve_round = -1; scr_hash_elem* elem = NULL; for (elem = scr_hash_elem_first(recv_hash); elem != NULL; elem = scr_hash_elem_next(elem)) { /* get the rank id */ int rank = scr_hash_elem_key_int(elem); /* get the round id */ scr_hash* round_hash = scr_hash_elem_hash(elem); scr_hash_elem* round_elem = scr_hash_elem_first(round_hash); char* round_str = scr_hash_elem_key(round_elem); int round = atoi(round_str); /* record this round and rank number if it's less than the current round */ if (round < retrieve_round || retrieve_round == -1) { retrieve_round = round; retrieve_rank = rank; } } /* done with the round hashes, free them off */ scr_hash_delete(&recv_hash); scr_hash_delete(&send_hash); /* free off our list of ranks */ scr_free(&ranks); /* for some redundancy schemes, we know at this point whether we * can recover all files */ int can_get_files = (retrieve_rank != -1); if (red->copy_type != SCR_COPY_XOR && !scr_alltrue(can_get_files)) { /* print a debug message indicating which rank is missing files */ if (! can_get_files) { scr_dbg(2, "Cannot find process that has my checkpoint files @ %s:%d", __FILE__, __LINE__ ); } return SCR_FAILURE; } /* get the maximum retrieve round */ int max_rounds = 0; MPI_Allreduce( &retrieve_round, &max_rounds, 1, MPI_INT, MPI_MAX, scr_comm_world ); /* tell destination which round we'll take our files in */ send_hash = scr_hash_new(); recv_hash = scr_hash_new(); if (retrieve_rank != -1) { scr_hash_setf(send_hash, NULL, "%d %d", retrieve_rank, retrieve_round); } scr_hash_exchange(send_hash, recv_hash, scr_comm_world); /* determine which ranks want to fetch their files from us */ for(elem = scr_hash_elem_first(recv_hash); elem != NULL; elem = scr_hash_elem_next(elem)) { /* get the round id */ scr_hash* round_hash = scr_hash_elem_hash(elem); scr_hash_elem* round_elem = scr_hash_elem_first(round_hash); char* round_str = scr_hash_elem_key(round_elem); int round = atoi(round_str); /* record whether this rank wants its files from us */ if (round >= 0 && round < nranks) { send_flag_by_round[round] = 1; } } /* done with the round hashes, free them off */ scr_hash_delete(&recv_hash); scr_hash_delete(&send_hash); int tmp_rc = 0; /* run through rounds and exchange files */ for (round = 0; round <= max_rounds; round++) { /* assume we don't need to send or receive any files this round */ int send_rank = MPI_PROC_NULL; int recv_rank = MPI_PROC_NULL; int send_num = 0; int recv_num = 0; /* check whether I can potentially send to anyone in this round */ if (round < nranks) { /* have someone's files, check whether they are asking * for them this round */ if (send_flag_by_round[round]) { /* need to send files this round, remember to whom and how many */ int dst_rank = have_rank_by_round[round]; send_rank = dst_rank; send_num = scr_filemap_num_files(map, id, dst_rank); } } /* if I'm supposed to get my files this round, set the recv_rank */ if (retrieve_round == round) { recv_rank = retrieve_rank; } /* TODO: another special case is to just move files if the * processes are on the same node */ /* if i'm sending to myself, just move (rename) each file */ if (send_rank == scr_my_rank_world) { /* get our file list */ int numfiles = 0; char** files = NULL; scr_filemap_list_files(map, id, send_rank, &numfiles, &files); /* TODO: sort files in reverse order by size */ /* iterate over and rename each file */ for (i=0; i < numfiles; i++) { /* get the current file name */ char* file = files[i]; /* lookup meta data for this file */ scr_meta* meta = scr_meta_new(); scr_filemap_get_meta(map, id, send_rank, file, meta); /* get the path for this file based on its type * and dataset id */ char* dir = NULL; if (scr_meta_check_filetype(meta, SCR_META_FILE_USER) == SCR_SUCCESS) { dir = scr_cache_dir_get(red, id); } else { dir = scr_cache_dir_hidden_get(red, id); } /* build the new file name */ scr_path* path_newfile = scr_path_from_str(file); scr_path_basename(path_newfile); scr_path_prepend_str(path_newfile, dir); char* newfile = scr_path_strdup(path_newfile); /* if the new file name is different from the old name, rename it */ if (strcmp(file, newfile) != 0) { /* record the new filename to our map and write it to disk */ scr_filemap_add_file(map, id, send_rank, newfile); scr_filemap_set_meta(map, id, send_rank, newfile, meta); scr_filemap_write(scr_map_file, map); /* rename the file */ scr_dbg(2, "Round %d: rename(%s, %s)", round, file, newfile); tmp_rc = rename(file, newfile); if (tmp_rc != 0) { /* TODO: to cross mount points, if tmp_rc == EXDEV, * open new file, copy, and delete orig */ scr_err("Moving checkpoint file: rename(%s, %s) %s errno=%d @ %s:%d", file, newfile, strerror(errno), errno, __FILE__, __LINE__ ); rc = SCR_FAILURE; } /* remove the old name from the filemap and write it to disk */ scr_filemap_remove_file(map, id, send_rank, file); scr_filemap_write(scr_map_file, map); } /* free the path and string */ scr_free(&newfile); scr_path_delete(&path_newfile); /* free directory string */ scr_free(&dir); /* free meta data */ scr_meta_delete(&meta); } /* free the list of filename pointers */ scr_free(&files); } else { /* if we have files for this round, but the correspdonding * rank doesn't need them, delete the files */ if (round < nranks && send_rank == MPI_PROC_NULL) { int dst_rank = have_rank_by_round[round]; scr_unlink_rank(map, id, dst_rank); } /* sending to and/or recieving from another node */ if (send_rank != MPI_PROC_NULL || recv_rank != MPI_PROC_NULL) { /* have someone to send to or receive from */ int have_outgoing = 0; int have_incoming = 0; if (send_rank != MPI_PROC_NULL) { have_outgoing = 1; } if (recv_rank != MPI_PROC_NULL) { have_incoming = 1; } /* first, determine how many files I will be receiving and * tell how many I will be sending */ MPI_Request request[2]; MPI_Status status[2]; int num_req = 0; if (have_incoming) { MPI_Irecv( &recv_num, 1, MPI_INT, recv_rank, 0, scr_comm_world, &request[num_req] ); num_req++; } if (have_outgoing) { MPI_Isend( &send_num, 1, MPI_INT, send_rank, 0, scr_comm_world, &request[num_req] ); num_req++; } if (num_req > 0) { MPI_Waitall(num_req, request, status); } /* record how many files I will receive (need to distinguish * between 0 files and not knowing) */ if (have_incoming) { scr_filemap_set_expected_files(map, id, scr_my_rank_world, recv_num); } /* turn off send or receive flags if the file count is 0, * nothing else to do */ if (send_num == 0) { have_outgoing = 0; send_rank = MPI_PROC_NULL; } if (recv_num == 0) { have_incoming = 0; recv_rank = MPI_PROC_NULL; } /* TODO: since we overwrite files in place in order to avoid * running out of storage space, we should sort files in order * of descending size for the next step */ /* get our file list for the destination */ int numfiles = 0; char** files = NULL; if (have_outgoing) { scr_filemap_list_files(map, id, send_rank, &numfiles, &files); } /* while we have a file to send or receive ... */ while (have_incoming || have_outgoing) { /* get the filename */ char* file = NULL; scr_meta* send_meta = NULL; if (have_outgoing) { file = files[numfiles - send_num]; send_meta = scr_meta_new(); scr_filemap_get_meta(map, id, send_rank, file, send_meta); } /* exchange meta data so we can determine type of incoming file */ scr_meta* recv_meta = scr_meta_new(); scr_hash_sendrecv(send_meta, send_rank, recv_meta, recv_rank, scr_comm_world); /* get the path for this file based on its type and dataset id */ char* dir = NULL; if (have_incoming) { if (scr_meta_check_filetype(recv_meta, SCR_META_FILE_USER) == SCR_SUCCESS) { dir = scr_cache_dir_get(red, id); } else { dir = scr_cache_dir_hidden_get(red, id); } } /* exhange file names with partners, * building full path of incoming file */ char file_partner[SCR_MAX_FILENAME]; scr_swap_file_names( file, send_rank, file_partner, sizeof(file_partner), recv_rank, dir, scr_comm_world ); /* free directory string */ scr_free(&dir); /* free incoming meta data (we'll get this again later) */ scr_meta_delete(&recv_meta); /* if we'll receive a file, record the name of our file * in the filemap and write it to disk */ recv_meta = NULL; if (recv_rank != MPI_PROC_NULL) { recv_meta = scr_meta_new(); scr_filemap_add_file(map, id, scr_my_rank_world, file_partner); scr_filemap_write(scr_map_file, map); } /* either sending or receiving a file this round, since we move files, * it will be deleted or overwritten */ if (scr_swap_files(MOVE_FILES, file, send_meta, send_rank, file_partner, recv_meta, recv_rank, scr_comm_world) != SCR_SUCCESS) { scr_err("Swapping files: %s to %d, %s from %d @ %s:%d", file, send_rank, file_partner, recv_rank, __FILE__, __LINE__ ); rc = SCR_FAILURE; } /* if we received a file, record its meta data and decrement * our receive count */ if (have_incoming) { /* record meta data for the file we received */ scr_filemap_set_meta(map, id, scr_my_rank_world, file_partner, recv_meta); scr_meta_delete(&recv_meta); /* decrement receive count */ recv_num--; if (recv_num == 0) { have_incoming = 0; recv_rank = MPI_PROC_NULL; } } /* if we sent a file, remove it from the filemap and decrement * our send count */ if (have_outgoing) { /* remove file from the filemap */ scr_filemap_remove_file(map, id, send_rank, file); scr_meta_delete(&send_meta); /* decrement our send count */ send_num--; if (send_num == 0) { have_outgoing = 0; send_rank = MPI_PROC_NULL; } } /* update filemap on disk */ scr_filemap_write(scr_map_file, map); } /* free our file list */ scr_free(&files); } } } /* if we have more rounds than max rounds, delete the remainder of our files */ for (round = max_rounds+1; round < nranks; round++) { /* have someone's files for this round, so delete them */ int dst_rank = have_rank_by_round[round]; scr_unlink_rank(map, id, dst_rank); } scr_free(&send_flag_by_round); scr_free(&have_rank_by_round); /* write out new filemap and free the memory resources */ scr_filemap_write(scr_map_file, map); /* clean out any incomplete files */ scr_cache_clean(map); /* TODO: if the exchange or redundancy rebuild failed, * we should also delete any *good* files we received */ /* return whether distribute succeeded, it does not ensure we have * all of our files, only that the transfer completed without failure */ return rc; }
/* this transfers redundancy descriptors for the given dataset id */ static int scr_distribute_reddescs(scr_filemap* map, int id, scr_reddesc* red) { int i; /* create a new hash to record redundancy descriptors that we have */ scr_hash* send_hash = scr_hash_new(); /* for this dataset, get list of ranks we have data for */ int nranks = 0; int* ranks = NULL; scr_filemap_list_ranks_by_dataset(map, id, &nranks, &ranks); /* for each rank we have files for, check whether we also have * its redundancy descriptor */ int invalid_rank_found = 0; for (i=0; i < nranks; i++) { /* get the rank id */ int rank = ranks[i]; /* check that the rank is within range */ if (rank < 0 || rank >= scr_ranks_world) { scr_err("Invalid rank id %d in world of %d @ %s:%d", rank, scr_ranks_world, __FILE__, __LINE__ ); invalid_rank_found = 1; } /* lookup the redundancy descriptor hash for this rank */ scr_hash* desc = scr_hash_new(); scr_filemap_get_desc(map, id, rank, desc); /* if this descriptor has entries, add it to our send hash, * delete the hash otherwise */ if (scr_hash_size(desc) > 0) { scr_hash_setf(send_hash, desc, "%d", rank); } else { scr_hash_delete(&desc); } } /* free off our list of ranks */ scr_free(&ranks); /* check that we didn't find an invalid rank on any process */ if (! scr_alltrue(invalid_rank_found == 0)) { scr_hash_delete(&send_hash); return SCR_FAILURE; } /* create an empty hash to receive any incoming descriptors */ /* exchange descriptors with other ranks */ scr_hash* recv_hash = scr_hash_new(); scr_hash_exchange(send_hash, recv_hash, scr_comm_world); /* check that everyone can get their descriptor */ int num_desc = scr_hash_size(recv_hash); if (! scr_alltrue(num_desc > 0)) { scr_hash_delete(&recv_hash); scr_hash_delete(&send_hash); scr_dbg(2, "Cannot find process that has my redundancy descriptor @ %s:%d", __FILE__, __LINE__ ); return SCR_FAILURE; } /* just go with the first redundancy descriptor in our list, * they should all be the same */ scr_hash_elem* desc_elem = scr_hash_elem_first(recv_hash); scr_hash* desc_hash = scr_hash_elem_hash(desc_elem); /* record the descriptor in our filemap */ scr_filemap_set_desc(map, id, scr_my_rank_world, desc_hash); scr_filemap_write(scr_map_file, map); /* TODO: at this point, we could delete descriptors for other * ranks for this checkpoint */ /* create our redundancy descriptor struct from the map */ scr_reddesc_create_from_filemap(map, id, scr_my_rank_world, red); /* free off our send and receive hashes */ scr_hash_delete(&recv_hash); scr_hash_delete(&send_hash); return SCR_SUCCESS; }
/* read in halt file (which program may have changed), update internal data structure, * set & unset any fields, and write out halt file all while locked */ int scr_halt_sync_and_set(const char* file, struct arglist* args, scr_hash* data) { /* set the mode on the file to be readable/writable by all * (enables a sysadmin to halt a user's job via scr_halt --all) */ mode_t old_mode = umask(0000); /* TODO: sleep and try the open several times if the first fails */ /* open the halt file for reading */ int fd = scr_open(file, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); if (fd < 0) { scr_err("Opening file for write: scr_open(%s) errno=%d %m @ %s:%d", file, errno, __FILE__, __LINE__ ); /* restore the normal file mask */ umask(old_mode); return SCR_FAILURE; } /* acquire an exclusive file lock before reading */ int ret = scr_write_lock(file,fd); if (ret != SCR_SUCCESS){ scr_close(file,fd); umask(old_mode); return ret; } /* read in the current data from the file */ scr_hash_read_fd(file, fd, data); /* set / unset values in file */ if (args->set_reason) { scr_hash_unset(data, SCR_HALT_KEY_EXIT_REASON); scr_hash_set_kv(data, SCR_HALT_KEY_EXIT_REASON, args->value_reason); } else if (args->unset_reason) { scr_hash_unset(data, SCR_HALT_KEY_EXIT_REASON); } if (args->set_checkpoints) { scr_hash_unset(data, SCR_HALT_KEY_CHECKPOINTS); scr_hash_setf(data, NULL, "%s %lu", SCR_HALT_KEY_CHECKPOINTS, args->value_checkpoints); } else if (args->unset_checkpoints) { scr_hash_unset(data, SCR_HALT_KEY_CHECKPOINTS); } if (args->set_before) { scr_hash_unset(data, SCR_HALT_KEY_EXIT_BEFORE); scr_hash_setf(data, NULL, "%s %lu", SCR_HALT_KEY_EXIT_BEFORE, args->value_before); } else if (args->unset_before) { scr_hash_unset(data, SCR_HALT_KEY_EXIT_BEFORE); } if (args->set_after) { scr_hash_unset(data, SCR_HALT_KEY_EXIT_AFTER); scr_hash_setf(data, NULL, "%s %lu", SCR_HALT_KEY_EXIT_AFTER, args->value_after); } else if (args->unset_after) { scr_hash_unset(data, SCR_HALT_KEY_EXIT_AFTER); } if (args->set_seconds) { scr_hash_unset(data, SCR_HALT_KEY_SECONDS); scr_hash_setf(data, NULL, "%s %lu", SCR_HALT_KEY_SECONDS, args->value_seconds); } else if (args->unset_seconds) { scr_hash_unset(data, SCR_HALT_KEY_SECONDS); } /* wind file pointer back to the start of the file */ lseek(fd, 0, SEEK_SET); /* write our updated data */ ssize_t bytes_written = scr_hash_write_fd(file, fd, data); /* truncate the file to the correct size (may be smaller than it was before) */ if (bytes_written >= 0) { ftruncate(fd, (off_t) bytes_written); } /* release the file lock */ ret = scr_unlock(file, fd); if (ret != SCR_SUCCESS){ scr_close(file,fd); umask(old_mode); return ret; } /* close file */ scr_close(file, fd); /* restore the normal file mask */ umask(old_mode); /* write current values to halt file */ return SCR_SUCCESS; }
/* since on a restart we may end up with more or fewer ranks on a node than the * previous run, rely on the master to read in and distribute the filemap to * other ranks on the node */ int scr_scatter_filemaps(scr_filemap* my_map) { /* TODO: if the control directory is on a device shared by lots of procs, * we should read and distribute this data in a more scalable way */ /* allocate empty send hash */ scr_hash* send_hash = scr_hash_new(); /* if i'm the master on this node, read in all filemaps */ if (scr_storedesc_cntl->rank == 0) { /* create an empty filemap */ scr_filemap* all_map = scr_filemap_new(); /* read in the master map */ scr_hash* hash = scr_hash_new(); scr_hash_read_path(scr_master_map_file, hash); /* for each filemap listed in the master map */ scr_hash_elem* elem; for (elem = scr_hash_elem_first(scr_hash_get(hash, "Filemap")); elem != NULL; elem = scr_hash_elem_next(elem)) { /* get the filename of this filemap */ char* file = scr_hash_elem_key(elem); /* TODO MEMFS: mount storage for each filemap */ /* read in the filemap */ scr_filemap* tmp_map = scr_filemap_new(); scr_path* path_file = scr_path_from_str(file); scr_filemap_read(path_file, tmp_map); scr_path_delete(&path_file); /* merge it with the all_map */ scr_filemap_merge(all_map, tmp_map); /* delete filemap */ scr_filemap_delete(&tmp_map); /* TODO: note that if we fail after unlinking this file but before * writing out the new file, we'll lose information */ /* delete the file */ scr_file_unlink(file); } /* free the hash object */ scr_hash_delete(&hash); /* write out new local 0 filemap */ if (scr_filemap_num_ranks(all_map) > 0) { scr_filemap_write(scr_map_file, all_map); } /* get global rank of each rank */ int* ranks = (int*) SCR_MALLOC(scr_storedesc_cntl->ranks * sizeof(int)); MPI_Gather( &scr_my_rank_world, 1, MPI_INT, ranks, 1, MPI_INT, 0, scr_storedesc_cntl->comm ); /* for each rank, send them their own file data if we have it */ int i; for (i=0; i < scr_storedesc_cntl->ranks; i++) { int rank = ranks[i]; if (scr_filemap_have_rank(all_map, rank)) { /* extract the filemap for this rank */ scr_filemap* tmp_map = scr_filemap_extract_rank(all_map, rank); /* get a reference to the hash object that we'll send to this rank, * and merge this filemap into it */ scr_hash* tmp_hash = scr_hash_getf(send_hash, "%d", i); if (tmp_hash == NULL) { /* if we don't find an existing entry in the send_hash, * create an empty hash and insert it */ scr_hash* empty_hash = scr_hash_new(); scr_hash_setf(send_hash, empty_hash, "%d", i); tmp_hash = empty_hash; } scr_hash_merge(tmp_hash, tmp_map); /* delete the filemap for this rank */ scr_filemap_delete(&tmp_map); } } /* free our rank list */ scr_free(&ranks); /* now just round robin the remainder across the set (load balancing) */ int num; int* remaining_ranks = NULL; scr_filemap_list_ranks(all_map, &num, &remaining_ranks); int j = 0; while (j < num) { /* pick a rank in to send to */ i = j % scr_storedesc_cntl->ranks; /* extract the filemap for this rank */ scr_filemap* tmp_map = scr_filemap_extract_rank(all_map, remaining_ranks[j]); /* get a reference to the hash object that we'll send to this rank, * and merge this filemap into it */ scr_hash* tmp_hash = scr_hash_getf(send_hash, "%d", i); if (tmp_hash == NULL) { /* if we don't find an existing entry in the send_hash, * create an empty hash and insert it */ scr_hash* empty_hash = scr_hash_new(); scr_hash_setf(send_hash, empty_hash, "%d", i); tmp_hash = empty_hash; } scr_hash_merge(tmp_hash, tmp_map); /* delete the filemap for this rank */ scr_filemap_delete(&tmp_map); j++; } scr_free(&remaining_ranks); /* delete the filemap */ scr_filemap_delete(&all_map); /* write out the new master filemap */ hash = scr_hash_new(); char file[SCR_MAX_FILENAME]; for (i=0; i < scr_storedesc_cntl->ranks; i++) { sprintf(file, "%s/filemap_%d.scrinfo", scr_cntl_prefix, i); scr_hash_set_kv(hash, "Filemap", file); } scr_hash_write_path(scr_master_map_file, hash); scr_hash_delete(&hash); } else { /* send our global rank to the master */ MPI_Gather( &scr_my_rank_world, 1, MPI_INT, NULL, 1, MPI_INT, 0, scr_storedesc_cntl->comm ); } /* receive our filemap from master */ scr_hash* recv_hash = scr_hash_new(); scr_hash_exchange(send_hash, recv_hash, scr_storedesc_cntl->comm); /* merge map sent from master into our map */ scr_hash* map_from_master = scr_hash_getf(recv_hash, "%d", 0); if (map_from_master != NULL) { scr_hash_merge(my_map, map_from_master); } /* write out our local filemap */ if (scr_filemap_num_ranks(my_map) > 0) { scr_filemap_write(scr_map_file, my_map); } /* free off our send and receive hashes */ scr_hash_delete(&recv_hash); scr_hash_delete(&send_hash); return SCR_SUCCESS; }
/* apply XOR redundancy scheme to dataset files */ static int scr_reddesc_apply_xor(scr_filemap* map, const scr_reddesc* c, int id) { int rc = SCR_SUCCESS; int i; /* get pointer to XOR state structure */ scr_reddesc_xor* state = (scr_reddesc_xor*) c->copy_state; /* allocate buffer to read a piece of my file */ char* send_buf = (char*) scr_align_malloc(scr_mpi_buf_size, scr_page_size); if (send_buf == NULL) { scr_abort(-1, "Allocating memory for send buffer: malloc(%d) errno=%d %s @ %s:%d", scr_mpi_buf_size, errno, strerror(errno), __FILE__, __LINE__ ); } /* allocate buffer to read a piece of the recevied chunk file */ char* recv_buf = (char*) scr_align_malloc(scr_mpi_buf_size, scr_page_size); if (recv_buf == NULL) { scr_abort(-1, "Allocating memory for recv buffer: malloc(%d) errno=%d %s @ %s:%d", scr_mpi_buf_size, errno, strerror(errno), __FILE__, __LINE__ ); } /* count the number of files I have and allocate space in structures for each of them */ int num_files = scr_filemap_num_files(map, id, scr_my_rank_world); int* fds = (int*) SCR_MALLOC(num_files * sizeof(int)); char** filenames = (char**) SCR_MALLOC(num_files * sizeof(char*)); unsigned long* filesizes = (unsigned long*) SCR_MALLOC(num_files * sizeof(unsigned long)); /* record partner's redundancy descriptor hash in our filemap */ scr_hash* lhs_desc_hash = scr_hash_new(); scr_hash* my_desc_hash = scr_hash_new(); scr_reddesc_store_to_hash(c, my_desc_hash); scr_hash_sendrecv(my_desc_hash, state->rhs_rank, lhs_desc_hash, state->lhs_rank, c->comm); scr_filemap_set_desc(map, id, state->lhs_rank_world, lhs_desc_hash); scr_hash_delete(&my_desc_hash); scr_hash_delete(&lhs_desc_hash); /* allocate a new xor file header hash */ scr_hash* header = scr_hash_new(); /* record the global ranks of the processes in our xor group */ scr_hash_merge(header, state->group_map); /* record dataset in header */ scr_hash* dataset = scr_hash_new(); scr_filemap_get_dataset(map, id, scr_my_rank_world, dataset); scr_hash_set(header, SCR_KEY_COPY_XOR_DATASET, dataset); /* open each file, get the filesize of each, and read the meta data of each */ scr_hash* current_files = scr_hash_new(); int file_count = 0; unsigned long my_bytes = 0; scr_hash_elem* file_elem; for (file_elem = scr_filemap_first_file(map, id, scr_my_rank_world); file_elem != NULL; file_elem = scr_hash_elem_next(file_elem)) { /* get the filename */ filenames[file_count] = scr_hash_elem_key(file_elem); /* get the filesize of this file and add the byte count to the total */ filesizes[file_count] = scr_file_size(filenames[file_count]); my_bytes += filesizes[file_count]; /* read the meta data for this file and insert it into the current_files hash */ scr_meta* file_hash = scr_meta_new(); scr_filemap_get_meta(map, id, scr_my_rank_world, filenames[file_count], file_hash); scr_hash_setf(current_files, file_hash, "%d", file_count); /* open the file */ fds[file_count] = scr_open(filenames[file_count], O_RDONLY); if (fds[file_count] < 0) { /* TODO: try again? */ scr_abort(-1, "Opening checkpoint file for copying: scr_open(%s, O_RDONLY) errno=%d %s @ %s:%d", filenames[file_count], errno, strerror(errno), __FILE__, __LINE__ ); } file_count++; } /* set total number of files we have, plus our rank */ scr_hash* current_hash = scr_hash_new(); scr_hash_set_kv_int(current_hash, SCR_KEY_COPY_XOR_RANK, scr_my_rank_world); scr_hash_set_kv_int(current_hash, SCR_KEY_COPY_XOR_FILES, file_count); scr_hash_set(current_hash, SCR_KEY_COPY_XOR_FILE, current_files); /* exchange file info with partners and add data to our header */ scr_hash* partner_hash = scr_hash_new(); scr_hash_sendrecv(current_hash, state->rhs_rank, partner_hash, state->lhs_rank, c->comm); scr_hash_set(header, SCR_KEY_COPY_XOR_CURRENT, current_hash); scr_hash_set(header, SCR_KEY_COPY_XOR_PARTNER, partner_hash); /* allreduce to get maximum filesize */ unsigned long max_bytes; MPI_Allreduce(&my_bytes, &max_bytes, 1, MPI_UNSIGNED_LONG, MPI_MAX, c->comm); /* TODO: use unsigned long integer arithmetic (with proper byte padding) instead of char to speed things up */ /* compute chunk size according to maximum file length and number of ranks in xor set */ /* if filesize doesn't divide evenly, then add one byte to chunk_size */ /* TODO: check that ranks > 1 for this divide to be safe (or at partner selection time) */ size_t chunk_size = max_bytes / (unsigned long) (c->ranks - 1); if ((c->ranks - 1) * chunk_size < max_bytes) { chunk_size++; } /* TODO: need something like this to handle 0-byte files? */ if (chunk_size == 0) { chunk_size++; } /* record the dataset id and the chunk size in the xor chunk header */ scr_hash_util_set_bytecount(header, SCR_KEY_COPY_XOR_CHUNK, chunk_size); /* set chunk filenames of form: xor.<group_id>_<xor_rank+1>_of_<xor_ranks>.scr */ char my_chunk_file[SCR_MAX_FILENAME]; char* dir = scr_cache_dir_hidden_get(c, id); sprintf(my_chunk_file, "%s/xor.%d_%d_of_%d.scr", dir, c->group_id, c->rank+1, c->ranks); scr_free(&dir); /* record chunk file in filemap before creating it */ scr_filemap_add_file(map, id, scr_my_rank_world, my_chunk_file); scr_filemap_write(scr_map_file, map); /* open my chunk file */ mode_t mode_file = scr_getmode(1, 1, 0); int fd_chunk = scr_open(my_chunk_file, O_WRONLY | O_CREAT | O_TRUNC, mode_file); if (fd_chunk < 0) { /* TODO: try again? */ scr_abort(-1, "Opening XOR chunk file for writing: scr_open(%s) errno=%d %s @ %s:%d", my_chunk_file, errno, strerror(errno), __FILE__, __LINE__ ); } /* write out the xor chunk header */ scr_hash_write_fd(my_chunk_file, fd_chunk, header); scr_hash_delete(&header); MPI_Request request[2]; MPI_Status status[2]; /* XOR Reduce_scatter */ size_t nread = 0; while (nread < chunk_size) { size_t count = chunk_size - nread; if (count > scr_mpi_buf_size) { count = scr_mpi_buf_size; } int chunk_id; for(chunk_id = c->ranks-1; chunk_id >= 0; chunk_id--) { /* read the next set of bytes for this chunk from my file into send_buf */ if (chunk_id > 0) { int chunk_id_rel = (c->rank + c->ranks + chunk_id) % c->ranks; if (chunk_id_rel > c->rank) { chunk_id_rel--; } unsigned long offset = chunk_size * (unsigned long) chunk_id_rel + nread; if (scr_read_pad_n(num_files, filenames, fds, send_buf, count, offset, filesizes) != SCR_SUCCESS) { rc = SCR_FAILURE; } } else { memset(send_buf, 0, count); } /* TODO: XORing with unsigned long would be faster here (if chunk size is multiple of this size) */ /* merge the blocks via xor operation */ if (chunk_id < c->ranks-1) { for (i = 0; i < count; i++) { send_buf[i] ^= recv_buf[i]; } } if (chunk_id > 0) { /* not our chunk to write, forward it on and get the next */ MPI_Irecv(recv_buf, count, MPI_BYTE, state->lhs_rank, 0, c->comm, &request[0]); MPI_Isend(send_buf, count, MPI_BYTE, state->rhs_rank, 0, c->comm, &request[1]); MPI_Waitall(2, request, status); } else { /* write send block to send chunk file */ if (scr_write_attempt(my_chunk_file, fd_chunk, send_buf, count) != count) { rc = SCR_FAILURE; } } } nread += count; } /* close my chunkfile, with fsync */ if (scr_close(my_chunk_file, fd_chunk) != SCR_SUCCESS) { rc = SCR_FAILURE; } /* close my dataset files */ for (i=0; i < num_files; i++) { scr_close(filenames[i], fds[i]); } /* free the buffers */ scr_free(&filesizes); /* in this case, we don't free each name, since we copied the pointer to the string in the filemap */ scr_free(&filenames); scr_free(&fds); scr_align_free(&send_buf); scr_align_free(&recv_buf); /* TODO: need to check for errors */ /* write meta file for xor chunk */ unsigned long my_chunk_file_size = scr_file_size(my_chunk_file); scr_meta* meta = scr_meta_new(); scr_meta_set_filename(meta, my_chunk_file); scr_meta_set_filetype(meta, SCR_META_FILE_XOR); scr_meta_set_filesize(meta, my_chunk_file_size); scr_meta_set_complete(meta, 1); /* TODODSET: move the ranks field elsewhere, for now it's needed by scr_index.c */ scr_meta_set_ranks(meta, scr_ranks_world); scr_filemap_set_meta(map, id, scr_my_rank_world, my_chunk_file, meta); scr_filemap_write(scr_map_file, map); scr_meta_delete(&meta); /* if crc_on_copy is set, compute and store CRC32 value for chunk file */ if (scr_crc_on_copy) { scr_compute_crc(map, id, scr_my_rank_world, my_chunk_file); /* TODO: would be nice to save this CRC in our partner's XOR file so we can check correctness on a rebuild */ } return rc; }