/* for file name listed in meta, fetch that file from src_dir and store
* a copy in dst_dir, record full path to copy in newfile, and
* return whether operation succeeded */
static int scr_fetch_file(
const char* dst_file,
const char* src_dir,
const scr_meta* meta)
{
int rc = SCR_SUCCESS;
/* build full path to source file */
scr_path* path_src_file = scr_path_from_str(dst_file);
scr_path_basename(path_src_file);
scr_path_prepend_str(path_src_file, src_dir);
char* src_file = scr_path_strdup(path_src_file);
/* fetch the file */
uLong crc;
uLong* crc_p = NULL;
if (scr_crc_on_flush) {
crc_p = &crc;
}
rc = scr_file_copy(src_file, dst_file, scr_file_buf_size, crc_p);
/* check that crc matches crc stored in meta */
uLong meta_crc;
if (scr_meta_get_crc32(meta, &meta_crc) == SCR_SUCCESS) {
if (rc == SCR_SUCCESS && scr_crc_on_flush && crc != meta_crc) {
rc = SCR_FAILURE;
scr_err("CRC32 mismatch detected when fetching file from %s to %s @ %s:%d",
src_file, dst_file, __FILE__, __LINE__
);
/* TODO: would be good to log this, but right now only rank 0
* can write log entries */
/*
if (scr_log_enable) {
time_t now = scr_log_seconds();
scr_log_event("CRC32 MISMATCH", filename, NULL, &now, NULL);
}
*/
}
}
/* free path and string for source file */
scr_free(&src_file);
scr_path_delete(&path_src_file);
return rc;
}
static int scr_compute_crc(scr_filemap* map, int id, int rank, const char* file)
{
/* compute crc for the file */
uLong crc_file;
if (scr_crc32(file, &crc_file) != SCR_SUCCESS) {
scr_err("Failed to compute crc for file %s @ %s:%d",
file, __FILE__, __LINE__
);
return SCR_FAILURE;
}
/* allocate a new meta data object */
scr_meta* meta = scr_meta_new();
if (meta == NULL) {
scr_abort(-1, "Failed to allocate meta data object @ %s:%d",
__FILE__, __LINE__
);
}
/* read meta data from filemap */
if (scr_filemap_get_meta(map, id, rank, file, meta) != SCR_SUCCESS) {
return SCR_FAILURE;
}
int rc = SCR_SUCCESS;
/* read crc value from meta data */
uLong crc_meta;
if (scr_meta_get_crc32(meta, &crc_meta) == SCR_SUCCESS) {
/* check that the values are the same */
if (crc_file != crc_meta) {
rc = SCR_FAILURE;
}
} else {
/* record crc in filemap */
scr_meta_set_crc32(meta, crc_file);
scr_filemap_set_meta(map, id, rank, file, meta);
}
/* free our meta data object */
scr_meta_delete(&meta);
return rc;
}
static int scr_swap_files_copy(
int have_outgoing, const char* file_send, scr_meta* meta_send, int rank_send, uLong* crc32_send,
int have_incoming, const char* file_recv, scr_meta* meta_recv, int rank_recv, uLong* crc32_recv,
MPI_Comm comm)
{
int rc = SCR_SUCCESS;
MPI_Request request[2];
MPI_Status status[2];
/* allocate MPI send buffer */
char *buf_send = NULL;
if (have_outgoing) {
buf_send = (char*) scr_align_malloc(scr_mpi_buf_size, scr_page_size);
if (buf_send == NULL) {
scr_abort(-1, "Allocating memory: malloc(%ld) errno=%d %s @ %s:%d",
scr_mpi_buf_size, errno, strerror(errno), __FILE__, __LINE__
);
return SCR_FAILURE;
}
}
/* allocate MPI recv buffer */
char *buf_recv = NULL;
if (have_incoming) {
buf_recv = (char*) scr_align_malloc(scr_mpi_buf_size, scr_page_size);
if (buf_recv == NULL) {
scr_abort(-1, "Allocating memory: malloc(%ld) errno=%d %s @ %s:%d",
scr_mpi_buf_size, errno, strerror(errno), __FILE__, __LINE__
);
return SCR_FAILURE;
}
}
/* open the file to send: read-only mode */
int fd_send = -1;
if (have_outgoing) {
fd_send = scr_open(file_send, O_RDONLY);
if (fd_send < 0) {
scr_abort(-1, "Opening file for send: scr_open(%s, O_RDONLY) errno=%d %s @ %s:%d",
file_send, errno, strerror(errno), __FILE__, __LINE__
);
}
}
/* open the file to recv: truncate, write-only mode */
int fd_recv = -1;
if (have_incoming) {
mode_t mode_file = scr_getmode(1, 1, 0);
fd_recv = scr_open(file_recv, O_WRONLY | O_CREAT | O_TRUNC, mode_file);
if (fd_recv < 0) {
scr_abort(-1, "Opening file for recv: scr_open(%s, O_WRONLY | O_CREAT | O_TRUNC, ...) errno=%d %s @ %s:%d",
file_recv, errno, strerror(errno), __FILE__, __LINE__
);
}
}
/* exchange file chunks */
int nread, nwrite;
int sending = 0;
if (have_outgoing) {
sending = 1;
}
int receiving = 0;
if (have_incoming) {
receiving = 1;
}
while (sending || receiving) {
/* if we are still receiving a file, post a receive */
if (receiving) {
MPI_Irecv(buf_recv, scr_mpi_buf_size, MPI_BYTE, rank_recv, 0, comm, &request[0]);
}
/* if we are still sending a file, read a chunk, send it, and wait */
if (sending) {
nread = scr_read(file_send, fd_send, buf_send, scr_mpi_buf_size);
if (scr_crc_on_copy && nread > 0) {
*crc32_send = crc32(*crc32_send, (const Bytef*) buf_send, (uInt) nread);
}
if (nread < 0) {
nread = 0;
}
MPI_Isend(buf_send, nread, MPI_BYTE, rank_send, 0, comm, &request[1]);
MPI_Wait(&request[1], &status[1]);
if (nread < scr_mpi_buf_size) {
sending = 0;
}
}
/* if we are still receiving a file,
* wait on our receive to complete and write the data */
if (receiving) {
MPI_Wait(&request[0], &status[0]);
MPI_Get_count(&status[0], MPI_BYTE, &nwrite);
if (scr_crc_on_copy && nwrite > 0) {
*crc32_recv = crc32(*crc32_recv, (const Bytef*) buf_recv, (uInt) nwrite);
}
scr_write(file_recv, fd_recv, buf_recv, nwrite);
if (nwrite < scr_mpi_buf_size) {
receiving = 0;
}
}
}
/* close the files */
if (have_outgoing) {
scr_close(file_send, fd_send);
}
if (have_incoming) {
scr_close(file_recv, fd_recv);
}
/* set crc field on our file if it hasn't been set already */
if (scr_crc_on_copy && have_outgoing) {
uLong meta_send_crc;
if (scr_meta_get_crc32(meta_send, &meta_send_crc) != SCR_SUCCESS) {
scr_meta_set_crc32(meta_send, *crc32_send);
} else {
/* TODO: we could check that the crc on the sent file matches and take some action if not */
}
}
/* free the MPI buffers */
scr_align_free(&buf_recv);
scr_align_free(&buf_send);
return rc;
}
/* scr_swap_files -- copy or move a file from one node to another
* if swap_type = COPY_FILES
* if file_send != NULL, send file_send to rank_send, who will make a copy,
* copy file from rank_recv if there is one to receive
* if swap_type = MOVE_FILES
* if file_send != NULL, move file_send to rank_send
* save file from rank_recv if there is one to receive
* To conserve space (e.g., RAM disc), if file_send exists,
* any incoming file will overwrite file_send in place, one block at a time.
* It is then truncated and renamed according the size and name of the incoming file,
* or it is deleted (moved) if there is no incoming file.
*/
int scr_swap_files(
int swap_type,
const char* file_send, scr_meta* meta_send, int rank_send,
const char* file_recv, scr_meta* meta_recv, int rank_recv,
MPI_Comm comm)
{
int rc = SCR_SUCCESS;
/* determine whether we have a file to send */
int have_outgoing = 0;
if (rank_send != MPI_PROC_NULL &&
file_send != NULL &&
strcmp(file_send, "") != 0)
{
have_outgoing = 1;
}
/* determine whether we are expecting to receive a file */
int have_incoming = 0;
if (rank_recv != MPI_PROC_NULL &&
file_recv != NULL &&
strcmp(file_recv, "") != 0)
{
have_incoming = 1;
}
/* exchange meta file info with partners */
scr_hash_sendrecv(meta_send, rank_send, meta_recv, rank_recv, comm);
/* initialize crc values */
uLong crc32_send = crc32(0L, Z_NULL, 0);
uLong crc32_recv = crc32(0L, Z_NULL, 0);
/* exchange files */
if (swap_type == COPY_FILES) {
scr_swap_files_copy(
have_outgoing, file_send, meta_send, rank_send, &crc32_send,
have_incoming, file_recv, meta_recv, rank_recv, &crc32_recv,
comm
);
} else if (swap_type == MOVE_FILES) {
scr_swap_files_move(
have_outgoing, file_send, meta_send, rank_send, &crc32_send,
have_incoming, file_recv, meta_recv, rank_recv, &crc32_recv,
comm
);
} else {
scr_err("Unknown file transfer type: %d @ %s:%d",
swap_type, __FILE__, __LINE__
);
return SCR_FAILURE;
}
/* mark received file as complete */
if (have_incoming) {
/* check that our written file is the correct size */
unsigned long filesize_wrote = scr_file_size(file_recv);
if (scr_meta_check_filesize(meta_recv, filesize_wrote) != SCR_SUCCESS) {
scr_err("Received file does not match expected size %s @ %s:%d",
file_recv, __FILE__, __LINE__
);
scr_meta_set_complete(meta_recv, 0);
rc = SCR_FAILURE;
}
/* check that there was no corruption in receiving the file */
if (scr_crc_on_copy) {
/* if we computed crc during the copy, and crc is set in the received meta data
* check that our computed value matches */
uLong crc32_recv_meta;
if (scr_meta_get_crc32(meta_recv, &crc32_recv_meta) == SCR_SUCCESS) {
if (crc32_recv != crc32_recv_meta) {
scr_err("CRC32 mismatch detected when receiving file %s @ %s:%d",
file_recv, __FILE__, __LINE__
);
scr_meta_set_complete(meta_recv, 0);
rc = SCR_FAILURE;
}
}
}
}
return rc;
}
static int scr_swap_files_move(
int have_outgoing, const char* file_send, scr_meta* meta_send, int rank_send, uLong* crc32_send,
int have_incoming, const char* file_recv, scr_meta* meta_recv, int rank_recv, uLong* crc32_recv,
MPI_Comm comm)
{
int rc = SCR_SUCCESS;
MPI_Request request[2];
MPI_Status status[2];
/* allocate MPI send buffer */
char *buf_send = NULL;
if (have_outgoing) {
buf_send = (char*) scr_align_malloc(scr_mpi_buf_size, scr_page_size);
if (buf_send == NULL) {
scr_abort(-1, "Allocating memory: malloc(%ld) errno=%d %s @ %s:%d",
scr_mpi_buf_size, errno, strerror(errno), __FILE__, __LINE__
);
return SCR_FAILURE;
}
}
/* allocate MPI recv buffer */
char *buf_recv = NULL;
if (have_incoming) {
buf_recv = (char*) scr_align_malloc(scr_mpi_buf_size, scr_page_size);
if (buf_recv == NULL) {
scr_abort(-1, "Allocating memory: malloc(%ld) errno=%d %s @ %s:%d",
scr_mpi_buf_size, errno, strerror(errno), __FILE__, __LINE__
);
return SCR_FAILURE;
}
}
/* since we'll overwrite our send file in place with the recv file,
* which may be larger, we need to keep track of how many bytes we've
* sent and whether we've sent them all */
unsigned long filesize_send = 0;
/* open our file */
int fd = -1;
if (have_outgoing) {
/* we'll overwrite our send file (or just read it if there is no incoming) */
filesize_send = scr_file_size(file_send);
fd = scr_open(file_send, O_RDWR);
if (fd < 0) {
/* TODO: skip writes and return error? */
scr_abort(-1, "Opening file for send/recv: scr_open(%s, O_RDWR) errno=%d %s @ %s:%d",
file_send, errno, strerror(errno), __FILE__, __LINE__
);
}
} else if (have_incoming) {
/* if we're in this branch, then we only have an incoming file,
* so we'll write our recv file from scratch */
mode_t mode_file = scr_getmode(1, 1, 0);
fd = scr_open(file_recv, O_WRONLY | O_CREAT | O_TRUNC, mode_file);
if (fd < 0) {
/* TODO: skip writes and return error? */
scr_abort(-1, "Opening file for recv: scr_open(%s, O_WRONLY | O_CREAT | O_TRUNC, ...) errno=%d %s @ %s:%d",
file_recv, errno, strerror(errno), __FILE__, __LINE__
);
}
}
/* exchange file chunks */
int sending = 0;
if (have_outgoing) {
sending = 1;
}
int receiving = 0;
if (have_incoming) {
receiving = 1;
}
int nread, nwrite;
off_t read_pos = 0, write_pos = 0;
while (sending || receiving) {
if (receiving) {
/* prepare a buffer to receive up to scr_mpi_buf_size bytes */
MPI_Irecv(buf_recv, scr_mpi_buf_size, MPI_BYTE, rank_recv, 0, comm, &request[0]);
}
if (sending) {
/* compute number of bytes to read */
unsigned long count = filesize_send - read_pos;
if (count > scr_mpi_buf_size) {
count = scr_mpi_buf_size;
}
/* read a chunk of up to scr_mpi_buf_size bytes into buf_send */
lseek(fd, read_pos, SEEK_SET); /* seek to read position */
nread = scr_read(file_send, fd, buf_send, count);
if (scr_crc_on_copy && nread > 0) {
*crc32_send = crc32(*crc32_send, (const Bytef*) buf_send, (uInt) nread);
}
if (nread < 0) {
nread = 0;
}
read_pos += (off_t) nread; /* update read pointer */
/* send chunk (if nread is smaller than scr_mpi_buf_size,
* then we've read the whole file) */
MPI_Isend(buf_send, nread, MPI_BYTE, rank_send, 0, comm, &request[1]);
MPI_Wait(&request[1], &status[1]);
/* check whether we've read the whole file */
if (filesize_send == read_pos && count < scr_mpi_buf_size) {
sending = 0;
}
}
if (receiving) {
/* count the number of bytes received */
MPI_Wait(&request[0], &status[0]);
MPI_Get_count(&status[0], MPI_BYTE, &nwrite);
if (scr_crc_on_copy && nwrite > 0) {
*crc32_recv = crc32(*crc32_recv, (const Bytef*) buf_recv, (uInt) nwrite);
}
/* write those bytes to file (if nwrite is smaller than scr_mpi_buf_size,
* then we've received the whole file) */
lseek(fd, write_pos, SEEK_SET); /* seek to write position */
scr_write(file_recv, fd, buf_recv, nwrite);
write_pos += (off_t) nwrite; /* update write pointer */
/* if nwrite is smaller than scr_mpi_buf_size,
* then assume we've received the whole file */
if (nwrite < scr_mpi_buf_size) {
receiving = 0;
}
}
}
/* close file and cleanup */
if (have_outgoing && have_incoming) {
/* sent and received a file; close it, truncate it to corect size, rename it */
scr_close(file_send, fd);
truncate(file_send, write_pos);
rename(file_send, file_recv);
} else if (have_outgoing) {
/* only sent a file; close it, delete it, and remove its completion marker */
scr_close(file_send, fd);
scr_file_unlink(file_send);
} else if (have_incoming) {
/* only received a file; just need to close it */
scr_close(file_recv, fd);
}
if (scr_crc_on_copy && have_outgoing) {
uLong meta_send_crc;
if (scr_meta_get_crc32(meta_send, &meta_send_crc) != SCR_SUCCESS) {
/* we transfer this meta data across below,
* so may as well update these fields so we can use them */
scr_meta_set_crc32(meta_send, *crc32_send);
/* do not complete file send, we just deleted it above */
} else {
/* TODO: we could check that the crc on the sent file matches and take some action if not */
}
}
/* free the MPI buffers */
scr_align_free(&buf_recv);
scr_align_free(&buf_send);
return rc;
}
/* given a filename, its meta data, its list of segments, and list of destination containers,
* copy file to container files */
static int scr_flush_file_to_containers(
const char* file,
scr_meta* meta,
scr_hash* segments,
const char* dst_dir)
{
/* check that we got something for a source file */
if (file == NULL || strcmp(file, "") == 0) {
scr_err("Invalid source file @ %s:%d",
__FILE__, __LINE__
);
return SCR_FAILURE;
}
/* check that our other arguments are valid */
if (meta == NULL || segments == NULL) {
scr_err("Invalid metadata or segments @ %s:%d",
__FILE__, __LINE__
);
return SCR_FAILURE;
}
/* open the file for reading */
int fd_src = scr_open(file, O_RDONLY);
if (fd_src < 0) {
scr_err("Opening file to copy: scr_open(%s) errno=%d %s @ %s:%d",
file, errno, strerror(errno), __FILE__, __LINE__
);
return SCR_FAILURE;
}
#if !defined(__APPLE__)
/* TODO:
posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED | POSIX_FADV_SEQUENTIAL)
that tells the kernel that you don't ever need the pages
from the file again, and it won't bother keeping them in the page cache.
*/
posix_fadvise(fd_src, 0, 0, POSIX_FADV_DONTNEED | POSIX_FADV_SEQUENTIAL);
#endif
/* get the buffer size we'll use to write to the file */
unsigned long buf_size = scr_file_buf_size;
/* allocate buffer to read in file chunks */
char* buf = (char*) SCR_MALLOC(buf_size);
/* initialize crc value */
uLong crc;
if (scr_crc_on_flush) {
crc = crc32(0L, Z_NULL, 0);
}
int rc = SCR_SUCCESS;
/* write out each segment */
scr_hash_sort_int(segments, SCR_HASH_SORT_ASCENDING);
scr_hash_elem* elem;
for (elem = scr_hash_elem_first(segments);
elem != NULL;
elem = scr_hash_elem_next(elem))
{
/* get the container info for this segment */
scr_hash* hash = scr_hash_elem_hash(elem);
/* get the offset into the container and the length of the segment (both in bytes) */
char* container_name;
unsigned long container_offset, segment_length;
if (scr_container_get_name_offset_length(hash,
&container_name, &container_offset, &segment_length) != SCR_SUCCESS)
{
scr_err("Failed to get segment offset and length @ %s:%d",
__FILE__, __LINE__
);
rc = SCR_FAILURE;
break;
}
/* build full name to destination file */
scr_path* dst_path = scr_path_from_str(dst_dir);
scr_path_append_str(dst_path, container_name);
scr_path_reduce(dst_path);
char* dst_file = scr_path_strdup(dst_path);
/* open container file for writing -- we don't truncate here because more than one
* process may be writing to the same file */
int fd_container = scr_open(dst_file, O_WRONLY);
if (fd_container < 0) {
scr_err("Opening file for writing: scr_open(%s) errno=%d %s @ %s:%d",
dst_file, errno, strerror(errno), __FILE__, __LINE__
);
rc = SCR_FAILURE;
break;
}
#if !defined(__APPLE__)
/* TODO:
posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED | POSIX_FADV_SEQUENTIAL)
that tells the kernel that you don't ever need the pages
from the file again, and it won't bother keeping them in the page cache.
*/
posix_fadvise(fd_container, 0, 0, POSIX_FADV_DONTNEED | POSIX_FADV_SEQUENTIAL);
#endif
/* seek to offset within container */
off_t pos = (off_t) container_offset;
if (lseek(fd_container, pos, SEEK_SET) == (off_t)-1) {
/* our seek failed, return an error */
scr_err("Failed to seek to byte %lu in %s @ %s:%d",
pos, dst_file, __FILE__, __LINE__
);
rc = SCR_FAILURE;
break;
}
/* copy data from file into container in chunks */
unsigned long remaining = segment_length;
while (remaining > 0) {
/* read / write up to buf_size bytes at a time from file */
unsigned long count = remaining;
if (count > buf_size) {
count = buf_size;
}
/* attempt to read buf_size bytes from file */
int nread = scr_read_attempt(file, fd_src, buf, count);
/* if we read some bytes, write them out */
if (nread > 0) {
/* optionally compute crc value as we go */
if (scr_crc_on_flush) {
crc = crc32(crc, (const Bytef*) buf, (uInt) nread);
}
/* write our nread bytes out */
int nwrite = scr_write_attempt(dst_file, fd_container, buf, nread);
/* check for a write error or a short write */
if (nwrite != nread) {
/* write had a problem, stop copying and return an error */
rc = SCR_FAILURE;
break;
}
/* subtract the bytes we've processed from the number remaining */
remaining -= (unsigned long) nread;
}
/* assume a short read is an error */
if (nread < count) {
/* read had a problem, stop copying and return an error */
rc = SCR_FAILURE;
break;
}
/* check for a read error, stop copying and return an error */
if (nread < 0) {
/* read had a problem, stop copying and return an error */
rc = SCR_FAILURE;
break;
}
}
/* close container */
if (scr_close(dst_file, fd_container) != SCR_SUCCESS) {
rc = SCR_FAILURE;
}
/* free the container file name and path */
scr_free(&dst_file);
scr_path_delete(&dst_path);
}
/* close the source file */
if (scr_close(file, fd_src) != SCR_SUCCESS) {
rc = SCR_FAILURE;
}
/* free buffer */
scr_free(&buf);
/* verify / set crc value */
if (rc == SCR_SUCCESS) {
uLong crc2;
if (scr_crc_on_flush) {
if (scr_meta_get_crc32(meta, &crc2) == SCR_SUCCESS) {
/* if a crc is already set in the meta data, check that we computed the same value */
if (crc != crc2) {
scr_err("CRC32 mismatch detected when flushing file %s @ %s:%d",
file, __FILE__, __LINE__
);
rc = SCR_FAILURE;
}
} else {
/* if there is no crc set, let's set it now */
scr_meta_set_crc32(meta, crc);
}
}
}
return rc;
}
/* flush files specified in list, and record corresponding entries for summary file */
static int scr_flush_files_list(scr_hash* file_list, scr_hash* summary)
{
/* assume we will succeed in this flush */
int rc = SCR_SUCCESS;
/* flush each of my files and fill in summary data structure */
scr_hash_elem* elem = NULL;
scr_hash* files = scr_hash_get(file_list, SCR_KEY_FILE);
for (elem = scr_hash_elem_first(files);
elem != NULL;
elem = scr_hash_elem_next(elem))
{
/* get the filename */
char* file = scr_hash_elem_key(elem);
/* convert file to path and extract name of file */
scr_path* path_name = scr_path_from_str(file);
scr_path_basename(path_name);
/* get the hash for this element */
scr_hash* hash = scr_hash_elem_hash(elem);
/* get meta data for this file */
scr_meta* meta = scr_hash_get(hash, SCR_KEY_META);
/* if segments are defined, we flush the file to its containers,
* otherwise we copy the file out as is */
scr_hash* segments = scr_hash_get(hash, SCR_SUMMARY_6_KEY_SEGMENT);
if (segments != NULL) {
/* TODO: PRESERVE get original filename here */
/* add this file to the summary file */
char* name = scr_path_strdup(path_name);
scr_hash* file_hash = scr_hash_set_kv(summary, SCR_SUMMARY_6_KEY_FILE, name);
scr_free(&name);
// USERDEF fixme!
/* flush the file to the containers listed in its segmenets */
if (scr_flush_file_to_containers(file, meta, segments, scr_prefix) == SCR_SUCCESS) {
/* successfully flushed this file, record the filesize */
unsigned long filesize = 0;
if (scr_meta_get_filesize(meta, &filesize) == SCR_SUCCESS) {
scr_hash_util_set_bytecount(file_hash, SCR_SUMMARY_6_KEY_SIZE, filesize);
}
/* record the crc32 if one was computed */
uLong crc = 0;
if (scr_meta_get_crc32(meta, &crc) == SCR_SUCCESS) {
scr_hash_util_set_crc32(file_hash, SCR_SUMMARY_6_KEY_CRC, crc);
}
/* record segment information in summary file */
scr_hash* segments_copy = scr_hash_new();
scr_hash_merge(segments_copy, segments);
scr_hash_set(file_hash, SCR_SUMMARY_6_KEY_SEGMENT, segments_copy);
} else {
/* the flush failed */
rc = SCR_FAILURE;
/* explicitly mark file as incomplete */
scr_hash_set_kv_int(file_hash, SCR_SUMMARY_6_KEY_COMPLETE, 0);
}
} else {
/* get directory to flush file to */
char* dir;
if (scr_hash_util_get_str(hash, SCR_KEY_PATH, &dir) == SCR_SUCCESS) {
/* create full path of destination file */
scr_path* path_full = scr_path_from_str(dir);
scr_path_append(path_full, path_name);
/* get relative path to flushed file from SCR_PREFIX directory */
scr_path* path_relative = scr_path_relative(scr_prefix_path, path_full);
if (! scr_path_is_null(path_relative)) {
/* record the name of the file in the summary hash, and get reference to a hash for this file */
char* name = scr_path_strdup(path_relative);
scr_hash* file_hash = scr_hash_set_kv(summary, SCR_SUMMARY_6_KEY_FILE, name);
scr_free(&name);
/* flush the file and fill in the meta data for this file */
if (scr_flush_a_file(file, dir, meta) == SCR_SUCCESS) {
/* successfully flushed this file, record the filesize */
unsigned long filesize = 0;
if (scr_meta_get_filesize(meta, &filesize) == SCR_SUCCESS) {
scr_hash_util_set_bytecount(file_hash, SCR_SUMMARY_6_KEY_SIZE, filesize);
}
/* record the crc32 if one was computed */
uLong crc = 0;
if (scr_meta_get_crc32(meta, &crc) == SCR_SUCCESS) {
scr_hash_util_set_crc32(file_hash, SCR_SUMMARY_6_KEY_CRC, crc);
}
} else {
/* the flush failed */
rc = SCR_FAILURE;
/* explicitly mark incomplete files */
scr_hash_set_kv_int(file_hash, SCR_SUMMARY_6_KEY_COMPLETE, 0);
}
} else {
scr_abort(-1, "Failed to get relative path to directory %s from %s @ %s:%d",
dir, scr_prefix, __FILE__, __LINE__
);
}
/* free relative and full paths */
scr_path_delete(&path_relative);
scr_path_delete(&path_full);
} else {
scr_abort(-1, "Failed to read directory to flush file to @ %s:%d",
__FILE__, __LINE__
);
}
}
/* free the file name path */
scr_path_delete(&path_name);
}
return rc;
}
/* flushes file named in src_file to dst_dir and fills in meta based on flush,
* returns success of flush */
static int scr_flush_a_file(const char* src_file, const char* dst_dir, scr_meta* meta)
{
int flushed = SCR_SUCCESS;
int tmp_rc;
/* build full name to destination file */
scr_path* dst_path = scr_path_from_str(src_file);
scr_path_basename(dst_path);
scr_path_prepend_str(dst_path, dst_dir);
scr_path_reduce(dst_path);
char* dst_file = scr_path_strdup(dst_path);
/* copy file */
int crc_valid = 0;
uLong crc;
uLong* crc_p = NULL;
if (scr_crc_on_flush) {
crc_valid = 1;
crc_p = &crc;
}
tmp_rc = scr_file_copy(src_file, dst_file, scr_file_buf_size, crc_p);
if (tmp_rc != SCR_SUCCESS) {
crc_valid = 0;
flushed = SCR_FAILURE;
}
scr_dbg(2, "scr_flush_a_file: Read and copied %s to %s with success code %d @ %s:%d",
src_file, dst_file, tmp_rc, __FILE__, __LINE__
);
/* if file has crc32, check it against the one computed during the copy,
* otherwise if scr_crc_on_flush is set, record crc32 */
if (crc_valid) {
uLong crc_meta;
if (scr_meta_get_crc32(meta, &crc_meta) == SCR_SUCCESS) {
if (crc != crc_meta) {
/* detected a crc mismatch during the copy */
/* TODO: unlink the copied file */
/* scr_file_unlink(dst_file); */
/* mark the file as invalid */
scr_meta_set_complete(meta, 0);
flushed = SCR_FAILURE;
scr_err("scr_flush_a_file: CRC32 mismatch detected when flushing file %s to %s @ %s:%d",
src_file, dst_file, __FILE__, __LINE__
);
/* TODO: would be good to log this, but right now only rank 0 can write log entries */
/*
if (scr_log_enable) {
time_t now = scr_log_seconds();
scr_log_event("CRC32 MISMATCH", dst_file, NULL, &now, NULL);
}
*/
}
} else {
/* the crc was not already in the metafile, but we just computed it, so set it */
scr_meta_set_crc32(meta, crc);
}
}
/* TODO: check that written filesize matches expected filesize */
/* fill out meta data, set complete field based on flush success */
/* (we don't update the meta file here, since perhaps the file in cache is ok and only the flush failed) */
int complete = (flushed == SCR_SUCCESS);
scr_meta_set_complete(meta, complete);
/* free destination file string and path */
scr_free(&dst_file);
scr_path_delete(&dst_path);
return flushed;
}
