/* read line from file into buf with given size */
ssize_t scr_read_line(const char* file, int fd, char* buf, size_t size)
{
/* read up to size-1 bytes from fd into buf until we find a newline or EOF */
ssize_t n = 0;
int found_end = 0;
while (n < size-1 && !found_end) {
/* read a character from the file */
char c;
ssize_t nread = scr_read(file, fd, &c, sizeof(c));
if (nread > 0) {
/* we read a character, copy it over to the buffer */
buf[n] = c;
n++;
/* check whether we hit the end of the line */
if (c == '\n') {
found_end = 1;
}
} else if (nread == 0) {
/* we hit the end of the file */
found_end = 1;
} else { /* nread < 0 */
/* we hit an error */
scr_err("Error reading from file %s @ %s:%d",
file, __FILE__, __LINE__
);
return -1;
}
}
/* tack on the NULL character */
buf[n] = '\0';
/* if we exit the while loop but didn't find the end of the line, the buffer was too small */
if (!found_end) {
scr_err("Buffer too small to read line from file %s @ %s:%d",
file, __FILE__, __LINE__
);
return -1;
}
/* NOTE: we don't want to count the NULL which we added, but there is no need to adjust n here */
return n;
}
/* opens, reads, and computes the crc32 value for the given filename */
int scr_crc32(const char* filename, uLong* crc)
{
/* check that we got a variable to write our answer to */
if (crc == NULL) {
return SCR_FAILURE;
}
/* initialize our crc value */
*crc = crc32(0L, Z_NULL, 0);
/* open the file for reading */
int fd = scr_open(filename, O_RDONLY);
if (fd < 0) {
scr_dbg(1, "Failed to open file to compute crc: %s errno=%d @ file %s:%d",
filename, errno, __FILE__, __LINE__
);
return SCR_FAILURE;
}
/* read the file data in and compute its crc32 */
int nread = 0;
unsigned long buffer_size = 1024*1024;
char buf[buffer_size];
do {
nread = scr_read(filename, fd, buf, buffer_size);
if (nread > 0) {
*crc = crc32(*crc, (const Bytef*) buf, (uInt) nread);
}
} while (nread == buffer_size);
/* if we got an error, don't print anything and bailout */
if (nread < 0) {
scr_dbg(1, "Error while reading file to compute crc: %s @ file %s:%d",
filename, __FILE__, __LINE__
);
close(fd);
return SCR_FAILURE;
}
/* close the file */
scr_close(filename, fd);
return SCR_SUCCESS;
}
static int scr_swap_files_copy(
int have_outgoing, const char* file_send, scr_meta* meta_send, int rank_send, uLong* crc32_send,
int have_incoming, const char* file_recv, scr_meta* meta_recv, int rank_recv, uLong* crc32_recv,
MPI_Comm comm)
{
int rc = SCR_SUCCESS;
MPI_Request request[2];
MPI_Status status[2];
/* allocate MPI send buffer */
char *buf_send = NULL;
if (have_outgoing) {
buf_send = (char*) scr_align_malloc(scr_mpi_buf_size, scr_page_size);
if (buf_send == NULL) {
scr_abort(-1, "Allocating memory: malloc(%ld) errno=%d %s @ %s:%d",
scr_mpi_buf_size, errno, strerror(errno), __FILE__, __LINE__
);
return SCR_FAILURE;
}
}
/* allocate MPI recv buffer */
char *buf_recv = NULL;
if (have_incoming) {
buf_recv = (char*) scr_align_malloc(scr_mpi_buf_size, scr_page_size);
if (buf_recv == NULL) {
scr_abort(-1, "Allocating memory: malloc(%ld) errno=%d %s @ %s:%d",
scr_mpi_buf_size, errno, strerror(errno), __FILE__, __LINE__
);
return SCR_FAILURE;
}
}
/* open the file to send: read-only mode */
int fd_send = -1;
if (have_outgoing) {
fd_send = scr_open(file_send, O_RDONLY);
if (fd_send < 0) {
scr_abort(-1, "Opening file for send: scr_open(%s, O_RDONLY) errno=%d %s @ %s:%d",
file_send, errno, strerror(errno), __FILE__, __LINE__
);
}
}
/* open the file to recv: truncate, write-only mode */
int fd_recv = -1;
if (have_incoming) {
mode_t mode_file = scr_getmode(1, 1, 0);
fd_recv = scr_open(file_recv, O_WRONLY | O_CREAT | O_TRUNC, mode_file);
if (fd_recv < 0) {
scr_abort(-1, "Opening file for recv: scr_open(%s, O_WRONLY | O_CREAT | O_TRUNC, ...) errno=%d %s @ %s:%d",
file_recv, errno, strerror(errno), __FILE__, __LINE__
);
}
}
/* exchange file chunks */
int nread, nwrite;
int sending = 0;
if (have_outgoing) {
sending = 1;
}
int receiving = 0;
if (have_incoming) {
receiving = 1;
}
while (sending || receiving) {
/* if we are still receiving a file, post a receive */
if (receiving) {
MPI_Irecv(buf_recv, scr_mpi_buf_size, MPI_BYTE, rank_recv, 0, comm, &request[0]);
}
/* if we are still sending a file, read a chunk, send it, and wait */
if (sending) {
nread = scr_read(file_send, fd_send, buf_send, scr_mpi_buf_size);
if (scr_crc_on_copy && nread > 0) {
*crc32_send = crc32(*crc32_send, (const Bytef*) buf_send, (uInt) nread);
}
if (nread < 0) {
nread = 0;
}
MPI_Isend(buf_send, nread, MPI_BYTE, rank_send, 0, comm, &request[1]);
MPI_Wait(&request[1], &status[1]);
if (nread < scr_mpi_buf_size) {
sending = 0;
}
}
/* if we are still receiving a file,
* wait on our receive to complete and write the data */
if (receiving) {
MPI_Wait(&request[0], &status[0]);
MPI_Get_count(&status[0], MPI_BYTE, &nwrite);
if (scr_crc_on_copy && nwrite > 0) {
*crc32_recv = crc32(*crc32_recv, (const Bytef*) buf_recv, (uInt) nwrite);
}
scr_write(file_recv, fd_recv, buf_recv, nwrite);
if (nwrite < scr_mpi_buf_size) {
receiving = 0;
}
}
}
/* close the files */
if (have_outgoing) {
scr_close(file_send, fd_send);
}
if (have_incoming) {
scr_close(file_recv, fd_recv);
}
/* set crc field on our file if it hasn't been set already */
if (scr_crc_on_copy && have_outgoing) {
uLong meta_send_crc;
if (scr_meta_get_crc32(meta_send, &meta_send_crc) != SCR_SUCCESS) {
scr_meta_set_crc32(meta_send, *crc32_send);
} else {
/* TODO: we could check that the crc on the sent file matches and take some action if not */
}
}
/* free the MPI buffers */
scr_align_free(&buf_recv);
scr_align_free(&buf_send);
return rc;
}
static int scr_swap_files_move(
int have_outgoing, const char* file_send, scr_meta* meta_send, int rank_send, uLong* crc32_send,
int have_incoming, const char* file_recv, scr_meta* meta_recv, int rank_recv, uLong* crc32_recv,
MPI_Comm comm)
{
int rc = SCR_SUCCESS;
MPI_Request request[2];
MPI_Status status[2];
/* allocate MPI send buffer */
char *buf_send = NULL;
if (have_outgoing) {
buf_send = (char*) scr_align_malloc(scr_mpi_buf_size, scr_page_size);
if (buf_send == NULL) {
scr_abort(-1, "Allocating memory: malloc(%ld) errno=%d %s @ %s:%d",
scr_mpi_buf_size, errno, strerror(errno), __FILE__, __LINE__
);
return SCR_FAILURE;
}
}
/* allocate MPI recv buffer */
char *buf_recv = NULL;
if (have_incoming) {
buf_recv = (char*) scr_align_malloc(scr_mpi_buf_size, scr_page_size);
if (buf_recv == NULL) {
scr_abort(-1, "Allocating memory: malloc(%ld) errno=%d %s @ %s:%d",
scr_mpi_buf_size, errno, strerror(errno), __FILE__, __LINE__
);
return SCR_FAILURE;
}
}
/* since we'll overwrite our send file in place with the recv file,
* which may be larger, we need to keep track of how many bytes we've
* sent and whether we've sent them all */
unsigned long filesize_send = 0;
/* open our file */
int fd = -1;
if (have_outgoing) {
/* we'll overwrite our send file (or just read it if there is no incoming) */
filesize_send = scr_file_size(file_send);
fd = scr_open(file_send, O_RDWR);
if (fd < 0) {
/* TODO: skip writes and return error? */
scr_abort(-1, "Opening file for send/recv: scr_open(%s, O_RDWR) errno=%d %s @ %s:%d",
file_send, errno, strerror(errno), __FILE__, __LINE__
);
}
} else if (have_incoming) {
/* if we're in this branch, then we only have an incoming file,
* so we'll write our recv file from scratch */
mode_t mode_file = scr_getmode(1, 1, 0);
fd = scr_open(file_recv, O_WRONLY | O_CREAT | O_TRUNC, mode_file);
if (fd < 0) {
/* TODO: skip writes and return error? */
scr_abort(-1, "Opening file for recv: scr_open(%s, O_WRONLY | O_CREAT | O_TRUNC, ...) errno=%d %s @ %s:%d",
file_recv, errno, strerror(errno), __FILE__, __LINE__
);
}
}
/* exchange file chunks */
int sending = 0;
if (have_outgoing) {
sending = 1;
}
int receiving = 0;
if (have_incoming) {
receiving = 1;
}
int nread, nwrite;
off_t read_pos = 0, write_pos = 0;
while (sending || receiving) {
if (receiving) {
/* prepare a buffer to receive up to scr_mpi_buf_size bytes */
MPI_Irecv(buf_recv, scr_mpi_buf_size, MPI_BYTE, rank_recv, 0, comm, &request[0]);
}
if (sending) {
/* compute number of bytes to read */
unsigned long count = filesize_send - read_pos;
if (count > scr_mpi_buf_size) {
count = scr_mpi_buf_size;
}
/* read a chunk of up to scr_mpi_buf_size bytes into buf_send */
lseek(fd, read_pos, SEEK_SET); /* seek to read position */
nread = scr_read(file_send, fd, buf_send, count);
if (scr_crc_on_copy && nread > 0) {
*crc32_send = crc32(*crc32_send, (const Bytef*) buf_send, (uInt) nread);
}
if (nread < 0) {
nread = 0;
}
read_pos += (off_t) nread; /* update read pointer */
/* send chunk (if nread is smaller than scr_mpi_buf_size,
* then we've read the whole file) */
MPI_Isend(buf_send, nread, MPI_BYTE, rank_send, 0, comm, &request[1]);
MPI_Wait(&request[1], &status[1]);
/* check whether we've read the whole file */
if (filesize_send == read_pos && count < scr_mpi_buf_size) {
sending = 0;
}
}
if (receiving) {
/* count the number of bytes received */
MPI_Wait(&request[0], &status[0]);
MPI_Get_count(&status[0], MPI_BYTE, &nwrite);
if (scr_crc_on_copy && nwrite > 0) {
*crc32_recv = crc32(*crc32_recv, (const Bytef*) buf_recv, (uInt) nwrite);
}
/* write those bytes to file (if nwrite is smaller than scr_mpi_buf_size,
* then we've received the whole file) */
lseek(fd, write_pos, SEEK_SET); /* seek to write position */
scr_write(file_recv, fd, buf_recv, nwrite);
write_pos += (off_t) nwrite; /* update write pointer */
/* if nwrite is smaller than scr_mpi_buf_size,
* then assume we've received the whole file */
if (nwrite < scr_mpi_buf_size) {
receiving = 0;
}
}
}
/* close file and cleanup */
if (have_outgoing && have_incoming) {
/* sent and received a file; close it, truncate it to corect size, rename it */
scr_close(file_send, fd);
truncate(file_send, write_pos);
rename(file_send, file_recv);
} else if (have_outgoing) {
/* only sent a file; close it, delete it, and remove its completion marker */
scr_close(file_send, fd);
scr_file_unlink(file_send);
} else if (have_incoming) {
/* only received a file; just need to close it */
scr_close(file_recv, fd);
}
if (scr_crc_on_copy && have_outgoing) {
uLong meta_send_crc;
if (scr_meta_get_crc32(meta_send, &meta_send_crc) != SCR_SUCCESS) {
/* we transfer this meta data across below,
* so may as well update these fields so we can use them */
scr_meta_set_crc32(meta_send, *crc32_send);
/* do not complete file send, we just deleted it above */
} else {
/* TODO: we could check that the crc on the sent file matches and take some action if not */
}
}
/* free the MPI buffers */
scr_align_free(&buf_recv);
scr_align_free(&buf_send);
return rc;
}
int main (int argc, char *argv[])
{
/* check that we were given at least one argument
* (the transfer file name) */
if (argc != 2) {
printf("Usage: scr_transfer <transferfile>\n");
return 1;
}
/* record the name of the transfer file */
scr_transfer_file = strdup(argv[1]);
if (scr_transfer_file == NULL) {
scr_err("scr_transfer: Copying transfer file name @ %s:%d",
__FILE__, __LINE__
);
return 1;
}
/* initialize our tracking variables */
read_params();
/* get file io mode */
mode_t mode_file = scr_getmode(1, 1, 0);
/* we cache the opened file descriptors to avoid extra opens,
* seeks, and closes */
int fd_src = -1;
int fd_dst = -1;
char* new_file_src = NULL;
char* old_file_src = NULL;
char* new_file_dst = NULL;
char* old_file_dst = NULL;
off_t new_position = 0;
off_t old_position = 0;
/* start in the stopped state */
state = STOPPED;
set_transfer_file_state(SCR_TRANSFER_KEY_STATE_STOP, 0);
/* TODO: enable this value to be set from config file */
/* TODO: page-align this buffer for faster performance */
/* allocate our file copy buffer */
size_t bufsize = scr_file_buf_size;
char* buf = malloc(bufsize);
if (buf == NULL) {
scr_err("scr_transfer: Failed to allocate %llu bytes for file copy buffer @ %s:%d",
(unsigned long long) bufsize, __FILE__, __LINE__
);
return 1;
}
int nread = 0;
double secs_run = 0.0;
double secs_slept = 0.0;
double secs_run_start = scr_seconds();
double secs_run_end = secs_run_start;
double secs_last_write = secs_run_start;
scr_hash* hash = scr_hash_new();
while (keep_running) {
/* loop here sleeping and checking transfer file periodically
* until state changes and / or some time elapses */
/* reset our timer for our last write */
double secs_remain = scr_transfer_secs;
while (keep_running && (state == STOPPED || secs_remain > 0.0)) {
/* remember our current state before reading transfer file */
int old_state = state;
/* read the transfer file, which fills in our hash and
* also updates state and bytes_per_second */
scr_hash_delete(&hash);
hash = read_transfer_file();
/* compute time we should sleep before writing more data based
* on bandwidth and percent of runtime limits */
if (state == RUNNING) {
/* get the current time */
double secs_now = scr_seconds();
/* based on the amount we last wrote and our allocated bandwidth,
* compute time we need to sleep before attempting our next write */
double secs_remain_bw = 0.0;
if (nread > 0 && bytes_per_second > 0.0) {
double secs_to_wait_bw = (double) nread / bytes_per_second;
double secs_waited_bw = secs_now - secs_last_write;
secs_remain_bw = secs_to_wait_bw - secs_waited_bw;
}
/* based on the percentage of time we are allowed to be running,
* compute time we need to sleep before attempting our next write */
double secs_remain_runtime = 0.0;
if (percent_runtime > 0.0) {
/* stop the run clock, add to the run time,
* and restart the run clock */
secs_run_end = secs_now;
secs_run += secs_run_end - secs_run_start;
secs_run_start = secs_run_end;
/* compute our total time, and the time we need to sleep */
double secs_total = secs_run + secs_slept;
secs_remain_runtime = secs_run / percent_runtime - secs_total;
}
/* take the maximum of these two values */
secs_remain = secs_remain_bw;
if (secs_remain_runtime > secs_remain) {
secs_remain = secs_remain_runtime;
}
}
/* check for a state transition */
if (state != old_state) {
if (state == RUNNING) {
/* if we switched to RUNNING, kick out without sleeping and
* reset the total run and sleep times */
secs_remain = 0.0;
secs_run = 0.0;
secs_slept = 0.0;
} else if (state == STOPPED) {
/* if we switched to STOPPED, close our files if open */
close_files(new_file_src, &fd_src, new_file_dst, &fd_dst);
clear_parameters(&new_file_src, &new_file_dst, &new_position);
clear_parameters(&old_file_src, &old_file_dst, &old_position);
/* after closing our files, update our state in the transfer file */
set_transfer_file_state(SCR_TRANSFER_KEY_STATE_STOP, 0);
}
}
/* assume we can sleep for the full remainder of the time */
double secs = secs_remain;
/* if we're not running, always sleep for the full time */
if (state != RUNNING) {
secs = scr_transfer_secs;
}
/* set a maximum time to sleep before we read the hash file again
* (ensures some responsiveness) */
if (secs > scr_transfer_secs) {
secs = scr_transfer_secs;
}
/* sleep if we need to */
if (secs > 0.0) {
/* stop the run clock and add to the total run time */
secs_run_end = scr_seconds();
secs_run += secs_run_end - secs_run_start;
/* sleep */
usleep((unsigned long) (secs * 1000000.0));
secs_slept += secs;
secs_remain -= secs;
/* restart the run clock */
secs_run_start = scr_seconds();
}
}
/* write data out */
if (state == RUNNING) {
/* look for a new file to transfer */
off_t filesize = 0;
find_file(hash, &new_file_src, &new_file_dst, &new_position, &filesize);
/* if we got a new file, close the old one (if open),
* open the new file */
if (bool_diff_files(new_file_src, old_file_src)) {
/* close the old descriptor if it's open */
if (fd_src >= 0) {
scr_close(old_file_src, fd_src);
fd_src = -1;
}
/* delete the old file name if we have one */
if (old_file_src != NULL) {
free(old_file_src);
old_file_src = NULL;
}
/* reset our position counter */
old_position = 0;
/* open the file and remember the filename if we have one */
if (new_file_src != NULL) {
fd_src = scr_open(new_file_src, O_RDONLY);
/* TODO: check for errors here */
old_file_src = strdup(new_file_src);
/* TODO: check for errors here */
}
}
/* if we got a new file, close the old one (if open),
* open the new file */
if (bool_diff_files(new_file_dst, old_file_dst)) {
/* close the old descriptor if it's open */
if (fd_dst >= 0) {
scr_close(old_file_dst, fd_dst);
fd_dst = -1;
}
/* delete the old file name if we have one */
if (old_file_dst != NULL) {
free(old_file_dst);
old_file_dst = NULL;
}
/* reset our position counter */
old_position = 0;
/* open the file and remember the filename if we have one */
if (new_file_dst != NULL) {
fd_dst = scr_open(new_file_dst, O_RDWR | O_CREAT, mode_file);
/* TODO: check for errors here */
old_file_dst = strdup(new_file_dst);
/* TODO: check for errors here */
}
}
/* we may have the same file, but perhaps the position changed
* (may need to seek) */
if (new_position != old_position) {
if (fd_src >= 0) {
lseek(fd_src, new_position, SEEK_SET);
/* TODO: check for errors here */
}
if (fd_dst >= 0) {
lseek(fd_dst, new_position, SEEK_SET);
/* TODO: check for errors here */
}
/* remember the new position */
old_position = new_position;
}
/* if we have two open files,
* copy a chunk from source file to destination file */
nread = 0;
if (fd_src >= 0 && fd_dst >= 0) {
/* compute number of bytes to read from file */
size_t count = (size_t) (filesize - new_position);
if (count > bufsize) {
count = bufsize;
}
/* read a chunk */
nread = scr_read(new_file_src, fd_src, buf, count);
/* if we read data, write it out */
if (nread > 0) {
/* record the time of our write */
secs_last_write = scr_seconds();
/* write the chunk and force it out with an fsync */
scr_write(new_file_dst, fd_dst, buf, nread);
fsync(fd_dst);
/* update our position */
new_position += (off_t) nread;
old_position = new_position;
/* record the updated position in the transfer file */
update_transfer_file(new_file_src, new_file_dst, new_position);
}
/* if we've written all of the bytes, close the files */
if (new_position == filesize) {
close_files(new_file_src, &fd_src, new_file_dst, &fd_dst);
clear_parameters(&new_file_src, &new_file_dst, &new_position);
clear_parameters(&old_file_src, &old_file_dst, &old_position);
}
} else {
/* TODO: we may have an error
* (failed to open the source or dest file) */
/* if we found no file to transfer, move to a STOPPED state */
if (new_file_src == NULL) {
state = STOPPED;
set_transfer_file_state(SCR_TRANSFER_KEY_STATE_STOP, 1);
}
}
}
}
/* free our file copy buffer */
if (buf != NULL) {
free(buf);
buf = NULL;
}
/* free the strdup'd tranfer file name */
if (scr_transfer_file != NULL) {
free(scr_transfer_file);
scr_transfer_file = NULL;
}
return 0;
}