void DTAR_enqueue_copy(CIRCLE_handle *handle) {
for (uint64_t idx = 0; idx < DTAR_count; idx++) {
mfu_filetype type = mfu_flist_file_get_type(DTAR_flist, idx);
/* add copy work only for files */
if (type == MFU_TYPE_FILE) {
uint64_t dataoffset = DTAR_offsets[idx] + DTAR_HDR_LENGTH;
const char * name = mfu_flist_file_get_name(DTAR_flist, idx);
uint64_t size = mfu_flist_file_get_size(DTAR_flist, idx);
/* compute number of chunks */
uint64_t num_chunks = size / DTAR_user_opts.chunk_size;
for (uint64_t chunk_idx = 0; chunk_idx < num_chunks; chunk_idx++) {
char* newop = DTAR_encode_operation(
COPY_DATA, name, size, chunk_idx, dataoffset);
handle->enqueue(newop);
mfu_free(&newop);
}
/* create copy work for possibly last item */
if (num_chunks * DTAR_user_opts.chunk_size < size || num_chunks == 0) {
char* newop = DTAR_encode_operation(
COPY_DATA, name, size, num_chunks, dataoffset);
handle->enqueue(newop);
mfu_free(&newop);
}
}
}
}
/* for given depth, just remove the files we know about */
static void remove_direct(mfu_flist list, uint64_t* rmcount)
{
/* each process directly removes its elements */
uint64_t idx;
uint64_t size = mfu_flist_size(list);
for (idx = 0; idx < size; idx++) {
/* get name and type of item */
const char* name = mfu_flist_file_get_name(list, idx);
mfu_filetype type = mfu_flist_file_get_type(list, idx);
/* delete item */
if (type == MFU_TYPE_DIR) {
remove_type('d', name);
}
else if (type == MFU_TYPE_FILE || type == MFU_TYPE_LINK) {
remove_type('f', name);
}
else {
remove_type('u', name);
}
}
/* report the number of items we deleted */
*rmcount = size;
return;
}
/* filter the list of files down based on the current stripe size and stripe count */
static mfu_flist filter_list(mfu_flist list, int stripe_count, uint64_t stripe_size, uint64_t min_size, uint64_t* total_count, uint64_t* total_size)
{
/* initialize counters for file and byte count */
uint64_t my_count = 0;
uint64_t my_size = 0;
/* this is going to be a subset of the full file list */
mfu_flist filtered = mfu_flist_subset(list);
uint64_t idx;
uint64_t size = mfu_flist_size(list);
for (idx = 0; idx < size; idx++) {
/* we only care about regular files */
mfu_filetype type = mfu_flist_file_get_type(list, idx);
if (type == MFU_TYPE_FILE) {
/* if our file is below the minimum file size, skip it */
uint64_t filesize = mfu_flist_file_get_size(list, idx);
if (filesize < min_size) {
continue;
}
const char* in_path = mfu_flist_file_get_name(list, idx);
uint64_t curr_stripe_size = 0;
uint64_t curr_stripe_count = 0;
/*
* attempt to get striping info,
* skip the file if we can't get the striping info we seek
*/
if (mfu_stripe_get(in_path, &curr_stripe_size, &curr_stripe_count) != 0) {
continue;
}
/* TODO: this should probably be better */
/* if the current stripe size or stripe count doesn't match, then a restripe the file */
if (curr_stripe_count != stripe_count || curr_stripe_size != stripe_size) {
mfu_flist_file_copy(list, idx, filtered);
/* increment file count and add file size to our running total */
my_count += 1;
my_size += filesize;
}
}
}
/* summarize and return the new list */
mfu_flist_summarize(filtered);
/* get sum of count and size */
MPI_Allreduce(&my_count, total_count, 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);
MPI_Allreduce(&my_size, total_size, 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);
return filtered;
}
/* print to stdout the stripe size and count of each file in the mfu_flist */
static void stripe_info_report(mfu_flist list)
{
uint64_t idx;
uint64_t size = mfu_flist_size(list);
int rank;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
/* print header */
if (rank == 0) {
printf("%10s %3.3s %8.8s %s\n", "Size", "Cnt", "Str Size", "File Path");
printf("%10s %3.3s %8.8s %s\n", "----", "---", "--------", "---------");
fflush(stdout);
}
MPI_Barrier(MPI_COMM_WORLD);
/* print out file info */
for (idx = 0; idx < size; idx++) {
mfu_filetype type = mfu_flist_file_get_type(list, idx);
/* report striping information for regular files only */
if (type == MFU_TYPE_FILE) {
const char* in_path = mfu_flist_file_get_name(list, idx);
uint64_t stripe_size = 0;
uint64_t stripe_count = 0;
char filesize[11];
char stripesize[9];
/*
* attempt to get striping info and print it out,
* skip the file if we can't get the striping info we seek
*/
if (mfu_stripe_get(in_path, &stripe_size, &stripe_count) != 0) {
continue;
}
/* format it nicely */
generate_pretty_size(filesize, sizeof(filesize), mfu_flist_file_get_size(list, idx));
generate_pretty_size(stripesize, sizeof(stripesize), stripe_size);
/* print the row */
printf("%10.10s %3" PRId64 " %8.8s %s\n", filesize, stripe_count, stripesize, in_path);
fflush(stdout);
}
}
}
/* TODO: condionally set setuid and setgid bits? */
void DCOPY_copy_permissions(
mfu_flist flist,
uint64_t idx,
const char* dest_path)
{
/* get mode and type */
mfu_filetype type = mfu_flist_file_get_type(flist, idx);
mode_t mode = (mode_t) mfu_flist_file_get_mode(flist, idx);
/* change mode */
if(type != MFU_TYPE_LINK) {
if(mfu_chmod(dest_path, mode) != 0) {
MFU_LOG(MFU_LOG_ERR, "Failed to change permissions on %s chmod() errno=%d %s",
dest_path, errno, strerror(errno)
);
}
}
return;
}
static void remove_create(CIRCLE_handle* handle)
{
char path[CIRCLE_MAX_STRING_LEN];
/* enqueues all items at rm_depth to be deleted */
uint64_t idx;
uint64_t size = mfu_flist_size(circle_list);
for (idx = 0; idx < size; idx++) {
/* get name and type of item */
const char* name = mfu_flist_file_get_name(circle_list, idx);
mfu_filetype type = mfu_flist_file_get_type(circle_list, idx);
/* encode type */
if (type == MFU_TYPE_DIR) {
path[0] = 'd';
}
else if (type == MFU_TYPE_FILE || type == MFU_TYPE_LINK) {
path[0] = 'f';
}
else {
path[0] = 'u';
}
/* encode name */
size_t len = strlen(name) + 2;
if (len <= CIRCLE_MAX_STRING_LEN) {
strcpy(&path[1], name);
handle->enqueue(path);
}
else {
MFU_LOG(MFU_LOG_ERR, "Filename longer than %lu",
(unsigned long)CIRCLE_MAX_STRING_LEN
);
}
}
return;
}
/* removes list of items, sets write bits on directories from
* top-to-bottom, then removes items one level at a time starting
* from the deepest */
void mfu_flist_unlink(mfu_flist flist)
{
int level;
/* wait for all tasks and start timer */
MPI_Barrier(MPI_COMM_WORLD);
double start_remove = MPI_Wtime();
/* split files into separate lists by directory depth */
int levels, minlevel;
mfu_flist* lists;
mfu_flist_array_by_depth(flist, &levels, &minlevel, &lists);
#if 0
/* dive from shallow to deep, ensure all directories have write bit set */
for (level = 0; level < levels; level++) {
/* get list of items for this level */
mfu_flist list = lists[level];
/* determine whether we have details at this level */
int detail = mfu_flist_have_detail(list);
/* iterate over items and set write bit on directories if needed */
uint64_t idx;
uint64_t size = mfu_flist_size(list);
for (idx = 0; idx < size; idx++) {
/* check whether we have a directory */
mfu_filetype type = mfu_flist_file_get_type(list, idx);
if (type == MFU_TYPE_DIR) {
/* assume we have to set the bit */
int set_write_bit = 1;
if (detail) {
mode_t mode = (mode_t) mfu_flist_file_get_mode(list, idx);
if (mode & S_IWUSR) {
/* we have the mode of the file, and the bit is already set */
set_write_bit = 0;
}
}
/* set the bit if needed */
if (set_write_bit) {
const char* name = mfu_flist_file_get_name(list, idx);
int rc = chmod(name, S_IRWXU);
if (rc != 0) {
MFU_LOG(MFU_LOG_ERR, "Failed to chmod directory `%s' (errno=%d %s)",
name, errno, strerror(errno)
);
}
}
}
}
/* wait for all procs to finish before we start next level */
MPI_Barrier(MPI_COMM_WORLD);
}
#endif
/* now remove files starting from deepest level */
for (level = levels - 1; level >= 0; level--) {
double start = MPI_Wtime();
/* get list of items for this level */
mfu_flist list = lists[level];
uint64_t count = 0;
//remove_direct(list, &count);
remove_spread(list, &count);
// remove_map(list, &count);
// remove_sort(list, &count);
// remove_libcircle(list, &count);
// TODO: remove sort w/ spread
/* wait for all procs to finish before we start
* with files at next level */
MPI_Barrier(MPI_COMM_WORLD);
double end = MPI_Wtime();
if (mfu_debug_level >= MFU_LOG_VERBOSE) {
uint64_t min, max, sum;
MPI_Allreduce(&count, &min, 1, MPI_UINT64_T, MPI_MIN, MPI_COMM_WORLD);
MPI_Allreduce(&count, &max, 1, MPI_UINT64_T, MPI_MAX, MPI_COMM_WORLD);
MPI_Allreduce(&count, &sum, 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);
double rate = 0.0;
if (end - start > 0.0) {
rate = (double)sum / (end - start);
}
double time_diff = end - start;
if (mfu_rank == 0) {
printf("level=%d min=%lu max=%lu sum=%lu rate=%f secs=%f\n",
(minlevel + level), (unsigned long)min, (unsigned long)max, (unsigned long)sum, rate, time_diff
);
fflush(stdout);
}
}
}
mfu_flist_array_free(levels, &lists);
/* wait for all tasks and stop timer */
MPI_Barrier(MPI_COMM_WORLD);
double end_remove = MPI_Wtime();
/* report remove count, time, and rate */
if (mfu_debug_level >= MFU_LOG_VERBOSE && mfu_rank == 0) {
uint64_t all_count = mfu_flist_global_size(flist);
double time_diff = end_remove - start_remove;
double rate = 0.0;
if (time_diff > 0.0) {
rate = ((double)all_count) / time_diff;
}
printf("Removed %lu items in %f seconds (%f items/sec)\n",
all_count, time_diff, rate
);
}
return;
}
/* for each depth, sort files by filename and then remove, to test
* whether it matters to limit the number of directories each process
* has to reference (e.g., locking) */
static void remove_sort(mfu_flist list, uint64_t* rmcount)
{
/* bail out if total count is 0 */
uint64_t all_count = mfu_flist_global_size(list);
if (all_count == 0) {
return;
}
/* get maximum file name and number of items */
int chars = (int) mfu_flist_file_max_name(list);
uint64_t my_count = mfu_flist_size(list);
/* create key datatype (filename) and comparison op */
MPI_Datatype dt_key;
DTCMP_Op op_str;
DTCMP_Str_create_ascend(chars, &dt_key, &op_str);
/* create keysat datatype (filename + type) */
MPI_Datatype types[2], dt_keysat;
types[0] = dt_key;
types[1] = MPI_CHAR;
DTCMP_Type_create_series(2, types, &dt_keysat);
/* allocate send buffer */
int sendcount = (int) my_count;
size_t sendbufsize = (size_t)(sendcount * (chars + 1));
char* sendbuf = (char*) MFU_MALLOC(sendbufsize);
/* copy data into buffer */
char* ptr = sendbuf;
uint64_t idx;
for (idx = 0; idx < my_count; idx++) {
/* encode the filename first */
const char* name = mfu_flist_file_get_name(list, idx);
strcpy(ptr, name);
ptr += chars;
/* last character encodes item type */
mfu_filetype type = mfu_flist_file_get_type(list, idx);
if (type == MFU_TYPE_DIR) {
ptr[0] = 'd';
}
else if (type == MFU_TYPE_FILE || type == MFU_TYPE_LINK) {
ptr[0] = 'f';
}
else {
ptr[0] = 'u';
}
ptr++;
}
/* sort items */
void* recvbuf;
int recvcount;
DTCMP_Handle handle;
DTCMP_Sortz(
sendbuf, sendcount, &recvbuf, &recvcount,
dt_key, dt_keysat, op_str, DTCMP_FLAG_NONE, MPI_COMM_WORLD, &handle
);
/* delete data */
int delcount = 0;
ptr = (char*)recvbuf;
while (delcount < recvcount) {
/* get item name */
char* name = ptr;
ptr += chars;
/* get item type */
char type = ptr[0];
ptr++;
/* delete item */
remove_type(type, name);
delcount++;
}
/* record number of items we deleted */
*rmcount = (uint64_t) delcount;
/* free output data */
DTCMP_Free(&handle);
/* free our send buffer */
mfu_free(&sendbuf);
/* free key comparison operation */
DTCMP_Op_free(&op_str);
/* free datatypes */
MPI_Type_free(&dt_keysat);
MPI_Type_free(&dt_key);
return;
}
/* for given depth, evenly spread the files among processes for
* improved load balancing */
static void remove_spread(mfu_flist flist, uint64_t* rmcount)
{
uint64_t idx;
/* initialize our remove count */
*rmcount = 0;
/* get our rank and number of ranks in job */
int rank, ranks;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &ranks);
/* allocate memory for alltoall exchanges */
size_t bufsize = (size_t)ranks * sizeof(int);
int* sendcounts = (int*) MFU_MALLOC(bufsize);
int* sendsizes = (int*) MFU_MALLOC(bufsize);
int* senddisps = (int*) MFU_MALLOC(bufsize);
int* recvsizes = (int*) MFU_MALLOC(bufsize);
int* recvdisps = (int*) MFU_MALLOC(bufsize);
/* get number of items */
uint64_t my_count = mfu_flist_size(flist);
uint64_t all_count = mfu_flist_global_size(flist);
uint64_t offset = mfu_flist_global_offset(flist);
/* compute number of bytes we'll send */
size_t sendbytes = 0;
for (idx = 0; idx < my_count; idx++) {
const char* name = mfu_flist_file_get_name(flist, idx);
size_t len = strlen(name) + 2;
sendbytes += len;
}
/* compute the number of items that each rank should have */
uint64_t low = all_count / (uint64_t)ranks;
uint64_t extra = all_count - low * (uint64_t)ranks;
/* compute number that we'll send to each rank and initialize sendsizes and offsets */
uint64_t i;
for (i = 0; i < (uint64_t)ranks; i++) {
/* compute starting element id and count for given rank */
uint64_t start, num;
if (i < extra) {
num = low + 1;
start = i * num;
}
else {
num = low;
start = (i - extra) * num + extra * (low + 1);
}
/* compute the number of items we'll send to this task */
uint64_t sendcnt = 0;
if (my_count > 0) {
if (start <= offset && offset < start + num) {
/* this rank overlaps our range,
* and its first element comes at or before our first element */
sendcnt = num - (offset - start);
if (my_count < sendcnt) {
/* the number the rank could receive from us
* is more than we have left */
sendcnt = my_count;
}
}
else if (offset < start && start < offset + my_count) {
/* this rank overlaps our range,
* and our first element comes strictly before its first element */
sendcnt = my_count - (start - offset);
if (num < sendcnt) {
/* the number the rank can receive from us
* is less than we have left */
sendcnt = num;
}
}
}
/* record the number of items we'll send to this task */
sendcounts[i] = (int) sendcnt;
/* set sizes and displacements to 0, we'll fix this later */
sendsizes[i] = 0;
senddisps[i] = 0;
}
/* allocate space */
char* sendbuf = (char*) MFU_MALLOC(sendbytes);
/* copy data into buffer */
int dest = -1;
int disp = 0;
for (idx = 0; idx < my_count; idx++) {
/* get name and type of item */
const char* name = mfu_flist_file_get_name(flist, idx);
mfu_filetype type = mfu_flist_file_get_type(flist, idx);
/* get rank that we're packing data for */
if (dest == -1) {
dest = get_first_nonzero(sendcounts, ranks);
if (dest == -1) {
/* error */
}
/* about to copy first item for this rank,
* record its displacement */
senddisps[dest] = disp;
}
/* identify region to be sent to rank */
char* path = sendbuf + disp;
/* first character encodes item type */
if (type == MFU_TYPE_DIR) {
path[0] = 'd';
}
else if (type == MFU_TYPE_FILE || type == MFU_TYPE_LINK) {
path[0] = 'f';
}
else {
path[0] = 'u';
}
/* now copy in the path */
strcpy(&path[1], name);
/* TODO: check that we don't overflow the int */
/* add bytes to sendsizes and increase displacement */
size_t count = strlen(name) + 2;
sendsizes[dest] += (int) count;
disp += (int) count;
/* decrement the count for this rank */
sendcounts[dest]--;
if (sendcounts[dest] == 0) {
dest = -1;
}
}
/* compute displacements */
senddisps[0] = 0;
for (i = 1; i < (uint64_t)ranks; i++) {
senddisps[i] = senddisps[i - 1] + sendsizes[i - 1];
}
/* alltoall to specify incoming counts */
MPI_Alltoall(sendsizes, 1, MPI_INT, recvsizes, 1, MPI_INT, MPI_COMM_WORLD);
/* compute size of recvbuf and displacements */
size_t recvbytes = 0;
recvdisps[0] = 0;
for (i = 0; i < (uint64_t)ranks; i++) {
recvbytes += (size_t) recvsizes[i];
if (i > 0) {
recvdisps[i] = recvdisps[i - 1] + recvsizes[i - 1];
}
}
/* allocate recvbuf */
char* recvbuf = (char*) MFU_MALLOC(recvbytes);
/* alltoallv to send data */
MPI_Alltoallv(
sendbuf, sendsizes, senddisps, MPI_CHAR,
recvbuf, recvsizes, recvdisps, MPI_CHAR, MPI_COMM_WORLD
);
/* delete data */
char* item = recvbuf;
while (item < recvbuf + recvbytes) {
/* get item name and type */
char type = item[0];
char* name = &item[1];
/* delete item */
remove_type(type, name);
/* keep tally of number of items we deleted */
*rmcount++;
/* go to next item */
size_t item_size = strlen(item) + 1;
item += item_size;
}
/* free memory */
mfu_free(&recvbuf);
mfu_free(&recvdisps);
mfu_free(&recvsizes);
mfu_free(&sendbuf);
mfu_free(&senddisps);
mfu_free(&sendsizes);
mfu_free(&sendcounts);
return;
}
/* print information about a file given the index and rank (used in print_files) */
static void print_file(mfu_flist flist, uint64_t idx)
{
/* store types as strings for print_file */
char type_str_unknown[] = "UNK";
char type_str_dir[] = "DIR";
char type_str_file[] = "REG";
char type_str_link[] = "LNK";
/* get filename */
const char* file = mfu_flist_file_get_name(flist, idx);
if (mfu_flist_have_detail(flist)) {
/* get mode */
mode_t mode = (mode_t) mfu_flist_file_get_mode(flist, idx);
//uint32_t uid = (uint32_t) mfu_flist_file_get_uid(flist, idx);
//uint32_t gid = (uint32_t) mfu_flist_file_get_gid(flist, idx);
uint64_t acc = mfu_flist_file_get_atime(flist, idx);
uint64_t mod = mfu_flist_file_get_mtime(flist, idx);
uint64_t cre = mfu_flist_file_get_ctime(flist, idx);
uint64_t size = mfu_flist_file_get_size(flist, idx);
const char* username = mfu_flist_file_get_username(flist, idx);
const char* groupname = mfu_flist_file_get_groupname(flist, idx);
char access_s[30];
char modify_s[30];
char create_s[30];
time_t access_t = (time_t) acc;
time_t modify_t = (time_t) mod;
time_t create_t = (time_t) cre;
size_t access_rc = strftime(access_s, sizeof(access_s) - 1, "%FT%T", localtime(&access_t));
//size_t modify_rc = strftime(modify_s, sizeof(modify_s) - 1, "%FT%T", localtime(&modify_t));
size_t modify_rc = strftime(modify_s, sizeof(modify_s) - 1, "%b %e %Y %H:%M", localtime(&modify_t));
size_t create_rc = strftime(create_s, sizeof(create_s) - 1, "%FT%T", localtime(&create_t));
if (access_rc == 0 || modify_rc == 0 || create_rc == 0) {
/* error */
access_s[0] = '\0';
modify_s[0] = '\0';
create_s[0] = '\0';
}
char mode_format[11];
mfu_format_mode(mode, mode_format);
double size_tmp;
const char* size_units;
mfu_format_bytes(size, &size_tmp, &size_units);
printf("%s %s %s %7.3f %2s %s %s\n",
mode_format, username, groupname,
size_tmp, size_units, modify_s, file
);
#if 0
printf("%s %s %s A%s M%s C%s %lu %s\n",
mode_format, username, groupname,
access_s, modify_s, create_s, (unsigned long)size, file
);
printf("Mode=%lx(%s) UID=%d(%s) GUI=%d(%s) Access=%s Modify=%s Create=%s Size=%lu File=%s\n",
(unsigned long)mode, mode_format, uid, username, gid, groupname,
access_s, modify_s, create_s, (unsigned long)size, file
);
#endif
}
else {
/* get type */
mfu_filetype type = mfu_flist_file_get_type(flist, idx);
char* type_str = type_str_unknown;
if (type == MFU_TYPE_DIR) {
type_str = type_str_dir;
}
else if (type == MFU_TYPE_FILE) {
type_str = type_str_file;
}
else if (type == MFU_TYPE_LINK) {
type_str = type_str_link;
}
printf("Type=%s File=%s\n",
type_str, file
);
}
}
static void mfu_flist_archive_create_libcircle(mfu_flist flist, const char* archivefile, mfu_archive_options_t* opts)
{
DTAR_flist = flist;
DTAR_user_opts = *opts;
MPI_Comm_rank(MPI_COMM_WORLD, &DTAR_rank);
/* TODO: stripe the archive file if on parallel file system */
/* init statistics */
DTAR_statistics.total_dirs = 0;
DTAR_statistics.total_files = 0;
DTAR_statistics.total_links = 0;
DTAR_statistics.total_size = 0;
DTAR_statistics.total_bytes_copied = 0;
time(&(DTAR_statistics.time_started));
DTAR_statistics.wtime_started = MPI_Wtime();
/* create the archive file */
DTAR_writer.name = archivefile;
DTAR_writer.flags = O_WRONLY | O_CREAT | O_CLOEXEC | O_LARGEFILE;
DTAR_writer.fd_tar = open(archivefile, DTAR_writer.flags, 0664);
/* get number of items in our portion of the list */
DTAR_count = mfu_flist_size(DTAR_flist);
/* allocate memory for file sizes and offsets */
uint64_t* fsizes = (uint64_t*) MFU_MALLOC(DTAR_count * sizeof(uint64_t));
DTAR_offsets = (uint64_t*) MFU_MALLOC(DTAR_count * sizeof(uint64_t));
/* compute local offsets for each item and total
* bytes we're contributing to the archive */
uint64_t idx;
uint64_t offset = 0;
for (idx = 0; idx < DTAR_count; idx++) {
/* assume the item takes no space */
fsizes[idx] = 0;
/* identify item type to compute its size in the archive */
mfu_filetype type = mfu_flist_file_get_type(DTAR_flist, idx);
if (type == MFU_TYPE_DIR || type == MFU_TYPE_LINK) {
/* directories and symlinks only need the header */
fsizes[idx] = DTAR_HDR_LENGTH;
} else if (type == MFU_TYPE_FILE) {
/* regular file requires a header, plus file content,
* and things are packed into blocks of 512 bytes */
uint64_t fsize = mfu_flist_file_get_size(DTAR_flist, idx);
/* determine whether file size is integer multiple of 512 bytes */
uint64_t rem = fsize % 512;
if (rem == 0) {
/* file content is multiple of 512 bytes, so perfect fit */
fsizes[idx] = fsize + DTAR_HDR_LENGTH;
} else {
/* TODO: check and explain this math */
fsizes[idx] = (fsize / 512 + 4) * 512;
}
}
/* increment our local offset for this item */
DTAR_offsets[idx] = offset;
offset += fsizes[idx];
}
/* execute scan to figure our global base offset in the archive file */
uint64_t global_offset = 0;
MPI_Scan(&offset, &global_offset, 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);
global_offset -= offset;
/* update offsets for each of our file to their global offset */
for (idx = 0; idx < DTAR_count; idx++) {
DTAR_offsets[idx] += global_offset;
}
/* create an archive */
struct archive* ar = archive_write_new();
archive_write_set_format_pax(ar);
int r = archive_write_open_fd(ar, DTAR_writer.fd_tar);
if (r != ARCHIVE_OK) {
MFU_LOG(MFU_LOG_ERR, "archive_write_open_fd(): %s", archive_error_string(ar));
DTAR_abort(EXIT_FAILURE);
}
/* write headers for our files */
for (idx = 0; idx < DTAR_count; idx++) {
mfu_filetype type = mfu_flist_file_get_type(DTAR_flist, idx);
if (type == MFU_TYPE_FILE || type == MFU_TYPE_DIR || type == MFU_TYPE_LINK) {
DTAR_write_header(ar, idx, DTAR_offsets[idx]);
}
}
/* prepare libcircle */
CIRCLE_init(0, NULL, CIRCLE_SPLIT_EQUAL | CIRCLE_CREATE_GLOBAL);
CIRCLE_loglevel loglevel = CIRCLE_LOG_WARN;
CIRCLE_enable_logging(loglevel);
/* register callbacks */
CIRCLE_cb_create(&DTAR_enqueue_copy);
CIRCLE_cb_process(&DTAR_perform_copy);
/* run the libcircle job to copy data into archive file */
CIRCLE_begin();
CIRCLE_finalize();
/* compute total bytes copied */
uint64_t archive_size = 0;
MPI_Allreduce(&offset, &archive_size, 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);
DTAR_statistics.total_size = archive_size;
DTAR_statistics.wtime_ended = MPI_Wtime();
time(&(DTAR_statistics.time_ended));
/* print stats */
double rel_time = DTAR_statistics.wtime_ended - \
DTAR_statistics.wtime_started;
if (DTAR_rank == 0) {
char starttime_str[256];
struct tm* localstart = localtime(&(DTAR_statistics.time_started));
strftime(starttime_str, 256, "%b-%d-%Y, %H:%M:%S", localstart);
char endtime_str[256];
struct tm* localend = localtime(&(DTAR_statistics.time_ended));
strftime(endtime_str, 256, "%b-%d-%Y, %H:%M:%S", localend);
/* add two 512 blocks at the end */
DTAR_statistics.total_size += 512 * 2;
/* convert bandwidth to unit */
double agg_rate_tmp;
double agg_rate = (double) DTAR_statistics.total_size / rel_time;
const char* agg_rate_units;
mfu_format_bytes(agg_rate, &agg_rate_tmp, &agg_rate_units);
MFU_LOG(MFU_LOG_INFO, "Started: %s", starttime_str);
MFU_LOG(MFU_LOG_INFO, "Completed: %s", endtime_str);
MFU_LOG(MFU_LOG_INFO, "Total archive size: %" PRIu64, DTAR_statistics.total_size);
MFU_LOG(MFU_LOG_INFO, "Rate: %.3lf %s " \
"(%.3" PRIu64 " bytes in %.3lf seconds)", \
agg_rate_tmp, agg_rate_units, DTAR_statistics.total_size, rel_time);
}
/* clean up */
mfu_free(&fsizes);
mfu_free(&DTAR_offsets);
/* close archive file */
archive_write_free(ar);
mfu_close(DTAR_writer.name, DTAR_writer.fd_tar);
}
