void DTAR_epilogue() {
double rel_time = DTAR_statistics.wtime_ended - \
DTAR_statistics.wtime_started;
if (DTAR_rank == 0) {
char starttime_str[256];
struct tm* localstart = localtime(&(DTAR_statistics.time_started));
strftime(starttime_str, 256, "%b-%d-%Y, %H:%M:%S", localstart);
char endtime_str[256];
struct tm* localend = localtime(&(DTAR_statistics.time_ended));
strftime(endtime_str, 256, "%b-%d-%Y, %H:%M:%S", localend);
/* add two 512 blocks at the end */
DTAR_statistics.total_size += 512*2;
/* convert bandwidth to unit */
double agg_rate_tmp;
double agg_rate = (double) DTAR_statistics.total_size / rel_time;
const char* agg_rate_units;
mfu_format_bytes(agg_rate, &agg_rate_tmp, &agg_rate_units);
MFU_LOG(MFU_LOG_INFO, "Started: %s", starttime_str);
MFU_LOG(MFU_LOG_INFO, "Completed: %s", endtime_str);
MFU_LOG(MFU_LOG_INFO, "Total archive size: %" PRIu64, DTAR_statistics.total_size);
MFU_LOG(MFU_LOG_INFO, "Rate: %.3lf %s " \
"(%.3" PRIu64 " bytes in %.3lf seconds)", \
agg_rate_tmp, agg_rate_units, DTAR_statistics.total_size, rel_time);
}
}
static void generate_pretty_size(char *out, unsigned int length, uint64_t size)
{
double size_tmp;
const char* size_units;
char *unit;
unsigned int unit_len;
mfu_format_bytes(size, &size_tmp, &size_units);
snprintf(out, length, "%.2f %s", size_tmp, size_units);
}
static void create_default_separators(struct distribute_option *option,
mfu_flist* flist,
uint64_t* size,
int* separators,
uint64_t* global_max_file_size)
{
/* get local max file size for Allreduce */
uint64_t local_max_file_size = 0;
for (int i = 0; i < *size; i++) {
uint64_t file_size = mfu_flist_file_get_size(*flist, i);
if (file_size > local_max_file_size) {
local_max_file_size = file_size;
}
}
/* get the max file size across all ranks */
MPI_Allreduce(&local_max_file_size, global_max_file_size, 1,
MPI_UINT64_T, MPI_MAX, MPI_COMM_WORLD);
/* print and convert max file size to appropriate units */
double max_size_tmp;
const char* max_size_units;
mfu_format_bytes(*global_max_file_size, &max_size_tmp, &max_size_units);
printf("Max File Size: %.3lf %s\n", max_size_tmp, max_size_units);
/* round next_pow_2 to next multiple of 10 */
uint64_t max_magnitude_bin = (ceil(log2(*global_max_file_size) / 10 )) * 10;
/* get bin ranges based on max file size */
option->separators[0] = 1;
/* plus one is for zero count bin */
*separators = max_magnitude_bin / 10;
int power = 10;
for (int i = 1; power <= max_magnitude_bin; i++) {
option->separators[i] = pow(2, power);
power+=10;
}
}
static void stripe_progress_fn(const uint64_t* vals, int count, int complete, int ranks, double secs)
{
/* compute percentage of items removed */
double percent = 0.0;
if (stripe_prog_bytes_total > 0) {
percent = 100.0 * (double)vals[0] / (double)stripe_prog_bytes_total;
}
/* compute average delete rate */
double rate = 0.0;
if (secs > 0) {
rate = (double)vals[0] / secs;
}
/* compute estimated time remaining */
double secs_remaining = -1.0;
if (rate > 0.0) {
secs_remaining = (double)(stripe_prog_bytes_total - vals[0]) / rate;
}
/* convert bytes to units */
double agg_size_tmp;
const char* agg_size_units;
mfu_format_bytes(vals[0], &agg_size_tmp, &agg_size_units);
/* convert bandwidth to units */
double agg_rate_tmp;
const char* agg_rate_units;
mfu_format_bw(rate, &agg_rate_tmp, &agg_rate_units);
if (complete < ranks) {
MFU_LOG(MFU_LOG_INFO, "Wrote %.3lf %s (%.2f%%) in %.3lf secs (%.3lf %s) %d secs remaining ...",
agg_size_tmp, agg_size_units, percent, secs, agg_rate_tmp, agg_rate_units, (int)secs_remaining);
} else {
MFU_LOG(MFU_LOG_INFO, "Wrote %.3lf %s (%.2f%%) in %.3lf secs (%.3lf %s) done",
agg_size_tmp, agg_size_units, percent, secs, agg_rate_tmp, agg_rate_units);
}
}
/* print information about a file given the index and rank (used in print_files) */
static void print_file(mfu_flist flist, uint64_t idx)
{
/* store types as strings for print_file */
char type_str_unknown[] = "UNK";
char type_str_dir[] = "DIR";
char type_str_file[] = "REG";
char type_str_link[] = "LNK";
/* get filename */
const char* file = mfu_flist_file_get_name(flist, idx);
if (mfu_flist_have_detail(flist)) {
/* get mode */
mode_t mode = (mode_t) mfu_flist_file_get_mode(flist, idx);
//uint32_t uid = (uint32_t) mfu_flist_file_get_uid(flist, idx);
//uint32_t gid = (uint32_t) mfu_flist_file_get_gid(flist, idx);
uint64_t acc = mfu_flist_file_get_atime(flist, idx);
uint64_t mod = mfu_flist_file_get_mtime(flist, idx);
uint64_t cre = mfu_flist_file_get_ctime(flist, idx);
uint64_t size = mfu_flist_file_get_size(flist, idx);
const char* username = mfu_flist_file_get_username(flist, idx);
const char* groupname = mfu_flist_file_get_groupname(flist, idx);
char access_s[30];
char modify_s[30];
char create_s[30];
time_t access_t = (time_t) acc;
time_t modify_t = (time_t) mod;
time_t create_t = (time_t) cre;
size_t access_rc = strftime(access_s, sizeof(access_s) - 1, "%FT%T", localtime(&access_t));
//size_t modify_rc = strftime(modify_s, sizeof(modify_s) - 1, "%FT%T", localtime(&modify_t));
size_t modify_rc = strftime(modify_s, sizeof(modify_s) - 1, "%b %e %Y %H:%M", localtime(&modify_t));
size_t create_rc = strftime(create_s, sizeof(create_s) - 1, "%FT%T", localtime(&create_t));
if (access_rc == 0 || modify_rc == 0 || create_rc == 0) {
/* error */
access_s[0] = '\0';
modify_s[0] = '\0';
create_s[0] = '\0';
}
char mode_format[11];
mfu_format_mode(mode, mode_format);
double size_tmp;
const char* size_units;
mfu_format_bytes(size, &size_tmp, &size_units);
printf("%s %s %s %7.3f %2s %s %s\n",
mode_format, username, groupname,
size_tmp, size_units, modify_s, file
);
#if 0
printf("%s %s %s A%s M%s C%s %lu %s\n",
mode_format, username, groupname,
access_s, modify_s, create_s, (unsigned long)size, file
);
printf("Mode=%lx(%s) UID=%d(%s) GUI=%d(%s) Access=%s Modify=%s Create=%s Size=%lu File=%s\n",
(unsigned long)mode, mode_format, uid, username, gid, groupname,
access_s, modify_s, create_s, (unsigned long)size, file
);
#endif
}
else {
/* get type */
mfu_filetype type = mfu_flist_file_get_type(flist, idx);
char* type_str = type_str_unknown;
if (type == MFU_TYPE_DIR) {
type_str = type_str_dir;
}
else if (type == MFU_TYPE_FILE) {
type_str = type_str_file;
}
else if (type == MFU_TYPE_LINK) {
type_str = type_str_link;
}
printf("Type=%s File=%s\n",
type_str, file
);
}
}
static void mfu_flist_archive_create_libcircle(mfu_flist flist, const char* archivefile, mfu_archive_options_t* opts)
{
DTAR_flist = flist;
DTAR_user_opts = *opts;
MPI_Comm_rank(MPI_COMM_WORLD, &DTAR_rank);
/* TODO: stripe the archive file if on parallel file system */
/* init statistics */
DTAR_statistics.total_dirs = 0;
DTAR_statistics.total_files = 0;
DTAR_statistics.total_links = 0;
DTAR_statistics.total_size = 0;
DTAR_statistics.total_bytes_copied = 0;
time(&(DTAR_statistics.time_started));
DTAR_statistics.wtime_started = MPI_Wtime();
/* create the archive file */
DTAR_writer.name = archivefile;
DTAR_writer.flags = O_WRONLY | O_CREAT | O_CLOEXEC | O_LARGEFILE;
DTAR_writer.fd_tar = open(archivefile, DTAR_writer.flags, 0664);
/* get number of items in our portion of the list */
DTAR_count = mfu_flist_size(DTAR_flist);
/* allocate memory for file sizes and offsets */
uint64_t* fsizes = (uint64_t*) MFU_MALLOC(DTAR_count * sizeof(uint64_t));
DTAR_offsets = (uint64_t*) MFU_MALLOC(DTAR_count * sizeof(uint64_t));
/* compute local offsets for each item and total
* bytes we're contributing to the archive */
uint64_t idx;
uint64_t offset = 0;
for (idx = 0; idx < DTAR_count; idx++) {
/* assume the item takes no space */
fsizes[idx] = 0;
/* identify item type to compute its size in the archive */
mfu_filetype type = mfu_flist_file_get_type(DTAR_flist, idx);
if (type == MFU_TYPE_DIR || type == MFU_TYPE_LINK) {
/* directories and symlinks only need the header */
fsizes[idx] = DTAR_HDR_LENGTH;
} else if (type == MFU_TYPE_FILE) {
/* regular file requires a header, plus file content,
* and things are packed into blocks of 512 bytes */
uint64_t fsize = mfu_flist_file_get_size(DTAR_flist, idx);
/* determine whether file size is integer multiple of 512 bytes */
uint64_t rem = fsize % 512;
if (rem == 0) {
/* file content is multiple of 512 bytes, so perfect fit */
fsizes[idx] = fsize + DTAR_HDR_LENGTH;
} else {
/* TODO: check and explain this math */
fsizes[idx] = (fsize / 512 + 4) * 512;
}
}
/* increment our local offset for this item */
DTAR_offsets[idx] = offset;
offset += fsizes[idx];
}
/* execute scan to figure our global base offset in the archive file */
uint64_t global_offset = 0;
MPI_Scan(&offset, &global_offset, 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);
global_offset -= offset;
/* update offsets for each of our file to their global offset */
for (idx = 0; idx < DTAR_count; idx++) {
DTAR_offsets[idx] += global_offset;
}
/* create an archive */
struct archive* ar = archive_write_new();
archive_write_set_format_pax(ar);
int r = archive_write_open_fd(ar, DTAR_writer.fd_tar);
if (r != ARCHIVE_OK) {
MFU_LOG(MFU_LOG_ERR, "archive_write_open_fd(): %s", archive_error_string(ar));
DTAR_abort(EXIT_FAILURE);
}
/* write headers for our files */
for (idx = 0; idx < DTAR_count; idx++) {
mfu_filetype type = mfu_flist_file_get_type(DTAR_flist, idx);
if (type == MFU_TYPE_FILE || type == MFU_TYPE_DIR || type == MFU_TYPE_LINK) {
DTAR_write_header(ar, idx, DTAR_offsets[idx]);
}
}
/* prepare libcircle */
CIRCLE_init(0, NULL, CIRCLE_SPLIT_EQUAL | CIRCLE_CREATE_GLOBAL);
CIRCLE_loglevel loglevel = CIRCLE_LOG_WARN;
CIRCLE_enable_logging(loglevel);
/* register callbacks */
CIRCLE_cb_create(&DTAR_enqueue_copy);
CIRCLE_cb_process(&DTAR_perform_copy);
/* run the libcircle job to copy data into archive file */
CIRCLE_begin();
CIRCLE_finalize();
/* compute total bytes copied */
uint64_t archive_size = 0;
MPI_Allreduce(&offset, &archive_size, 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);
DTAR_statistics.total_size = archive_size;
DTAR_statistics.wtime_ended = MPI_Wtime();
time(&(DTAR_statistics.time_ended));
/* print stats */
double rel_time = DTAR_statistics.wtime_ended - \
DTAR_statistics.wtime_started;
if (DTAR_rank == 0) {
char starttime_str[256];
struct tm* localstart = localtime(&(DTAR_statistics.time_started));
strftime(starttime_str, 256, "%b-%d-%Y, %H:%M:%S", localstart);
char endtime_str[256];
struct tm* localend = localtime(&(DTAR_statistics.time_ended));
strftime(endtime_str, 256, "%b-%d-%Y, %H:%M:%S", localend);
/* add two 512 blocks at the end */
DTAR_statistics.total_size += 512 * 2;
/* convert bandwidth to unit */
double agg_rate_tmp;
double agg_rate = (double) DTAR_statistics.total_size / rel_time;
const char* agg_rate_units;
mfu_format_bytes(agg_rate, &agg_rate_tmp, &agg_rate_units);
MFU_LOG(MFU_LOG_INFO, "Started: %s", starttime_str);
MFU_LOG(MFU_LOG_INFO, "Completed: %s", endtime_str);
MFU_LOG(MFU_LOG_INFO, "Total archive size: %" PRIu64, DTAR_statistics.total_size);
MFU_LOG(MFU_LOG_INFO, "Rate: %.3lf %s " \
"(%.3" PRIu64 " bytes in %.3lf seconds)", \
agg_rate_tmp, agg_rate_units, DTAR_statistics.total_size, rel_time);
}
/* clean up */
mfu_free(&fsizes);
mfu_free(&DTAR_offsets);
/* close archive file */
archive_write_free(ar);
mfu_close(DTAR_writer.name, DTAR_writer.fd_tar);
}
static int print_flist_distribution(int file_histogram,
struct distribute_option *option,
mfu_flist* pflist, int rank)
{
/* file list to use */
mfu_flist flist = *pflist;
/* get local size for each rank, and max file sizes */
uint64_t size = mfu_flist_size(flist);
uint64_t global_max_file_size;
int separators = 0;
if (file_histogram) {
/* create default separators */
create_default_separators(option, &flist, &size, &separators,
&global_max_file_size);
} else {
separators = option->separator_number;
}
/* allocate a count for each bin, initialize the bin counts to 0
* it is separator + 1 because the last bin is the last separator
* to the DISTRIBUTE_MAX */
uint64_t* dist = (uint64_t*) MFU_MALLOC((separators + 1) * sizeof(uint64_t));
/* initialize the bin counts to 0 */
for (int i = 0; i <= separators; i++) {
dist[i] = 0;
}
/* for each file, identify appropriate bin and increment its count */
for (int i = 0; i < size; i++) {
/* get the size of the file */
uint64_t file_size = mfu_flist_file_get_size(flist, i);
/* loop through the bins and find the one the file belongs to,
* set last bin to -1, if a bin is not found while looping through the
* list of file size separators, then it belongs in the last bin
* so (last file size - MAX bin) */
int max_bin_flag = -1;
for (int j = 0; j < separators; j++) {
if (file_size <= option->separators[j]) {
/* found the bin set bin index & increment its count */
dist[j]++;
/* a file for this bin was found so can't belong to
* last bin (so set the flag) & exit the loop */
max_bin_flag = 1;
break;
}
}
/* if max_bin_flag is still -1 then the file belongs to the last bin */
if (max_bin_flag < 0) {
dist[separators]++;
}
}
/* get the total sum across all of the bins */
uint64_t* disttotal = (uint64_t*) MFU_MALLOC((separators + 1) * sizeof(uint64_t));
MPI_Allreduce(dist, disttotal, (uint64_t)separators + 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);
/* Print the file distribution */
if (rank == 0) {
/* number of files in a bin */
uint64_t number;
double size_tmp;
const char* size_units;
printf("%-27s %s\n", "Range", "Number");
for (int i = 0; i <= separators; i++) {
printf("%s", "[ ");
if (i == 0) {
printf("%7.3lf %2s", 0.000, "B");
} else {
mfu_format_bytes((uint64_t)option->separators[i - 1],
&size_tmp, &size_units);
printf("%7.3lf %2s", size_tmp, size_units);
}
printf("%s", " - ");
if (file_histogram) {
mfu_format_bytes((uint64_t)option->separators[i],
&size_tmp, &size_units);
number = disttotal[i];
mfu_format_bytes((uint64_t)option->separators[i],
&size_tmp, &size_units);
printf("%7.3lf %2s ) %"PRIu64"\n", size_tmp,
size_units, number);
} else {
if (i == separators) {
number = disttotal[i];
printf("%10s ) %"PRIu64"\n", "MAX", number);
} else {
number = disttotal[i];
mfu_format_bytes((uint64_t)option->separators[i],
&size_tmp, &size_units);
printf("%7.3lf %2s ) %"PRIu64"\n", size_tmp, size_units, number);
}
}
}
}
/* free the memory used to hold bin counts */
mfu_free(&disttotal);
mfu_free(&dist);
return 0;
}
