void DTAR_epilogue() { double rel_time = DTAR_statistics.wtime_ended - \ DTAR_statistics.wtime_started; if (DTAR_rank == 0) { char starttime_str[256]; struct tm* localstart = localtime(&(DTAR_statistics.time_started)); strftime(starttime_str, 256, "%b-%d-%Y, %H:%M:%S", localstart); char endtime_str[256]; struct tm* localend = localtime(&(DTAR_statistics.time_ended)); strftime(endtime_str, 256, "%b-%d-%Y, %H:%M:%S", localend); /* add two 512 blocks at the end */ DTAR_statistics.total_size += 512*2; /* convert bandwidth to unit */ double agg_rate_tmp; double agg_rate = (double) DTAR_statistics.total_size / rel_time; const char* agg_rate_units; mfu_format_bytes(agg_rate, &agg_rate_tmp, &agg_rate_units); MFU_LOG(MFU_LOG_INFO, "Started: %s", starttime_str); MFU_LOG(MFU_LOG_INFO, "Completed: %s", endtime_str); MFU_LOG(MFU_LOG_INFO, "Total archive size: %" PRIu64, DTAR_statistics.total_size); MFU_LOG(MFU_LOG_INFO, "Rate: %.3lf %s " \ "(%.3" PRIu64 " bytes in %.3lf seconds)", \ agg_rate_tmp, agg_rate_units, DTAR_statistics.total_size, rel_time); } }
static void generate_pretty_size(char *out, unsigned int length, uint64_t size) { double size_tmp; const char* size_units; char *unit; unsigned int unit_len; mfu_format_bytes(size, &size_tmp, &size_units); snprintf(out, length, "%.2f %s", size_tmp, size_units); }
static void create_default_separators(struct distribute_option *option, mfu_flist* flist, uint64_t* size, int* separators, uint64_t* global_max_file_size) { /* get local max file size for Allreduce */ uint64_t local_max_file_size = 0; for (int i = 0; i < *size; i++) { uint64_t file_size = mfu_flist_file_get_size(*flist, i); if (file_size > local_max_file_size) { local_max_file_size = file_size; } } /* get the max file size across all ranks */ MPI_Allreduce(&local_max_file_size, global_max_file_size, 1, MPI_UINT64_T, MPI_MAX, MPI_COMM_WORLD); /* print and convert max file size to appropriate units */ double max_size_tmp; const char* max_size_units; mfu_format_bytes(*global_max_file_size, &max_size_tmp, &max_size_units); printf("Max File Size: %.3lf %s\n", max_size_tmp, max_size_units); /* round next_pow_2 to next multiple of 10 */ uint64_t max_magnitude_bin = (ceil(log2(*global_max_file_size) / 10 )) * 10; /* get bin ranges based on max file size */ option->separators[0] = 1; /* plus one is for zero count bin */ *separators = max_magnitude_bin / 10; int power = 10; for (int i = 1; power <= max_magnitude_bin; i++) { option->separators[i] = pow(2, power); power+=10; } }
static void stripe_progress_fn(const uint64_t* vals, int count, int complete, int ranks, double secs) { /* compute percentage of items removed */ double percent = 0.0; if (stripe_prog_bytes_total > 0) { percent = 100.0 * (double)vals[0] / (double)stripe_prog_bytes_total; } /* compute average delete rate */ double rate = 0.0; if (secs > 0) { rate = (double)vals[0] / secs; } /* compute estimated time remaining */ double secs_remaining = -1.0; if (rate > 0.0) { secs_remaining = (double)(stripe_prog_bytes_total - vals[0]) / rate; } /* convert bytes to units */ double agg_size_tmp; const char* agg_size_units; mfu_format_bytes(vals[0], &agg_size_tmp, &agg_size_units); /* convert bandwidth to units */ double agg_rate_tmp; const char* agg_rate_units; mfu_format_bw(rate, &agg_rate_tmp, &agg_rate_units); if (complete < ranks) { MFU_LOG(MFU_LOG_INFO, "Wrote %.3lf %s (%.2f%%) in %.3lf secs (%.3lf %s) %d secs remaining ...", agg_size_tmp, agg_size_units, percent, secs, agg_rate_tmp, agg_rate_units, (int)secs_remaining); } else { MFU_LOG(MFU_LOG_INFO, "Wrote %.3lf %s (%.2f%%) in %.3lf secs (%.3lf %s) done", agg_size_tmp, agg_size_units, percent, secs, agg_rate_tmp, agg_rate_units); } }
/* print information about a file given the index and rank (used in print_files) */ static void print_file(mfu_flist flist, uint64_t idx) { /* store types as strings for print_file */ char type_str_unknown[] = "UNK"; char type_str_dir[] = "DIR"; char type_str_file[] = "REG"; char type_str_link[] = "LNK"; /* get filename */ const char* file = mfu_flist_file_get_name(flist, idx); if (mfu_flist_have_detail(flist)) { /* get mode */ mode_t mode = (mode_t) mfu_flist_file_get_mode(flist, idx); //uint32_t uid = (uint32_t) mfu_flist_file_get_uid(flist, idx); //uint32_t gid = (uint32_t) mfu_flist_file_get_gid(flist, idx); uint64_t acc = mfu_flist_file_get_atime(flist, idx); uint64_t mod = mfu_flist_file_get_mtime(flist, idx); uint64_t cre = mfu_flist_file_get_ctime(flist, idx); uint64_t size = mfu_flist_file_get_size(flist, idx); const char* username = mfu_flist_file_get_username(flist, idx); const char* groupname = mfu_flist_file_get_groupname(flist, idx); char access_s[30]; char modify_s[30]; char create_s[30]; time_t access_t = (time_t) acc; time_t modify_t = (time_t) mod; time_t create_t = (time_t) cre; size_t access_rc = strftime(access_s, sizeof(access_s) - 1, "%FT%T", localtime(&access_t)); //size_t modify_rc = strftime(modify_s, sizeof(modify_s) - 1, "%FT%T", localtime(&modify_t)); size_t modify_rc = strftime(modify_s, sizeof(modify_s) - 1, "%b %e %Y %H:%M", localtime(&modify_t)); size_t create_rc = strftime(create_s, sizeof(create_s) - 1, "%FT%T", localtime(&create_t)); if (access_rc == 0 || modify_rc == 0 || create_rc == 0) { /* error */ access_s[0] = '\0'; modify_s[0] = '\0'; create_s[0] = '\0'; } char mode_format[11]; mfu_format_mode(mode, mode_format); double size_tmp; const char* size_units; mfu_format_bytes(size, &size_tmp, &size_units); printf("%s %s %s %7.3f %2s %s %s\n", mode_format, username, groupname, size_tmp, size_units, modify_s, file ); #if 0 printf("%s %s %s A%s M%s C%s %lu %s\n", mode_format, username, groupname, access_s, modify_s, create_s, (unsigned long)size, file ); printf("Mode=%lx(%s) UID=%d(%s) GUI=%d(%s) Access=%s Modify=%s Create=%s Size=%lu File=%s\n", (unsigned long)mode, mode_format, uid, username, gid, groupname, access_s, modify_s, create_s, (unsigned long)size, file ); #endif } else { /* get type */ mfu_filetype type = mfu_flist_file_get_type(flist, idx); char* type_str = type_str_unknown; if (type == MFU_TYPE_DIR) { type_str = type_str_dir; } else if (type == MFU_TYPE_FILE) { type_str = type_str_file; } else if (type == MFU_TYPE_LINK) { type_str = type_str_link; } printf("Type=%s File=%s\n", type_str, file ); } }
static void mfu_flist_archive_create_libcircle(mfu_flist flist, const char* archivefile, mfu_archive_options_t* opts) { DTAR_flist = flist; DTAR_user_opts = *opts; MPI_Comm_rank(MPI_COMM_WORLD, &DTAR_rank); /* TODO: stripe the archive file if on parallel file system */ /* init statistics */ DTAR_statistics.total_dirs = 0; DTAR_statistics.total_files = 0; DTAR_statistics.total_links = 0; DTAR_statistics.total_size = 0; DTAR_statistics.total_bytes_copied = 0; time(&(DTAR_statistics.time_started)); DTAR_statistics.wtime_started = MPI_Wtime(); /* create the archive file */ DTAR_writer.name = archivefile; DTAR_writer.flags = O_WRONLY | O_CREAT | O_CLOEXEC | O_LARGEFILE; DTAR_writer.fd_tar = open(archivefile, DTAR_writer.flags, 0664); /* get number of items in our portion of the list */ DTAR_count = mfu_flist_size(DTAR_flist); /* allocate memory for file sizes and offsets */ uint64_t* fsizes = (uint64_t*) MFU_MALLOC(DTAR_count * sizeof(uint64_t)); DTAR_offsets = (uint64_t*) MFU_MALLOC(DTAR_count * sizeof(uint64_t)); /* compute local offsets for each item and total * bytes we're contributing to the archive */ uint64_t idx; uint64_t offset = 0; for (idx = 0; idx < DTAR_count; idx++) { /* assume the item takes no space */ fsizes[idx] = 0; /* identify item type to compute its size in the archive */ mfu_filetype type = mfu_flist_file_get_type(DTAR_flist, idx); if (type == MFU_TYPE_DIR || type == MFU_TYPE_LINK) { /* directories and symlinks only need the header */ fsizes[idx] = DTAR_HDR_LENGTH; } else if (type == MFU_TYPE_FILE) { /* regular file requires a header, plus file content, * and things are packed into blocks of 512 bytes */ uint64_t fsize = mfu_flist_file_get_size(DTAR_flist, idx); /* determine whether file size is integer multiple of 512 bytes */ uint64_t rem = fsize % 512; if (rem == 0) { /* file content is multiple of 512 bytes, so perfect fit */ fsizes[idx] = fsize + DTAR_HDR_LENGTH; } else { /* TODO: check and explain this math */ fsizes[idx] = (fsize / 512 + 4) * 512; } } /* increment our local offset for this item */ DTAR_offsets[idx] = offset; offset += fsizes[idx]; } /* execute scan to figure our global base offset in the archive file */ uint64_t global_offset = 0; MPI_Scan(&offset, &global_offset, 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD); global_offset -= offset; /* update offsets for each of our file to their global offset */ for (idx = 0; idx < DTAR_count; idx++) { DTAR_offsets[idx] += global_offset; } /* create an archive */ struct archive* ar = archive_write_new(); archive_write_set_format_pax(ar); int r = archive_write_open_fd(ar, DTAR_writer.fd_tar); if (r != ARCHIVE_OK) { MFU_LOG(MFU_LOG_ERR, "archive_write_open_fd(): %s", archive_error_string(ar)); DTAR_abort(EXIT_FAILURE); } /* write headers for our files */ for (idx = 0; idx < DTAR_count; idx++) { mfu_filetype type = mfu_flist_file_get_type(DTAR_flist, idx); if (type == MFU_TYPE_FILE || type == MFU_TYPE_DIR || type == MFU_TYPE_LINK) { DTAR_write_header(ar, idx, DTAR_offsets[idx]); } } /* prepare libcircle */ CIRCLE_init(0, NULL, CIRCLE_SPLIT_EQUAL | CIRCLE_CREATE_GLOBAL); CIRCLE_loglevel loglevel = CIRCLE_LOG_WARN; CIRCLE_enable_logging(loglevel); /* register callbacks */ CIRCLE_cb_create(&DTAR_enqueue_copy); CIRCLE_cb_process(&DTAR_perform_copy); /* run the libcircle job to copy data into archive file */ CIRCLE_begin(); CIRCLE_finalize(); /* compute total bytes copied */ uint64_t archive_size = 0; MPI_Allreduce(&offset, &archive_size, 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD); DTAR_statistics.total_size = archive_size; DTAR_statistics.wtime_ended = MPI_Wtime(); time(&(DTAR_statistics.time_ended)); /* print stats */ double rel_time = DTAR_statistics.wtime_ended - \ DTAR_statistics.wtime_started; if (DTAR_rank == 0) { char starttime_str[256]; struct tm* localstart = localtime(&(DTAR_statistics.time_started)); strftime(starttime_str, 256, "%b-%d-%Y, %H:%M:%S", localstart); char endtime_str[256]; struct tm* localend = localtime(&(DTAR_statistics.time_ended)); strftime(endtime_str, 256, "%b-%d-%Y, %H:%M:%S", localend); /* add two 512 blocks at the end */ DTAR_statistics.total_size += 512 * 2; /* convert bandwidth to unit */ double agg_rate_tmp; double agg_rate = (double) DTAR_statistics.total_size / rel_time; const char* agg_rate_units; mfu_format_bytes(agg_rate, &agg_rate_tmp, &agg_rate_units); MFU_LOG(MFU_LOG_INFO, "Started: %s", starttime_str); MFU_LOG(MFU_LOG_INFO, "Completed: %s", endtime_str); MFU_LOG(MFU_LOG_INFO, "Total archive size: %" PRIu64, DTAR_statistics.total_size); MFU_LOG(MFU_LOG_INFO, "Rate: %.3lf %s " \ "(%.3" PRIu64 " bytes in %.3lf seconds)", \ agg_rate_tmp, agg_rate_units, DTAR_statistics.total_size, rel_time); } /* clean up */ mfu_free(&fsizes); mfu_free(&DTAR_offsets); /* close archive file */ archive_write_free(ar); mfu_close(DTAR_writer.name, DTAR_writer.fd_tar); }
static int print_flist_distribution(int file_histogram, struct distribute_option *option, mfu_flist* pflist, int rank) { /* file list to use */ mfu_flist flist = *pflist; /* get local size for each rank, and max file sizes */ uint64_t size = mfu_flist_size(flist); uint64_t global_max_file_size; int separators = 0; if (file_histogram) { /* create default separators */ create_default_separators(option, &flist, &size, &separators, &global_max_file_size); } else { separators = option->separator_number; } /* allocate a count for each bin, initialize the bin counts to 0 * it is separator + 1 because the last bin is the last separator * to the DISTRIBUTE_MAX */ uint64_t* dist = (uint64_t*) MFU_MALLOC((separators + 1) * sizeof(uint64_t)); /* initialize the bin counts to 0 */ for (int i = 0; i <= separators; i++) { dist[i] = 0; } /* for each file, identify appropriate bin and increment its count */ for (int i = 0; i < size; i++) { /* get the size of the file */ uint64_t file_size = mfu_flist_file_get_size(flist, i); /* loop through the bins and find the one the file belongs to, * set last bin to -1, if a bin is not found while looping through the * list of file size separators, then it belongs in the last bin * so (last file size - MAX bin) */ int max_bin_flag = -1; for (int j = 0; j < separators; j++) { if (file_size <= option->separators[j]) { /* found the bin set bin index & increment its count */ dist[j]++; /* a file for this bin was found so can't belong to * last bin (so set the flag) & exit the loop */ max_bin_flag = 1; break; } } /* if max_bin_flag is still -1 then the file belongs to the last bin */ if (max_bin_flag < 0) { dist[separators]++; } } /* get the total sum across all of the bins */ uint64_t* disttotal = (uint64_t*) MFU_MALLOC((separators + 1) * sizeof(uint64_t)); MPI_Allreduce(dist, disttotal, (uint64_t)separators + 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD); /* Print the file distribution */ if (rank == 0) { /* number of files in a bin */ uint64_t number; double size_tmp; const char* size_units; printf("%-27s %s\n", "Range", "Number"); for (int i = 0; i <= separators; i++) { printf("%s", "[ "); if (i == 0) { printf("%7.3lf %2s", 0.000, "B"); } else { mfu_format_bytes((uint64_t)option->separators[i - 1], &size_tmp, &size_units); printf("%7.3lf %2s", size_tmp, size_units); } printf("%s", " - "); if (file_histogram) { mfu_format_bytes((uint64_t)option->separators[i], &size_tmp, &size_units); number = disttotal[i]; mfu_format_bytes((uint64_t)option->separators[i], &size_tmp, &size_units); printf("%7.3lf %2s ) %"PRIu64"\n", size_tmp, size_units, number); } else { if (i == separators) { number = disttotal[i]; printf("%10s ) %"PRIu64"\n", "MAX", number); } else { number = disttotal[i]; mfu_format_bytes((uint64_t)option->separators[i], &size_tmp, &size_units); printf("%7.3lf %2s ) %"PRIu64"\n", size_tmp, size_units, number); } } } } /* free the memory used to hold bin counts */ mfu_free(&disttotal); mfu_free(&dist); return 0; }