void DTAR_enqueue_copy(CIRCLE_handle *handle) {
for (uint64_t idx = 0; idx < DTAR_count; idx++) {
mfu_filetype type = mfu_flist_file_get_type(DTAR_flist, idx);
/* add copy work only for files */
if (type == MFU_TYPE_FILE) {
uint64_t dataoffset = DTAR_offsets[idx] + DTAR_HDR_LENGTH;
const char * name = mfu_flist_file_get_name(DTAR_flist, idx);
uint64_t size = mfu_flist_file_get_size(DTAR_flist, idx);
/* compute number of chunks */
uint64_t num_chunks = size / DTAR_user_opts.chunk_size;
for (uint64_t chunk_idx = 0; chunk_idx < num_chunks; chunk_idx++) {
char* newop = DTAR_encode_operation(
COPY_DATA, name, size, chunk_idx, dataoffset);
handle->enqueue(newop);
mfu_free(&newop);
}
/* create copy work for possibly last item */
if (num_chunks * DTAR_user_opts.chunk_size < size || num_chunks == 0) {
char* newop = DTAR_encode_operation(
COPY_DATA, name, size, num_chunks, dataoffset);
handle->enqueue(newop);
mfu_free(&newop);
}
}
}
}
/* filter the list of files down based on the current stripe size and stripe count */
static mfu_flist filter_list(mfu_flist list, int stripe_count, uint64_t stripe_size, uint64_t min_size, uint64_t* total_count, uint64_t* total_size)
{
/* initialize counters for file and byte count */
uint64_t my_count = 0;
uint64_t my_size = 0;
/* this is going to be a subset of the full file list */
mfu_flist filtered = mfu_flist_subset(list);
uint64_t idx;
uint64_t size = mfu_flist_size(list);
for (idx = 0; idx < size; idx++) {
/* we only care about regular files */
mfu_filetype type = mfu_flist_file_get_type(list, idx);
if (type == MFU_TYPE_FILE) {
/* if our file is below the minimum file size, skip it */
uint64_t filesize = mfu_flist_file_get_size(list, idx);
if (filesize < min_size) {
continue;
}
const char* in_path = mfu_flist_file_get_name(list, idx);
uint64_t curr_stripe_size = 0;
uint64_t curr_stripe_count = 0;
/*
* attempt to get striping info,
* skip the file if we can't get the striping info we seek
*/
if (mfu_stripe_get(in_path, &curr_stripe_size, &curr_stripe_count) != 0) {
continue;
}
/* TODO: this should probably be better */
/* if the current stripe size or stripe count doesn't match, then a restripe the file */
if (curr_stripe_count != stripe_count || curr_stripe_size != stripe_size) {
mfu_flist_file_copy(list, idx, filtered);
/* increment file count and add file size to our running total */
my_count += 1;
my_size += filesize;
}
}
}
/* summarize and return the new list */
mfu_flist_summarize(filtered);
/* get sum of count and size */
MPI_Allreduce(&my_count, total_count, 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);
MPI_Allreduce(&my_size, total_size, 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);
return filtered;
}
/* print to stdout the stripe size and count of each file in the mfu_flist */
static void stripe_info_report(mfu_flist list)
{
uint64_t idx;
uint64_t size = mfu_flist_size(list);
int rank;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
/* print header */
if (rank == 0) {
printf("%10s %3.3s %8.8s %s\n", "Size", "Cnt", "Str Size", "File Path");
printf("%10s %3.3s %8.8s %s\n", "----", "---", "--------", "---------");
fflush(stdout);
}
MPI_Barrier(MPI_COMM_WORLD);
/* print out file info */
for (idx = 0; idx < size; idx++) {
mfu_filetype type = mfu_flist_file_get_type(list, idx);
/* report striping information for regular files only */
if (type == MFU_TYPE_FILE) {
const char* in_path = mfu_flist_file_get_name(list, idx);
uint64_t stripe_size = 0;
uint64_t stripe_count = 0;
char filesize[11];
char stripesize[9];
/*
* attempt to get striping info and print it out,
* skip the file if we can't get the striping info we seek
*/
if (mfu_stripe_get(in_path, &stripe_size, &stripe_count) != 0) {
continue;
}
/* format it nicely */
generate_pretty_size(filesize, sizeof(filesize), mfu_flist_file_get_size(list, idx));
generate_pretty_size(stripesize, sizeof(stripesize), stripe_size);
/* print the row */
printf("%10.10s %3" PRId64 " %8.8s %s\n", filesize, stripe_count, stripesize, in_path);
fflush(stdout);
}
}
}
static void create_default_separators(struct distribute_option *option,
mfu_flist* flist,
uint64_t* size,
int* separators,
uint64_t* global_max_file_size)
{
/* get local max file size for Allreduce */
uint64_t local_max_file_size = 0;
for (int i = 0; i < *size; i++) {
uint64_t file_size = mfu_flist_file_get_size(*flist, i);
if (file_size > local_max_file_size) {
local_max_file_size = file_size;
}
}
/* get the max file size across all ranks */
MPI_Allreduce(&local_max_file_size, global_max_file_size, 1,
MPI_UINT64_T, MPI_MAX, MPI_COMM_WORLD);
/* print and convert max file size to appropriate units */
double max_size_tmp;
const char* max_size_units;
mfu_format_bytes(*global_max_file_size, &max_size_tmp, &max_size_units);
printf("Max File Size: %.3lf %s\n", max_size_tmp, max_size_units);
/* round next_pow_2 to next multiple of 10 */
uint64_t max_magnitude_bin = (ceil(log2(*global_max_file_size) / 10 )) * 10;
/* get bin ranges based on max file size */
option->separators[0] = 1;
/* plus one is for zero count bin */
*separators = max_magnitude_bin / 10;
int power = 10;
for (int i = 1; power <= max_magnitude_bin; i++) {
option->separators[i] = pow(2, power);
power+=10;
}
}
static int sort_files_stat(const char* sortfields, mfu_flist* pflist)
{
/* get list from caller */
mfu_flist flist = *pflist;
/* create a new list as subset of original list */
mfu_flist flist2 = mfu_flist_subset(flist);
uint64_t incount = mfu_flist_size(flist);
uint64_t chars = mfu_flist_file_max_name(flist);
uint64_t chars_user = mfu_flist_user_max_name(flist);
uint64_t chars_group = mfu_flist_group_max_name(flist);
/* create datatype for packed file list element */
MPI_Datatype dt_sat;
size_t bytes = mfu_flist_file_pack_size(flist);
MPI_Type_contiguous((int)bytes, MPI_BYTE, &dt_sat);
/* get our rank and the size of comm_world */
int rank, ranks;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &ranks);
/* build type for file path */
MPI_Datatype dt_filepath, dt_user, dt_group;
MPI_Type_contiguous((int)chars, MPI_CHAR, &dt_filepath);
MPI_Type_contiguous((int)chars_user, MPI_CHAR, &dt_user);
MPI_Type_contiguous((int)chars_group, MPI_CHAR, &dt_group);
MPI_Type_commit(&dt_filepath);
MPI_Type_commit(&dt_user);
MPI_Type_commit(&dt_group);
/* build comparison op for filenames */
DTCMP_Op op_filepath, op_user, op_group;
if (DTCMP_Op_create(dt_filepath, my_strcmp, &op_filepath) != DTCMP_SUCCESS) {
MFU_ABORT(1, "Failed to create sorting operation for filepath");
}
if (DTCMP_Op_create(dt_user, my_strcmp, &op_user) != DTCMP_SUCCESS) {
MFU_ABORT(1, "Failed to create sorting operation for username");
}
if (DTCMP_Op_create(dt_group, my_strcmp, &op_group) != DTCMP_SUCCESS) {
MFU_ABORT(1, "Failed to create sorting operation for groupname");
}
/* build comparison op for filenames */
DTCMP_Op op_filepath_rev, op_user_rev, op_group_rev;
if (DTCMP_Op_create(dt_filepath, my_strcmp_rev, &op_filepath_rev) != DTCMP_SUCCESS) {
MFU_ABORT(1, "Failed to create reverse sorting operation for groupname");
}
if (DTCMP_Op_create(dt_user, my_strcmp_rev, &op_user_rev) != DTCMP_SUCCESS) {
MFU_ABORT(1, "Failed to create reverse sorting operation for groupname");
}
if (DTCMP_Op_create(dt_group, my_strcmp_rev, &op_group_rev) != DTCMP_SUCCESS) {
MFU_ABORT(1, "Failed to create reverse sorting operation for groupname");
}
/* TODO: process sort fields */
const int MAXFIELDS = 7;
MPI_Datatype types[MAXFIELDS];
DTCMP_Op ops[MAXFIELDS];
sort_field fields[MAXFIELDS];
size_t lengths[MAXFIELDS];
int nfields = 0;
for (nfields = 0; nfields < MAXFIELDS; nfields++) {
types[nfields] = MPI_DATATYPE_NULL;
ops[nfields] = DTCMP_OP_NULL;
}
nfields = 0;
char* sortfields_copy = MFU_STRDUP(sortfields);
char* token = strtok(sortfields_copy, ",");
while (token != NULL) {
int valid = 1;
if (strcmp(token, "name") == 0) {
types[nfields] = dt_filepath;
ops[nfields] = op_filepath;
fields[nfields] = FILENAME;
lengths[nfields] = chars;
}
else if (strcmp(token, "-name") == 0) {
types[nfields] = dt_filepath;
ops[nfields] = op_filepath_rev;
fields[nfields] = FILENAME;
lengths[nfields] = chars;
}
else if (strcmp(token, "user") == 0) {
types[nfields] = dt_user;
ops[nfields] = op_user;
fields[nfields] = USERNAME;
lengths[nfields] = chars_user;
}
else if (strcmp(token, "-user") == 0) {
types[nfields] = dt_user;
ops[nfields] = op_user_rev;
fields[nfields] = USERNAME;
lengths[nfields] = chars_user;
}
else if (strcmp(token, "group") == 0) {
types[nfields] = dt_group;
ops[nfields] = op_group;
fields[nfields] = GROUPNAME;
lengths[nfields] = chars_group;
}
else if (strcmp(token, "-group") == 0) {
types[nfields] = dt_group;
ops[nfields] = op_group_rev;
fields[nfields] = GROUPNAME;
lengths[nfields] = chars_group;
}
else if (strcmp(token, "uid") == 0) {
types[nfields] = MPI_UINT32_T;
ops[nfields] = DTCMP_OP_UINT32T_ASCEND;
fields[nfields] = USERID;
lengths[nfields] = 4;
}
else if (strcmp(token, "-uid") == 0) {
types[nfields] = MPI_UINT32_T;
ops[nfields] = DTCMP_OP_UINT32T_DESCEND;
fields[nfields] = USERID;
lengths[nfields] = 4;
}
else if (strcmp(token, "gid") == 0) {
types[nfields] = MPI_UINT32_T;
ops[nfields] = DTCMP_OP_UINT32T_ASCEND;
fields[nfields] = GROUPID;
lengths[nfields] = 4;
}
else if (strcmp(token, "-gid") == 0) {
types[nfields] = MPI_UINT32_T;
ops[nfields] = DTCMP_OP_UINT32T_DESCEND;
fields[nfields] = GROUPID;
lengths[nfields] = 4;
}
else if (strcmp(token, "atime") == 0) {
types[nfields] = MPI_UINT32_T;
ops[nfields] = DTCMP_OP_UINT32T_ASCEND;
fields[nfields] = ATIME;
lengths[nfields] = 4;
}
else if (strcmp(token, "-atime") == 0) {
types[nfields] = MPI_UINT32_T;
ops[nfields] = DTCMP_OP_UINT32T_DESCEND;
fields[nfields] = ATIME;
lengths[nfields] = 4;
}
else if (strcmp(token, "mtime") == 0) {
types[nfields] = MPI_UINT32_T;
ops[nfields] = DTCMP_OP_UINT32T_ASCEND;
fields[nfields] = MTIME;
lengths[nfields] = 4;
}
else if (strcmp(token, "-mtime") == 0) {
types[nfields] = MPI_UINT32_T;
ops[nfields] = DTCMP_OP_UINT32T_DESCEND;
fields[nfields] = MTIME;
lengths[nfields] = 4;
}
else if (strcmp(token, "ctime") == 0) {
types[nfields] = MPI_UINT32_T;
ops[nfields] = DTCMP_OP_UINT32T_ASCEND;
fields[nfields] = CTIME;
lengths[nfields] = 4;
}
else if (strcmp(token, "-ctime") == 0) {
types[nfields] = MPI_UINT32_T;
ops[nfields] = DTCMP_OP_UINT32T_DESCEND;
fields[nfields] = CTIME;
lengths[nfields] = 4;
}
else if (strcmp(token, "size") == 0) {
types[nfields] = MPI_UINT64_T;
ops[nfields] = DTCMP_OP_UINT64T_ASCEND;
fields[nfields] = FILESIZE;
lengths[nfields] = 8;
}
else if (strcmp(token, "-size") == 0) {
types[nfields] = MPI_UINT64_T;
ops[nfields] = DTCMP_OP_UINT64T_DESCEND;
fields[nfields] = FILESIZE;
lengths[nfields] = 8;
}
else {
/* invalid token */
valid = 0;
if (rank == 0) {
MFU_LOG(MFU_LOG_ERR, "Invalid sort field: %s\n", token);
}
}
if (valid) {
nfields++;
}
if (nfields > MAXFIELDS) {
/* TODO: print warning if we have too many fields */
break;
}
token = strtok(NULL, ",");
}
mfu_free(&sortfields_copy);
/* build key type */
MPI_Datatype dt_key;
if (DTCMP_Type_create_series(nfields, types, &dt_key) != DTCMP_SUCCESS) {
MFU_ABORT(1, "Failed to create key type");
}
/* create op to sort by access time, then filename */
DTCMP_Op op_key;
if (DTCMP_Op_create_series(nfields, ops, &op_key) != DTCMP_SUCCESS) {
MFU_ABORT(1, "Failed to create sorting operation for key");
}
/* build keysat type */
MPI_Datatype dt_keysat, keysat_types[2];
keysat_types[0] = dt_key;
keysat_types[1] = dt_sat;
if (DTCMP_Type_create_series(2, keysat_types, &dt_keysat) != DTCMP_SUCCESS) {
MFU_ABORT(1, "Failed to create keysat type");
}
/* get extent of key type */
MPI_Aint key_lb, key_extent;
MPI_Type_get_extent(dt_key, &key_lb, &key_extent);
/* get extent of keysat type */
MPI_Aint keysat_lb, keysat_extent;
MPI_Type_get_extent(dt_keysat, &keysat_lb, &keysat_extent);
/* get extent of sat type */
MPI_Aint stat_lb, stat_extent;
MPI_Type_get_extent(dt_sat, &stat_lb, &stat_extent);
/* compute size of sort element and allocate buffer */
size_t sortbufsize = (size_t)keysat_extent * incount;
void* sortbuf = MFU_MALLOC(sortbufsize);
/* copy data into sort elements */
uint64_t idx = 0;
char* sortptr = (char*) sortbuf;
while (idx < incount) {
/* copy in access time */
int i;
for (i = 0; i < nfields; i++) {
if (fields[i] == FILENAME) {
const char* name = mfu_flist_file_get_name(flist, idx);
strcpy(sortptr, name);
}
else if (fields[i] == USERNAME) {
const char* name = mfu_flist_file_get_username(flist, idx);
strcpy(sortptr, name);
}
else if (fields[i] == GROUPNAME) {
const char* name = mfu_flist_file_get_groupname(flist, idx);
strcpy(sortptr, name);
}
else if (fields[i] == USERID) {
uint32_t val32 = (uint32_t) mfu_flist_file_get_uid(flist, idx);
memcpy(sortptr, &val32, 4);
}
else if (fields[i] == GROUPID) {
uint32_t val32 = (uint32_t) mfu_flist_file_get_gid(flist, idx);
memcpy(sortptr, &val32, 4);
}
else if (fields[i] == ATIME) {
uint32_t val32 = (uint32_t) mfu_flist_file_get_atime(flist, idx);
memcpy(sortptr, &val32, 4);
}
else if (fields[i] == MTIME) {
uint32_t val32 = (uint32_t) mfu_flist_file_get_mtime(flist, idx);
memcpy(sortptr, &val32, 4);
}
else if (fields[i] == CTIME) {
uint32_t val32 = (uint32_t) mfu_flist_file_get_ctime(flist, idx);
memcpy(sortptr, &val32, 4);
}
else if (fields[i] == FILESIZE) {
uint64_t val64 = mfu_flist_file_get_size(flist, idx);
memcpy(sortptr, &val64, 8);
}
sortptr += lengths[i];
}
/* pack file element */
sortptr += mfu_flist_file_pack(sortptr, flist, idx);
idx++;
}
/* sort data */
void* outsortbuf;
int outsortcount;
DTCMP_Handle handle;
int sort_rc = DTCMP_Sortz(
sortbuf, (int)incount, &outsortbuf, &outsortcount,
dt_key, dt_keysat, op_key, DTCMP_FLAG_NONE,
MPI_COMM_WORLD, &handle
);
if (sort_rc != DTCMP_SUCCESS) {
MFU_ABORT(1, "Failed to sort data");
}
/* step through sorted data filenames */
idx = 0;
sortptr = (char*) outsortbuf;
while (idx < (uint64_t)outsortcount) {
sortptr += key_extent;
sortptr += mfu_flist_file_unpack(sortptr, flist2);
idx++;
}
/* build summary of new list */
mfu_flist_summarize(flist2);
/* free memory */
DTCMP_Free(&handle);
/* free ops */
DTCMP_Op_free(&op_key);
DTCMP_Op_free(&op_group_rev);
DTCMP_Op_free(&op_user_rev);
DTCMP_Op_free(&op_filepath_rev);
DTCMP_Op_free(&op_group);
DTCMP_Op_free(&op_user);
DTCMP_Op_free(&op_filepath);
/* free types */
MPI_Type_free(&dt_keysat);
MPI_Type_free(&dt_key);
MPI_Type_free(&dt_group);
MPI_Type_free(&dt_user);
MPI_Type_free(&dt_filepath);
/* free input buffer holding sort elements */
mfu_free(&sortbuf);
/* free the satellite type */
MPI_Type_free(&dt_sat);
/* return new list and free old one */
*pflist = flist2;
mfu_flist_free(&flist);
return MFU_SUCCESS;
}
/* print information about a file given the index and rank (used in print_files) */
static void print_file(mfu_flist flist, uint64_t idx)
{
/* store types as strings for print_file */
char type_str_unknown[] = "UNK";
char type_str_dir[] = "DIR";
char type_str_file[] = "REG";
char type_str_link[] = "LNK";
/* get filename */
const char* file = mfu_flist_file_get_name(flist, idx);
if (mfu_flist_have_detail(flist)) {
/* get mode */
mode_t mode = (mode_t) mfu_flist_file_get_mode(flist, idx);
//uint32_t uid = (uint32_t) mfu_flist_file_get_uid(flist, idx);
//uint32_t gid = (uint32_t) mfu_flist_file_get_gid(flist, idx);
uint64_t acc = mfu_flist_file_get_atime(flist, idx);
uint64_t mod = mfu_flist_file_get_mtime(flist, idx);
uint64_t cre = mfu_flist_file_get_ctime(flist, idx);
uint64_t size = mfu_flist_file_get_size(flist, idx);
const char* username = mfu_flist_file_get_username(flist, idx);
const char* groupname = mfu_flist_file_get_groupname(flist, idx);
char access_s[30];
char modify_s[30];
char create_s[30];
time_t access_t = (time_t) acc;
time_t modify_t = (time_t) mod;
time_t create_t = (time_t) cre;
size_t access_rc = strftime(access_s, sizeof(access_s) - 1, "%FT%T", localtime(&access_t));
//size_t modify_rc = strftime(modify_s, sizeof(modify_s) - 1, "%FT%T", localtime(&modify_t));
size_t modify_rc = strftime(modify_s, sizeof(modify_s) - 1, "%b %e %Y %H:%M", localtime(&modify_t));
size_t create_rc = strftime(create_s, sizeof(create_s) - 1, "%FT%T", localtime(&create_t));
if (access_rc == 0 || modify_rc == 0 || create_rc == 0) {
/* error */
access_s[0] = '\0';
modify_s[0] = '\0';
create_s[0] = '\0';
}
char mode_format[11];
mfu_format_mode(mode, mode_format);
double size_tmp;
const char* size_units;
mfu_format_bytes(size, &size_tmp, &size_units);
printf("%s %s %s %7.3f %2s %s %s\n",
mode_format, username, groupname,
size_tmp, size_units, modify_s, file
);
#if 0
printf("%s %s %s A%s M%s C%s %lu %s\n",
mode_format, username, groupname,
access_s, modify_s, create_s, (unsigned long)size, file
);
printf("Mode=%lx(%s) UID=%d(%s) GUI=%d(%s) Access=%s Modify=%s Create=%s Size=%lu File=%s\n",
(unsigned long)mode, mode_format, uid, username, gid, groupname,
access_s, modify_s, create_s, (unsigned long)size, file
);
#endif
}
else {
/* get type */
mfu_filetype type = mfu_flist_file_get_type(flist, idx);
char* type_str = type_str_unknown;
if (type == MFU_TYPE_DIR) {
type_str = type_str_dir;
}
else if (type == MFU_TYPE_FILE) {
type_str = type_str_file;
}
else if (type == MFU_TYPE_LINK) {
type_str = type_str_link;
}
printf("Type=%s File=%s\n",
type_str, file
);
}
}
static void mfu_flist_archive_create_libcircle(mfu_flist flist, const char* archivefile, mfu_archive_options_t* opts)
{
DTAR_flist = flist;
DTAR_user_opts = *opts;
MPI_Comm_rank(MPI_COMM_WORLD, &DTAR_rank);
/* TODO: stripe the archive file if on parallel file system */
/* init statistics */
DTAR_statistics.total_dirs = 0;
DTAR_statistics.total_files = 0;
DTAR_statistics.total_links = 0;
DTAR_statistics.total_size = 0;
DTAR_statistics.total_bytes_copied = 0;
time(&(DTAR_statistics.time_started));
DTAR_statistics.wtime_started = MPI_Wtime();
/* create the archive file */
DTAR_writer.name = archivefile;
DTAR_writer.flags = O_WRONLY | O_CREAT | O_CLOEXEC | O_LARGEFILE;
DTAR_writer.fd_tar = open(archivefile, DTAR_writer.flags, 0664);
/* get number of items in our portion of the list */
DTAR_count = mfu_flist_size(DTAR_flist);
/* allocate memory for file sizes and offsets */
uint64_t* fsizes = (uint64_t*) MFU_MALLOC(DTAR_count * sizeof(uint64_t));
DTAR_offsets = (uint64_t*) MFU_MALLOC(DTAR_count * sizeof(uint64_t));
/* compute local offsets for each item and total
* bytes we're contributing to the archive */
uint64_t idx;
uint64_t offset = 0;
for (idx = 0; idx < DTAR_count; idx++) {
/* assume the item takes no space */
fsizes[idx] = 0;
/* identify item type to compute its size in the archive */
mfu_filetype type = mfu_flist_file_get_type(DTAR_flist, idx);
if (type == MFU_TYPE_DIR || type == MFU_TYPE_LINK) {
/* directories and symlinks only need the header */
fsizes[idx] = DTAR_HDR_LENGTH;
} else if (type == MFU_TYPE_FILE) {
/* regular file requires a header, plus file content,
* and things are packed into blocks of 512 bytes */
uint64_t fsize = mfu_flist_file_get_size(DTAR_flist, idx);
/* determine whether file size is integer multiple of 512 bytes */
uint64_t rem = fsize % 512;
if (rem == 0) {
/* file content is multiple of 512 bytes, so perfect fit */
fsizes[idx] = fsize + DTAR_HDR_LENGTH;
} else {
/* TODO: check and explain this math */
fsizes[idx] = (fsize / 512 + 4) * 512;
}
}
/* increment our local offset for this item */
DTAR_offsets[idx] = offset;
offset += fsizes[idx];
}
/* execute scan to figure our global base offset in the archive file */
uint64_t global_offset = 0;
MPI_Scan(&offset, &global_offset, 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);
global_offset -= offset;
/* update offsets for each of our file to their global offset */
for (idx = 0; idx < DTAR_count; idx++) {
DTAR_offsets[idx] += global_offset;
}
/* create an archive */
struct archive* ar = archive_write_new();
archive_write_set_format_pax(ar);
int r = archive_write_open_fd(ar, DTAR_writer.fd_tar);
if (r != ARCHIVE_OK) {
MFU_LOG(MFU_LOG_ERR, "archive_write_open_fd(): %s", archive_error_string(ar));
DTAR_abort(EXIT_FAILURE);
}
/* write headers for our files */
for (idx = 0; idx < DTAR_count; idx++) {
mfu_filetype type = mfu_flist_file_get_type(DTAR_flist, idx);
if (type == MFU_TYPE_FILE || type == MFU_TYPE_DIR || type == MFU_TYPE_LINK) {
DTAR_write_header(ar, idx, DTAR_offsets[idx]);
}
}
/* prepare libcircle */
CIRCLE_init(0, NULL, CIRCLE_SPLIT_EQUAL | CIRCLE_CREATE_GLOBAL);
CIRCLE_loglevel loglevel = CIRCLE_LOG_WARN;
CIRCLE_enable_logging(loglevel);
/* register callbacks */
CIRCLE_cb_create(&DTAR_enqueue_copy);
CIRCLE_cb_process(&DTAR_perform_copy);
/* run the libcircle job to copy data into archive file */
CIRCLE_begin();
CIRCLE_finalize();
/* compute total bytes copied */
uint64_t archive_size = 0;
MPI_Allreduce(&offset, &archive_size, 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);
DTAR_statistics.total_size = archive_size;
DTAR_statistics.wtime_ended = MPI_Wtime();
time(&(DTAR_statistics.time_ended));
/* print stats */
double rel_time = DTAR_statistics.wtime_ended - \
DTAR_statistics.wtime_started;
if (DTAR_rank == 0) {
char starttime_str[256];
struct tm* localstart = localtime(&(DTAR_statistics.time_started));
strftime(starttime_str, 256, "%b-%d-%Y, %H:%M:%S", localstart);
char endtime_str[256];
struct tm* localend = localtime(&(DTAR_statistics.time_ended));
strftime(endtime_str, 256, "%b-%d-%Y, %H:%M:%S", localend);
/* add two 512 blocks at the end */
DTAR_statistics.total_size += 512 * 2;
/* convert bandwidth to unit */
double agg_rate_tmp;
double agg_rate = (double) DTAR_statistics.total_size / rel_time;
const char* agg_rate_units;
mfu_format_bytes(agg_rate, &agg_rate_tmp, &agg_rate_units);
MFU_LOG(MFU_LOG_INFO, "Started: %s", starttime_str);
MFU_LOG(MFU_LOG_INFO, "Completed: %s", endtime_str);
MFU_LOG(MFU_LOG_INFO, "Total archive size: %" PRIu64, DTAR_statistics.total_size);
MFU_LOG(MFU_LOG_INFO, "Rate: %.3lf %s " \
"(%.3" PRIu64 " bytes in %.3lf seconds)", \
agg_rate_tmp, agg_rate_units, DTAR_statistics.total_size, rel_time);
}
/* clean up */
mfu_free(&fsizes);
mfu_free(&DTAR_offsets);
/* close archive file */
archive_write_free(ar);
mfu_close(DTAR_writer.name, DTAR_writer.fd_tar);
}
static int print_flist_distribution(int file_histogram,
struct distribute_option *option,
mfu_flist* pflist, int rank)
{
/* file list to use */
mfu_flist flist = *pflist;
/* get local size for each rank, and max file sizes */
uint64_t size = mfu_flist_size(flist);
uint64_t global_max_file_size;
int separators = 0;
if (file_histogram) {
/* create default separators */
create_default_separators(option, &flist, &size, &separators,
&global_max_file_size);
} else {
separators = option->separator_number;
}
/* allocate a count for each bin, initialize the bin counts to 0
* it is separator + 1 because the last bin is the last separator
* to the DISTRIBUTE_MAX */
uint64_t* dist = (uint64_t*) MFU_MALLOC((separators + 1) * sizeof(uint64_t));
/* initialize the bin counts to 0 */
for (int i = 0; i <= separators; i++) {
dist[i] = 0;
}
/* for each file, identify appropriate bin and increment its count */
for (int i = 0; i < size; i++) {
/* get the size of the file */
uint64_t file_size = mfu_flist_file_get_size(flist, i);
/* loop through the bins and find the one the file belongs to,
* set last bin to -1, if a bin is not found while looping through the
* list of file size separators, then it belongs in the last bin
* so (last file size - MAX bin) */
int max_bin_flag = -1;
for (int j = 0; j < separators; j++) {
if (file_size <= option->separators[j]) {
/* found the bin set bin index & increment its count */
dist[j]++;
/* a file for this bin was found so can't belong to
* last bin (so set the flag) & exit the loop */
max_bin_flag = 1;
break;
}
}
/* if max_bin_flag is still -1 then the file belongs to the last bin */
if (max_bin_flag < 0) {
dist[separators]++;
}
}
/* get the total sum across all of the bins */
uint64_t* disttotal = (uint64_t*) MFU_MALLOC((separators + 1) * sizeof(uint64_t));
MPI_Allreduce(dist, disttotal, (uint64_t)separators + 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);
/* Print the file distribution */
if (rank == 0) {
/* number of files in a bin */
uint64_t number;
double size_tmp;
const char* size_units;
printf("%-27s %s\n", "Range", "Number");
for (int i = 0; i <= separators; i++) {
printf("%s", "[ ");
if (i == 0) {
printf("%7.3lf %2s", 0.000, "B");
} else {
mfu_format_bytes((uint64_t)option->separators[i - 1],
&size_tmp, &size_units);
printf("%7.3lf %2s", size_tmp, size_units);
}
printf("%s", " - ");
if (file_histogram) {
mfu_format_bytes((uint64_t)option->separators[i],
&size_tmp, &size_units);
number = disttotal[i];
mfu_format_bytes((uint64_t)option->separators[i],
&size_tmp, &size_units);
printf("%7.3lf %2s ) %"PRIu64"\n", size_tmp,
size_units, number);
} else {
if (i == separators) {
number = disttotal[i];
printf("%10s ) %"PRIu64"\n", "MAX", number);
} else {
number = disttotal[i];
mfu_format_bytes((uint64_t)option->separators[i],
&size_tmp, &size_units);
printf("%7.3lf %2s ) %"PRIu64"\n", size_tmp, size_units, number);
}
}
}
}
/* free the memory used to hold bin counts */
mfu_free(&disttotal);
mfu_free(&dist);
return 0;
}
int main(int argc, char** argv)
{
uint64_t i;
int status;
uint64_t file_size;
uint64_t chunk_size = DDUP_CHUNK_SIZE;
SHA256_CTX* ctx_ptr;
MPI_Init(NULL, NULL);
mfu_init();
int rank, ranks;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &ranks);
/* pointer to mfu_walk_opts */
mfu_walk_opts_t* walk_opts = mfu_walk_opts_new();
mfu_debug_level = MFU_LOG_VERBOSE;
static struct option long_options[] = {
{"debug", 0, 0, 'd'},
{"verbose", 0, 0, 'v'},
{"quiet", 0, 0, 'q'},
{"help", 0, 0, 'h'},
{0, 0, 0, 0}
};
/* Parse options */
int usage = 0;
int help = 0;
int c;
int option_index = 0;
while ((c = getopt_long(argc, argv, "d:vqh", \
long_options, &option_index)) != -1)
{
switch (c) {
case 'd':
if (strncmp(optarg, "fatal", 5) == 0) {
mfu_debug_level = MFU_LOG_FATAL;
if (rank == 0)
MFU_LOG(MFU_LOG_INFO,
"Debug level set to: fatal");
}
else if (strncmp(optarg, "err", 3) == 0) {
mfu_debug_level = MFU_LOG_ERR;
if (rank == 0)
MFU_LOG(MFU_LOG_INFO,
"Debug level set to: "
"errors");
}
else if (strncmp(optarg, "warn", 4) == 0) {
mfu_debug_level = MFU_LOG_WARN;
if (rank == 0)
MFU_LOG(MFU_LOG_INFO,
"Debug level set to: "
"warnings");
}
else if (strncmp(optarg, "info", 4) == 0) {
mfu_debug_level = MFU_LOG_INFO;
if (rank == 0)
MFU_LOG(MFU_LOG_INFO,
"Debug level set to: info");
}
else if (strncmp(optarg, "dbg", 3) == 0) {
mfu_debug_level = MFU_LOG_DBG;
if (rank == 0)
MFU_LOG(MFU_LOG_INFO,
"Debug level set to: debug");
}
else {
if (rank == 0)
MFU_LOG(MFU_LOG_INFO,
"Debug level `%s' not "
"recognized. Defaulting to "
"`info'.", optarg);
}
case 'h':
usage = 1;
help = 1;
case 'v':
mfu_debug_level = MFU_LOG_VERBOSE;
break;
case 'q':
mfu_debug_level = MFU_LOG_NONE;
break;
case '?':
usage = 1;
help = 1;
break;
default:
usage = 1;
break;
}
}
/* check that user gave us one and only one directory */
int numargs = argc - optind;
if (numargs != 1) {
/* missing the directory, so post a message, and print usage */
if (rank == 0) {
MFU_LOG(MFU_LOG_ERR, "You must specify a directory path");
}
usage = 1;
}
/* print usage and bail if needed */
if (usage) {
if (rank == 0) {
print_usage();
}
/* set error code base on whether user requested usage or not */
if (help) {
status = 0;
} else {
status = -1;
}
MPI_Barrier(MPI_COMM_WORLD);
goto out;
}
/* get the directory name */
const char* dir = argv[optind];
/* create MPI datatypes */
MPI_Datatype key;
MPI_Datatype keysat;
mpi_type_init(&key, &keysat);
/* create DTCMP comparison operation */
DTCMP_Op cmp;
mtcmp_cmp_init(&cmp);
/* allocate buffer to read data from file */
char* chunk_buf = (char*)MFU_MALLOC(DDUP_CHUNK_SIZE);
/* allocate a file list */
mfu_flist flist = mfu_flist_new();
/* Walk the path(s) to build the flist */
mfu_flist_walk_path(dir, walk_opts, flist);
/* TODO: spread list among procs? */
/* get local number of items in flist */
uint64_t checking_files = mfu_flist_size(flist);
/* allocate memory to hold SHA256 context values */
struct file_item* file_items = (struct file_item*) MFU_MALLOC(checking_files * sizeof(*file_items));
/* Allocate two lists of length size, where each
* element has (DDUP_KEY_SIZE + 1) uint64_t values
* (id, checksum, index)
*/
size_t list_bytes = checking_files * (DDUP_KEY_SIZE + 1) * sizeof(uint64_t);
uint64_t* list = (uint64_t*) MFU_MALLOC(list_bytes);
uint64_t* new_list = (uint64_t*) MFU_MALLOC(list_bytes);
/* Initialize the list */
uint64_t* ptr = list;
uint64_t new_checking_files = 0;
for (i = 0; i < checking_files; i++) {
/* check that item is a regular file */
mode_t mode = (mode_t) mfu_flist_file_get_mode(flist, i);
if (! S_ISREG(mode)) {
continue;
}
/* get the file size */
file_size = mfu_flist_file_get_size(flist, i);
if (file_size == 0) {
/* Files with size zero are not interesting at all */
continue;
}
/* for first pass, group all files with same file size */
ptr[0] = file_size;
/* we'll leave the middle part of the key unset */
/* record our index in flist */
ptr[DDUP_KEY_SIZE] = i;
/* initialize the SHA256 hash state for this file */
SHA256_Init(&file_items[i].ctx);
/* increment our file count */
new_checking_files++;
/* advance to next spot in the list */
ptr += DDUP_KEY_SIZE + 1;
}
/* reduce our list count based on any files filtered out above */
checking_files = new_checking_files;
/* allocate arrays to hold result from DTCMP_Rankv call to
* assign group and rank values to each item */
uint64_t output_bytes = checking_files * sizeof(uint64_t);
uint64_t* group_id = (uint64_t*) MFU_MALLOC(output_bytes);
uint64_t* group_ranks = (uint64_t*) MFU_MALLOC(output_bytes);
uint64_t* group_rank = (uint64_t*) MFU_MALLOC(output_bytes);
/* get total number of items across all tasks */
uint64_t sum_checking_files;
MPI_Allreduce(&checking_files, &sum_checking_files, 1,
MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);
uint64_t chunk_id = 0;
while (sum_checking_files > 1) {
/* update the chunk id we'll read from all files */
chunk_id++;
/* iterate over our list and compute SHA256 value for each */
ptr = list;
for (i = 0; i < checking_files; i++) {
/* get the flist index for this item */
uint64_t idx = ptr[DDUP_KEY_SIZE];
/* look up file name */
const char* fname = mfu_flist_file_get_name(flist, idx);
/* look up file size */
file_size = mfu_flist_file_get_size(flist, idx);
/* read a chunk of data from the file into chunk_buf */
uint64_t data_size;
status = read_data(fname, chunk_buf, chunk_id,
chunk_size, file_size, &data_size);
if (status) {
/* File size has been changed, TODO: handle */
printf("failed to read file %s, maybe file "
"size has been modified during the "
"process", fname);
}
/* update the SHA256 context for this file */
ctx_ptr = &file_items[idx].ctx;
SHA256_Update(ctx_ptr, chunk_buf, data_size);
/*
* Use SHA256 value as key.
* This is actually an hack, but SHA256_Final can't
* be called multiple times with out changing ctx
*/
SHA256_CTX ctx_tmp;
memcpy(&ctx_tmp, ctx_ptr, sizeof(ctx_tmp));
SHA256_Final((unsigned char*)(ptr + 1), &ctx_tmp);
/* move on to next file in the list */
ptr += DDUP_KEY_SIZE + 1;
}
/* Assign group ids and compute group sizes */
uint64_t groups;
DTCMP_Rankv(
(int)checking_files, list,
&groups, group_id, group_ranks, group_rank,
key, keysat, cmp, DTCMP_FLAG_NONE, MPI_COMM_WORLD
);
/* any files assigned to a group of size 1 is unique,
* any files in groups sizes > 1 for which we've read
* all bytes are the same, and filter all other files
* into a new list for another iteration */
new_checking_files = 0;
ptr = list;
uint64_t* new_ptr = new_list;
for (i = 0; i < checking_files; i++) {
/* Get index into flist for this item */
uint64_t idx = ptr[DDUP_KEY_SIZE];
/* look up file name */
const char* fname = mfu_flist_file_get_name(flist, idx);
/* look up file size */
file_size = mfu_flist_file_get_size(flist, idx);
/* get a pointer to the SHA256 context for this file */
ctx_ptr = &file_items[idx].ctx;
if (group_ranks[i] == 1) {
/*
* Only one file in this group,
* mfu_flist_file_name(flist, idx) is unique
*/
} else if (file_size <= (chunk_id * chunk_size)) {
/*
* We've run out of bytes to checksum, and we
* still have a group size > 1
* mfu_flist_file_name(flist, idx) is a
* duplicate with other files that also have
* matching group_id[i]
*/
unsigned char digest[SHA256_DIGEST_LENGTH];
SHA256_Final(digest, ctx_ptr);
char digest_string[SHA256_DIGEST_LENGTH * 2 + 1];
dump_sha256_digest(digest_string, digest);
printf("%s %s\n", fname, digest_string);
} else {
/* Have multiple files with the same checksum,
* but still have bytes left to read, so keep
* this file
*/
/* use new group ID to segregate files,
* this id will be unique for all files of the
* same size and having the same hash up to
* this point */
new_ptr[0] = group_id[i];
/* Copy over flist index into new list entry */
new_ptr[DDUP_KEY_SIZE] = idx;
/* got one more in the new list */
new_checking_files++;
/* move on to next item in new list */
new_ptr += DDUP_KEY_SIZE + 1;
MFU_LOG(MFU_LOG_DBG, "checking file "
"\"%s\" for chunk index %d of size %"
PRIu64"\n", fname, (int)chunk_id,
chunk_size);
}
/* move on to next file in the list */
ptr += DDUP_KEY_SIZE + 1;
}
/* Swap lists */
uint64_t* tmp_list;
tmp_list = list;
list = new_list;
new_list = tmp_list;
/* Update size of current list */
checking_files = new_checking_files;
/* Get new global list size */
MPI_Allreduce(&checking_files, &sum_checking_files, 1,
MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);
}
/* free the walk options */
mfu_walk_opts_delete(&walk_opts);
mfu_free(&group_rank);
mfu_free(&group_ranks);
mfu_free(&group_id);
mfu_free(&new_list);
mfu_free(&list);
mfu_free(&file_items);
mfu_free(&chunk_buf);
mfu_flist_free(&flist);
mtcmp_cmp_fini(&cmp);
mpi_type_fini(&key, &keysat);
status = 0;
out:
mfu_finalize();
MPI_Finalize();
return status;
}
