/* TODO: condionally set setuid and setgid bits? */
void DCOPY_copy_permissions(
mfu_flist flist,
uint64_t idx,
const char* dest_path)
{
/* get mode and type */
mfu_filetype type = mfu_flist_file_get_type(flist, idx);
mode_t mode = (mode_t) mfu_flist_file_get_mode(flist, idx);
/* change mode */
if(type != MFU_TYPE_LINK) {
if(mfu_chmod(dest_path, mode) != 0) {
MFU_LOG(MFU_LOG_ERR, "Failed to change permissions on %s chmod() errno=%d %s",
dest_path, errno, strerror(errno)
);
}
}
return;
}
/* removes list of items, sets write bits on directories from
* top-to-bottom, then removes items one level at a time starting
* from the deepest */
void mfu_flist_unlink(mfu_flist flist)
{
int level;
/* wait for all tasks and start timer */
MPI_Barrier(MPI_COMM_WORLD);
double start_remove = MPI_Wtime();
/* split files into separate lists by directory depth */
int levels, minlevel;
mfu_flist* lists;
mfu_flist_array_by_depth(flist, &levels, &minlevel, &lists);
#if 0
/* dive from shallow to deep, ensure all directories have write bit set */
for (level = 0; level < levels; level++) {
/* get list of items for this level */
mfu_flist list = lists[level];
/* determine whether we have details at this level */
int detail = mfu_flist_have_detail(list);
/* iterate over items and set write bit on directories if needed */
uint64_t idx;
uint64_t size = mfu_flist_size(list);
for (idx = 0; idx < size; idx++) {
/* check whether we have a directory */
mfu_filetype type = mfu_flist_file_get_type(list, idx);
if (type == MFU_TYPE_DIR) {
/* assume we have to set the bit */
int set_write_bit = 1;
if (detail) {
mode_t mode = (mode_t) mfu_flist_file_get_mode(list, idx);
if (mode & S_IWUSR) {
/* we have the mode of the file, and the bit is already set */
set_write_bit = 0;
}
}
/* set the bit if needed */
if (set_write_bit) {
const char* name = mfu_flist_file_get_name(list, idx);
int rc = chmod(name, S_IRWXU);
if (rc != 0) {
MFU_LOG(MFU_LOG_ERR, "Failed to chmod directory `%s' (errno=%d %s)",
name, errno, strerror(errno)
);
}
}
}
}
/* wait for all procs to finish before we start next level */
MPI_Barrier(MPI_COMM_WORLD);
}
#endif
/* now remove files starting from deepest level */
for (level = levels - 1; level >= 0; level--) {
double start = MPI_Wtime();
/* get list of items for this level */
mfu_flist list = lists[level];
uint64_t count = 0;
//remove_direct(list, &count);
remove_spread(list, &count);
// remove_map(list, &count);
// remove_sort(list, &count);
// remove_libcircle(list, &count);
// TODO: remove sort w/ spread
/* wait for all procs to finish before we start
* with files at next level */
MPI_Barrier(MPI_COMM_WORLD);
double end = MPI_Wtime();
if (mfu_debug_level >= MFU_LOG_VERBOSE) {
uint64_t min, max, sum;
MPI_Allreduce(&count, &min, 1, MPI_UINT64_T, MPI_MIN, MPI_COMM_WORLD);
MPI_Allreduce(&count, &max, 1, MPI_UINT64_T, MPI_MAX, MPI_COMM_WORLD);
MPI_Allreduce(&count, &sum, 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);
double rate = 0.0;
if (end - start > 0.0) {
rate = (double)sum / (end - start);
}
double time_diff = end - start;
if (mfu_rank == 0) {
printf("level=%d min=%lu max=%lu sum=%lu rate=%f secs=%f\n",
(minlevel + level), (unsigned long)min, (unsigned long)max, (unsigned long)sum, rate, time_diff
);
fflush(stdout);
}
}
}
mfu_flist_array_free(levels, &lists);
/* wait for all tasks and stop timer */
MPI_Barrier(MPI_COMM_WORLD);
double end_remove = MPI_Wtime();
/* report remove count, time, and rate */
if (mfu_debug_level >= MFU_LOG_VERBOSE && mfu_rank == 0) {
uint64_t all_count = mfu_flist_global_size(flist);
double time_diff = end_remove - start_remove;
double rate = 0.0;
if (time_diff > 0.0) {
rate = ((double)all_count) / time_diff;
}
printf("Removed %lu items in %f seconds (%f items/sec)\n",
all_count, time_diff, rate
);
}
return;
}
int main(int argc, char* argv[])
{
MPI_Init(&argc, &argv);
mfu_init();
/* get our rank and number of ranks in the job */
int rank, ranks;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &ranks);
/* pointer to mfu_walk_opts */
mfu_walk_opts_t* walk_opts = mfu_walk_opts_new();
uint64_t idx;
int option_index = 0;
int usage = 0;
int report = 0;
unsigned int numpaths = 0;
mfu_param_path* paths = NULL;
unsigned long long bytes;
/* verbose by default */
mfu_debug_level = MFU_LOG_VERBOSE;
/* default to 1MB stripe size, stripe across all OSTs, and all files are candidates */
int stripes = -1;
uint64_t stripe_size = 1048576;
uint64_t min_size = 0;
static struct option long_options[] = {
{"count", 1, 0, 'c'},
{"size", 1, 0, 's'},
{"minsize", 1, 0, 'm'},
{"report", 0, 0, 'r'},
{"progress", 1, 0, 'P'},
{"verbose", 0, 0, 'v'},
{"quiet", 0, 0, 'q'},
{"help", 0, 0, 'h'},
{0, 0, 0, 0}
};
while (1) {
int c = getopt_long(argc, argv, "c:s:m:rvqh",
long_options, &option_index);
if (c == -1) {
break;
}
switch (c) {
case 'c':
/* stripe count */
stripes = atoi(optarg);
break;
case 's':
/* stripe size in bytes */
if (mfu_abtoull(optarg, &bytes) != MFU_SUCCESS) {
if (rank == 0) {
MFU_LOG(MFU_LOG_ERR, "Failed to parse stripe size: %s", optarg);
}
MPI_Abort(MPI_COMM_WORLD, 1);
}
stripe_size = (uint64_t)bytes;
break;
case 'm':
/* min file size in bytes */
if (mfu_abtoull(optarg, &bytes) != MFU_SUCCESS) {
if (rank == 0) {
MFU_LOG(MFU_LOG_ERR, "Failed to parse minimum file size: %s", optarg);
}
MPI_Abort(MPI_COMM_WORLD, 1);
}
min_size = (uint64_t)bytes;
break;
case 'r':
/* report striping info */
report = 1;
break;
case 'P':
mfu_progress_timeout = atoi(optarg);
break;
case 'v':
mfu_debug_level = MFU_LOG_VERBOSE;
break;
case 'q':
mfu_debug_level = MFU_LOG_NONE;
break;
case 'h':
/* display usage */
usage = 1;
break;
case '?':
/* display usage */
usage = 1;
break;
default:
if (rank == 0) {
printf("?? getopt returned character code 0%o ??\n", c);
}
}
}
/* check that we got a valid progress value */
if (mfu_progress_timeout < 0) {
if (rank == 0) {
MFU_LOG(MFU_LOG_ERR, "Seconds in --progress must be non-negative: %d invalid", mfu_progress_timeout);
}
usage = 1;
}
/* paths to walk come after the options */
if (optind < argc) {
/* determine number of paths specified by user */
numpaths = argc - optind;
/* allocate space for each path */
paths = (mfu_param_path*) MFU_MALLOC((size_t)numpaths * sizeof(mfu_param_path));
/* process each path */
char** p = &argv[optind];
mfu_param_path_set_all((uint64_t)numpaths, (const char**)p, paths);
optind += numpaths;
} else {
usage = 1;
}
/* if we need to print usage, print it and exit */
if (usage) {
if (rank == 0) {
print_usage();
}
mfu_finalize();
MPI_Finalize();
return 1;
}
/* nothing to do if lustre support is disabled */
#ifndef LUSTRE_SUPPORT
if (rank == 0) {
MFU_LOG(MFU_LOG_ERR, "Lustre support is disabled.");
}
MPI_Abort(MPI_COMM_WORLD, 1);
#endif
/* stripe count must be -1 for all available or greater than 0 */
if (stripes < -1) {
if (rank == 0) {
MFU_LOG(MFU_LOG_ERR, "Stripe count must be -1 for all servers, 0 for lustre file system default, or a positive value");
}
MPI_Abort(MPI_COMM_WORLD, 1);
}
/* lustre requires stripe sizes to be aligned */
if (stripe_size > 0 && stripe_size % 65536 != 0) {
if (rank == 0) {
MFU_LOG(MFU_LOG_ERR, "Stripe size must be a multiple of 65536");
}
MPI_Abort(MPI_COMM_WORLD, 1);
}
/* TODO: verify that source / target are on Lustre */
/* walk list of input paths and stat as we walk */
mfu_flist flist = mfu_flist_new();
mfu_flist_walk_param_paths(numpaths, paths, walk_opts, flist);
/* filter down our list to files which don't meet our striping requirements */
mfu_flist filtered = filter_list(flist, stripes, stripe_size, min_size, &create_prog_count_total, &stripe_prog_bytes_total);
mfu_flist_free(&flist);
MPI_Barrier(MPI_COMM_WORLD);
/* report the file size and stripe count of all files we found */
if (report) {
/* report the files in our filtered list */
stripe_info_report(filtered);
/* free the paths and our list */
mfu_flist_free(&filtered);
mfu_param_path_free_all(numpaths, paths);
mfu_free(&paths);
/* finalize */
mfu_finalize();
MPI_Finalize();
return 0;
}
/* generate a global suffix for our temp files and have each node check it's list */
char suffix[8];
uint64_t retry;
/* seed our random number generator */
srand(time(NULL));
/* keep trying to make a valid random suffix...*/
do {
uint64_t attempt = 0;
/* make rank 0 responsible for generating a random suffix */
if (rank == 0) {
generate_suffix(suffix, sizeof(suffix));
}
/* broadcast the random suffix to all ranks */
MPI_Bcast(suffix, sizeof(suffix), MPI_CHAR, 0, MPI_COMM_WORLD);
/* check that the file doesn't already exist */
uint64_t size = mfu_flist_size(filtered);
for (idx = 0; idx < size; idx++) {
char temp_path[PATH_MAX];
strcpy(temp_path, mfu_flist_file_get_name(filtered, idx));
strcat(temp_path, suffix);
if(!mfu_access(temp_path, F_OK)) {
/* the file already exists */
attempt = 1;
break;
}
}
/* do a reduce to figure out if a rank has a file collision */
MPI_Allreduce(&attempt, &retry, 1, MPI_UINT64_T, MPI_MAX, MPI_COMM_WORLD);
} while(retry != 0);
/* initialize progress messages while creating files */
create_prog_count = 0;
create_prog = mfu_progress_start(mfu_progress_timeout, 1, MPI_COMM_WORLD, create_progress_fn);
/* create new files so we can restripe */
uint64_t size = mfu_flist_size(filtered);
for (idx = 0; idx < size; idx++) {
char temp_path[PATH_MAX];
strcpy(temp_path, mfu_flist_file_get_name(filtered, idx));
strcat(temp_path, suffix);
/* create a striped file at the temp file path */
mfu_stripe_set(temp_path, stripe_size, stripes);
/* update our status for file create progress */
create_prog_count++;
mfu_progress_update(&create_prog_count, create_prog);
}
/* finalize file create progress messages */
mfu_progress_complete(&create_prog_count, &create_prog);
MPI_Barrier(MPI_COMM_WORLD);
/* initialize progress messages while copying data */
stripe_prog_bytes = 0;
stripe_prog = mfu_progress_start(mfu_progress_timeout, 1, MPI_COMM_WORLD, stripe_progress_fn);
/* found a suffix, now we need to break our files into chunks based on stripe size */
mfu_file_chunk* file_chunks = mfu_file_chunk_list_alloc(filtered, stripe_size);
mfu_file_chunk* p = file_chunks;
while (p != NULL) {
/* build path to temp file */
char temp_path[PATH_MAX];
strcpy(temp_path, p->name);
strcat(temp_path, suffix);
/* write each chunk in our list */
write_file_chunk(p, temp_path);
/* move on to next file chunk */
p = p->next;
}
mfu_file_chunk_list_free(&file_chunks);
/* finalize progress messages */
mfu_progress_complete(&stripe_prog_bytes, &stripe_prog);
MPI_Barrier(MPI_COMM_WORLD);
/* remove input file and rename temp file */
for (idx = 0; idx < size; idx++) {
/* build path to temp file */
const char *in_path = mfu_flist_file_get_name(filtered, idx);
char out_path[PATH_MAX];
strcpy(out_path, in_path);
strcat(out_path, suffix);
/* change the mode of the newly restriped file to be the same as the old one */
mode_t mode = (mode_t) mfu_flist_file_get_mode(filtered, idx);
if (mfu_chmod(out_path, mode) != 0) {
MFU_LOG(MFU_LOG_ERR, "Failed to chmod file %s (%s)", out_path, strerror(errno));
MPI_Abort(MPI_COMM_WORLD, 1);
}
/* rename the new, restriped file to the old name */
if (rename(out_path, in_path) != 0) {
MFU_LOG(MFU_LOG_ERR, "Failed to rename file %s to %s", out_path, in_path);
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
/* wait for everyone to finish */
MPI_Barrier(MPI_COMM_WORLD);
/* free the walk options */
mfu_walk_opts_delete(&walk_opts);
/* free filtered list, path parameters */
mfu_flist_free(&filtered);
mfu_param_path_free_all(numpaths, paths);
mfu_free(&paths);
mfu_finalize();
MPI_Finalize();
return 0;
}
/* print information about a file given the index and rank (used in print_files) */
static void print_file(mfu_flist flist, uint64_t idx)
{
/* store types as strings for print_file */
char type_str_unknown[] = "UNK";
char type_str_dir[] = "DIR";
char type_str_file[] = "REG";
char type_str_link[] = "LNK";
/* get filename */
const char* file = mfu_flist_file_get_name(flist, idx);
if (mfu_flist_have_detail(flist)) {
/* get mode */
mode_t mode = (mode_t) mfu_flist_file_get_mode(flist, idx);
//uint32_t uid = (uint32_t) mfu_flist_file_get_uid(flist, idx);
//uint32_t gid = (uint32_t) mfu_flist_file_get_gid(flist, idx);
uint64_t acc = mfu_flist_file_get_atime(flist, idx);
uint64_t mod = mfu_flist_file_get_mtime(flist, idx);
uint64_t cre = mfu_flist_file_get_ctime(flist, idx);
uint64_t size = mfu_flist_file_get_size(flist, idx);
const char* username = mfu_flist_file_get_username(flist, idx);
const char* groupname = mfu_flist_file_get_groupname(flist, idx);
char access_s[30];
char modify_s[30];
char create_s[30];
time_t access_t = (time_t) acc;
time_t modify_t = (time_t) mod;
time_t create_t = (time_t) cre;
size_t access_rc = strftime(access_s, sizeof(access_s) - 1, "%FT%T", localtime(&access_t));
//size_t modify_rc = strftime(modify_s, sizeof(modify_s) - 1, "%FT%T", localtime(&modify_t));
size_t modify_rc = strftime(modify_s, sizeof(modify_s) - 1, "%b %e %Y %H:%M", localtime(&modify_t));
size_t create_rc = strftime(create_s, sizeof(create_s) - 1, "%FT%T", localtime(&create_t));
if (access_rc == 0 || modify_rc == 0 || create_rc == 0) {
/* error */
access_s[0] = '\0';
modify_s[0] = '\0';
create_s[0] = '\0';
}
char mode_format[11];
mfu_format_mode(mode, mode_format);
double size_tmp;
const char* size_units;
mfu_format_bytes(size, &size_tmp, &size_units);
printf("%s %s %s %7.3f %2s %s %s\n",
mode_format, username, groupname,
size_tmp, size_units, modify_s, file
);
#if 0
printf("%s %s %s A%s M%s C%s %lu %s\n",
mode_format, username, groupname,
access_s, modify_s, create_s, (unsigned long)size, file
);
printf("Mode=%lx(%s) UID=%d(%s) GUI=%d(%s) Access=%s Modify=%s Create=%s Size=%lu File=%s\n",
(unsigned long)mode, mode_format, uid, username, gid, groupname,
access_s, modify_s, create_s, (unsigned long)size, file
);
#endif
}
else {
/* get type */
mfu_filetype type = mfu_flist_file_get_type(flist, idx);
char* type_str = type_str_unknown;
if (type == MFU_TYPE_DIR) {
type_str = type_str_dir;
}
else if (type == MFU_TYPE_FILE) {
type_str = type_str_file;
}
else if (type == MFU_TYPE_LINK) {
type_str = type_str_link;
}
printf("Type=%s File=%s\n",
type_str, file
);
}
}
int main(int argc, char** argv)
{
uint64_t i;
int status;
uint64_t file_size;
uint64_t chunk_size = DDUP_CHUNK_SIZE;
SHA256_CTX* ctx_ptr;
MPI_Init(NULL, NULL);
mfu_init();
int rank, ranks;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &ranks);
/* pointer to mfu_walk_opts */
mfu_walk_opts_t* walk_opts = mfu_walk_opts_new();
mfu_debug_level = MFU_LOG_VERBOSE;
static struct option long_options[] = {
{"debug", 0, 0, 'd'},
{"verbose", 0, 0, 'v'},
{"quiet", 0, 0, 'q'},
{"help", 0, 0, 'h'},
{0, 0, 0, 0}
};
/* Parse options */
int usage = 0;
int help = 0;
int c;
int option_index = 0;
while ((c = getopt_long(argc, argv, "d:vqh", \
long_options, &option_index)) != -1)
{
switch (c) {
case 'd':
if (strncmp(optarg, "fatal", 5) == 0) {
mfu_debug_level = MFU_LOG_FATAL;
if (rank == 0)
MFU_LOG(MFU_LOG_INFO,
"Debug level set to: fatal");
}
else if (strncmp(optarg, "err", 3) == 0) {
mfu_debug_level = MFU_LOG_ERR;
if (rank == 0)
MFU_LOG(MFU_LOG_INFO,
"Debug level set to: "
"errors");
}
else if (strncmp(optarg, "warn", 4) == 0) {
mfu_debug_level = MFU_LOG_WARN;
if (rank == 0)
MFU_LOG(MFU_LOG_INFO,
"Debug level set to: "
"warnings");
}
else if (strncmp(optarg, "info", 4) == 0) {
mfu_debug_level = MFU_LOG_INFO;
if (rank == 0)
MFU_LOG(MFU_LOG_INFO,
"Debug level set to: info");
}
else if (strncmp(optarg, "dbg", 3) == 0) {
mfu_debug_level = MFU_LOG_DBG;
if (rank == 0)
MFU_LOG(MFU_LOG_INFO,
"Debug level set to: debug");
}
else {
if (rank == 0)
MFU_LOG(MFU_LOG_INFO,
"Debug level `%s' not "
"recognized. Defaulting to "
"`info'.", optarg);
}
case 'h':
usage = 1;
help = 1;
case 'v':
mfu_debug_level = MFU_LOG_VERBOSE;
break;
case 'q':
mfu_debug_level = MFU_LOG_NONE;
break;
case '?':
usage = 1;
help = 1;
break;
default:
usage = 1;
break;
}
}
/* check that user gave us one and only one directory */
int numargs = argc - optind;
if (numargs != 1) {
/* missing the directory, so post a message, and print usage */
if (rank == 0) {
MFU_LOG(MFU_LOG_ERR, "You must specify a directory path");
}
usage = 1;
}
/* print usage and bail if needed */
if (usage) {
if (rank == 0) {
print_usage();
}
/* set error code base on whether user requested usage or not */
if (help) {
status = 0;
} else {
status = -1;
}
MPI_Barrier(MPI_COMM_WORLD);
goto out;
}
/* get the directory name */
const char* dir = argv[optind];
/* create MPI datatypes */
MPI_Datatype key;
MPI_Datatype keysat;
mpi_type_init(&key, &keysat);
/* create DTCMP comparison operation */
DTCMP_Op cmp;
mtcmp_cmp_init(&cmp);
/* allocate buffer to read data from file */
char* chunk_buf = (char*)MFU_MALLOC(DDUP_CHUNK_SIZE);
/* allocate a file list */
mfu_flist flist = mfu_flist_new();
/* Walk the path(s) to build the flist */
mfu_flist_walk_path(dir, walk_opts, flist);
/* TODO: spread list among procs? */
/* get local number of items in flist */
uint64_t checking_files = mfu_flist_size(flist);
/* allocate memory to hold SHA256 context values */
struct file_item* file_items = (struct file_item*) MFU_MALLOC(checking_files * sizeof(*file_items));
/* Allocate two lists of length size, where each
* element has (DDUP_KEY_SIZE + 1) uint64_t values
* (id, checksum, index)
*/
size_t list_bytes = checking_files * (DDUP_KEY_SIZE + 1) * sizeof(uint64_t);
uint64_t* list = (uint64_t*) MFU_MALLOC(list_bytes);
uint64_t* new_list = (uint64_t*) MFU_MALLOC(list_bytes);
/* Initialize the list */
uint64_t* ptr = list;
uint64_t new_checking_files = 0;
for (i = 0; i < checking_files; i++) {
/* check that item is a regular file */
mode_t mode = (mode_t) mfu_flist_file_get_mode(flist, i);
if (! S_ISREG(mode)) {
continue;
}
/* get the file size */
file_size = mfu_flist_file_get_size(flist, i);
if (file_size == 0) {
/* Files with size zero are not interesting at all */
continue;
}
/* for first pass, group all files with same file size */
ptr[0] = file_size;
/* we'll leave the middle part of the key unset */
/* record our index in flist */
ptr[DDUP_KEY_SIZE] = i;
/* initialize the SHA256 hash state for this file */
SHA256_Init(&file_items[i].ctx);
/* increment our file count */
new_checking_files++;
/* advance to next spot in the list */
ptr += DDUP_KEY_SIZE + 1;
}
/* reduce our list count based on any files filtered out above */
checking_files = new_checking_files;
/* allocate arrays to hold result from DTCMP_Rankv call to
* assign group and rank values to each item */
uint64_t output_bytes = checking_files * sizeof(uint64_t);
uint64_t* group_id = (uint64_t*) MFU_MALLOC(output_bytes);
uint64_t* group_ranks = (uint64_t*) MFU_MALLOC(output_bytes);
uint64_t* group_rank = (uint64_t*) MFU_MALLOC(output_bytes);
/* get total number of items across all tasks */
uint64_t sum_checking_files;
MPI_Allreduce(&checking_files, &sum_checking_files, 1,
MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);
uint64_t chunk_id = 0;
while (sum_checking_files > 1) {
/* update the chunk id we'll read from all files */
chunk_id++;
/* iterate over our list and compute SHA256 value for each */
ptr = list;
for (i = 0; i < checking_files; i++) {
/* get the flist index for this item */
uint64_t idx = ptr[DDUP_KEY_SIZE];
/* look up file name */
const char* fname = mfu_flist_file_get_name(flist, idx);
/* look up file size */
file_size = mfu_flist_file_get_size(flist, idx);
/* read a chunk of data from the file into chunk_buf */
uint64_t data_size;
status = read_data(fname, chunk_buf, chunk_id,
chunk_size, file_size, &data_size);
if (status) {
/* File size has been changed, TODO: handle */
printf("failed to read file %s, maybe file "
"size has been modified during the "
"process", fname);
}
/* update the SHA256 context for this file */
ctx_ptr = &file_items[idx].ctx;
SHA256_Update(ctx_ptr, chunk_buf, data_size);
/*
* Use SHA256 value as key.
* This is actually an hack, but SHA256_Final can't
* be called multiple times with out changing ctx
*/
SHA256_CTX ctx_tmp;
memcpy(&ctx_tmp, ctx_ptr, sizeof(ctx_tmp));
SHA256_Final((unsigned char*)(ptr + 1), &ctx_tmp);
/* move on to next file in the list */
ptr += DDUP_KEY_SIZE + 1;
}
/* Assign group ids and compute group sizes */
uint64_t groups;
DTCMP_Rankv(
(int)checking_files, list,
&groups, group_id, group_ranks, group_rank,
key, keysat, cmp, DTCMP_FLAG_NONE, MPI_COMM_WORLD
);
/* any files assigned to a group of size 1 is unique,
* any files in groups sizes > 1 for which we've read
* all bytes are the same, and filter all other files
* into a new list for another iteration */
new_checking_files = 0;
ptr = list;
uint64_t* new_ptr = new_list;
for (i = 0; i < checking_files; i++) {
/* Get index into flist for this item */
uint64_t idx = ptr[DDUP_KEY_SIZE];
/* look up file name */
const char* fname = mfu_flist_file_get_name(flist, idx);
/* look up file size */
file_size = mfu_flist_file_get_size(flist, idx);
/* get a pointer to the SHA256 context for this file */
ctx_ptr = &file_items[idx].ctx;
if (group_ranks[i] == 1) {
/*
* Only one file in this group,
* mfu_flist_file_name(flist, idx) is unique
*/
} else if (file_size <= (chunk_id * chunk_size)) {
/*
* We've run out of bytes to checksum, and we
* still have a group size > 1
* mfu_flist_file_name(flist, idx) is a
* duplicate with other files that also have
* matching group_id[i]
*/
unsigned char digest[SHA256_DIGEST_LENGTH];
SHA256_Final(digest, ctx_ptr);
char digest_string[SHA256_DIGEST_LENGTH * 2 + 1];
dump_sha256_digest(digest_string, digest);
printf("%s %s\n", fname, digest_string);
} else {
/* Have multiple files with the same checksum,
* but still have bytes left to read, so keep
* this file
*/
/* use new group ID to segregate files,
* this id will be unique for all files of the
* same size and having the same hash up to
* this point */
new_ptr[0] = group_id[i];
/* Copy over flist index into new list entry */
new_ptr[DDUP_KEY_SIZE] = idx;
/* got one more in the new list */
new_checking_files++;
/* move on to next item in new list */
new_ptr += DDUP_KEY_SIZE + 1;
MFU_LOG(MFU_LOG_DBG, "checking file "
"\"%s\" for chunk index %d of size %"
PRIu64"\n", fname, (int)chunk_id,
chunk_size);
}
/* move on to next file in the list */
ptr += DDUP_KEY_SIZE + 1;
}
/* Swap lists */
uint64_t* tmp_list;
tmp_list = list;
list = new_list;
new_list = tmp_list;
/* Update size of current list */
checking_files = new_checking_files;
/* Get new global list size */
MPI_Allreduce(&checking_files, &sum_checking_files, 1,
MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);
}
/* free the walk options */
mfu_walk_opts_delete(&walk_opts);
mfu_free(&group_rank);
mfu_free(&group_ranks);
mfu_free(&group_id);
mfu_free(&new_list);
mfu_free(&list);
mfu_free(&file_items);
mfu_free(&chunk_buf);
mfu_flist_free(&flist);
mtcmp_cmp_fini(&cmp);
mpi_type_fini(&key, &keysat);
status = 0;
out:
mfu_finalize();
MPI_Finalize();
return status;
}
