int DTCMP_Rank(
int count,
const void* buf,
uint64_t* groups,
uint64_t group_id[],
uint64_t group_ranks[],
uint64_t group_rank[],
MPI_Datatype key,
MPI_Datatype keysat,
DTCMP_Op cmp,
DTCMP_Flags hints,
MPI_Comm comm)
{
/* check that we're initialized */
if (dtcmp_init_count <= 0) {
return DTCMP_FAILURE;
}
/* defer to Rankv for now */
return DTCMP_Rankv(
count, buf,
groups, group_id, group_ranks, group_rank,
key, keysat, cmp, hints, comm
);
}
int main(int argc, char** argv)
{
uint64_t i;
int status;
uint64_t file_size;
uint64_t chunk_size = DDUP_CHUNK_SIZE;
SHA256_CTX* ctx_ptr;
MPI_Init(NULL, NULL);
mfu_init();
int rank, ranks;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &ranks);
/* pointer to mfu_walk_opts */
mfu_walk_opts_t* walk_opts = mfu_walk_opts_new();
mfu_debug_level = MFU_LOG_VERBOSE;
static struct option long_options[] = {
{"debug", 0, 0, 'd'},
{"verbose", 0, 0, 'v'},
{"quiet", 0, 0, 'q'},
{"help", 0, 0, 'h'},
{0, 0, 0, 0}
};
/* Parse options */
int usage = 0;
int help = 0;
int c;
int option_index = 0;
while ((c = getopt_long(argc, argv, "d:vqh", \
long_options, &option_index)) != -1)
{
switch (c) {
case 'd':
if (strncmp(optarg, "fatal", 5) == 0) {
mfu_debug_level = MFU_LOG_FATAL;
if (rank == 0)
MFU_LOG(MFU_LOG_INFO,
"Debug level set to: fatal");
}
else if (strncmp(optarg, "err", 3) == 0) {
mfu_debug_level = MFU_LOG_ERR;
if (rank == 0)
MFU_LOG(MFU_LOG_INFO,
"Debug level set to: "
"errors");
}
else if (strncmp(optarg, "warn", 4) == 0) {
mfu_debug_level = MFU_LOG_WARN;
if (rank == 0)
MFU_LOG(MFU_LOG_INFO,
"Debug level set to: "
"warnings");
}
else if (strncmp(optarg, "info", 4) == 0) {
mfu_debug_level = MFU_LOG_INFO;
if (rank == 0)
MFU_LOG(MFU_LOG_INFO,
"Debug level set to: info");
}
else if (strncmp(optarg, "dbg", 3) == 0) {
mfu_debug_level = MFU_LOG_DBG;
if (rank == 0)
MFU_LOG(MFU_LOG_INFO,
"Debug level set to: debug");
}
else {
if (rank == 0)
MFU_LOG(MFU_LOG_INFO,
"Debug level `%s' not "
"recognized. Defaulting to "
"`info'.", optarg);
}
case 'h':
usage = 1;
help = 1;
case 'v':
mfu_debug_level = MFU_LOG_VERBOSE;
break;
case 'q':
mfu_debug_level = MFU_LOG_NONE;
break;
case '?':
usage = 1;
help = 1;
break;
default:
usage = 1;
break;
}
}
/* check that user gave us one and only one directory */
int numargs = argc - optind;
if (numargs != 1) {
/* missing the directory, so post a message, and print usage */
if (rank == 0) {
MFU_LOG(MFU_LOG_ERR, "You must specify a directory path");
}
usage = 1;
}
/* print usage and bail if needed */
if (usage) {
if (rank == 0) {
print_usage();
}
/* set error code base on whether user requested usage or not */
if (help) {
status = 0;
} else {
status = -1;
}
MPI_Barrier(MPI_COMM_WORLD);
goto out;
}
/* get the directory name */
const char* dir = argv[optind];
/* create MPI datatypes */
MPI_Datatype key;
MPI_Datatype keysat;
mpi_type_init(&key, &keysat);
/* create DTCMP comparison operation */
DTCMP_Op cmp;
mtcmp_cmp_init(&cmp);
/* allocate buffer to read data from file */
char* chunk_buf = (char*)MFU_MALLOC(DDUP_CHUNK_SIZE);
/* allocate a file list */
mfu_flist flist = mfu_flist_new();
/* Walk the path(s) to build the flist */
mfu_flist_walk_path(dir, walk_opts, flist);
/* TODO: spread list among procs? */
/* get local number of items in flist */
uint64_t checking_files = mfu_flist_size(flist);
/* allocate memory to hold SHA256 context values */
struct file_item* file_items = (struct file_item*) MFU_MALLOC(checking_files * sizeof(*file_items));
/* Allocate two lists of length size, where each
* element has (DDUP_KEY_SIZE + 1) uint64_t values
* (id, checksum, index)
*/
size_t list_bytes = checking_files * (DDUP_KEY_SIZE + 1) * sizeof(uint64_t);
uint64_t* list = (uint64_t*) MFU_MALLOC(list_bytes);
uint64_t* new_list = (uint64_t*) MFU_MALLOC(list_bytes);
/* Initialize the list */
uint64_t* ptr = list;
uint64_t new_checking_files = 0;
for (i = 0; i < checking_files; i++) {
/* check that item is a regular file */
mode_t mode = (mode_t) mfu_flist_file_get_mode(flist, i);
if (! S_ISREG(mode)) {
continue;
}
/* get the file size */
file_size = mfu_flist_file_get_size(flist, i);
if (file_size == 0) {
/* Files with size zero are not interesting at all */
continue;
}
/* for first pass, group all files with same file size */
ptr[0] = file_size;
/* we'll leave the middle part of the key unset */
/* record our index in flist */
ptr[DDUP_KEY_SIZE] = i;
/* initialize the SHA256 hash state for this file */
SHA256_Init(&file_items[i].ctx);
/* increment our file count */
new_checking_files++;
/* advance to next spot in the list */
ptr += DDUP_KEY_SIZE + 1;
}
/* reduce our list count based on any files filtered out above */
checking_files = new_checking_files;
/* allocate arrays to hold result from DTCMP_Rankv call to
* assign group and rank values to each item */
uint64_t output_bytes = checking_files * sizeof(uint64_t);
uint64_t* group_id = (uint64_t*) MFU_MALLOC(output_bytes);
uint64_t* group_ranks = (uint64_t*) MFU_MALLOC(output_bytes);
uint64_t* group_rank = (uint64_t*) MFU_MALLOC(output_bytes);
/* get total number of items across all tasks */
uint64_t sum_checking_files;
MPI_Allreduce(&checking_files, &sum_checking_files, 1,
MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);
uint64_t chunk_id = 0;
while (sum_checking_files > 1) {
/* update the chunk id we'll read from all files */
chunk_id++;
/* iterate over our list and compute SHA256 value for each */
ptr = list;
for (i = 0; i < checking_files; i++) {
/* get the flist index for this item */
uint64_t idx = ptr[DDUP_KEY_SIZE];
/* look up file name */
const char* fname = mfu_flist_file_get_name(flist, idx);
/* look up file size */
file_size = mfu_flist_file_get_size(flist, idx);
/* read a chunk of data from the file into chunk_buf */
uint64_t data_size;
status = read_data(fname, chunk_buf, chunk_id,
chunk_size, file_size, &data_size);
if (status) {
/* File size has been changed, TODO: handle */
printf("failed to read file %s, maybe file "
"size has been modified during the "
"process", fname);
}
/* update the SHA256 context for this file */
ctx_ptr = &file_items[idx].ctx;
SHA256_Update(ctx_ptr, chunk_buf, data_size);
/*
* Use SHA256 value as key.
* This is actually an hack, but SHA256_Final can't
* be called multiple times with out changing ctx
*/
SHA256_CTX ctx_tmp;
memcpy(&ctx_tmp, ctx_ptr, sizeof(ctx_tmp));
SHA256_Final((unsigned char*)(ptr + 1), &ctx_tmp);
/* move on to next file in the list */
ptr += DDUP_KEY_SIZE + 1;
}
/* Assign group ids and compute group sizes */
uint64_t groups;
DTCMP_Rankv(
(int)checking_files, list,
&groups, group_id, group_ranks, group_rank,
key, keysat, cmp, DTCMP_FLAG_NONE, MPI_COMM_WORLD
);
/* any files assigned to a group of size 1 is unique,
* any files in groups sizes > 1 for which we've read
* all bytes are the same, and filter all other files
* into a new list for another iteration */
new_checking_files = 0;
ptr = list;
uint64_t* new_ptr = new_list;
for (i = 0; i < checking_files; i++) {
/* Get index into flist for this item */
uint64_t idx = ptr[DDUP_KEY_SIZE];
/* look up file name */
const char* fname = mfu_flist_file_get_name(flist, idx);
/* look up file size */
file_size = mfu_flist_file_get_size(flist, idx);
/* get a pointer to the SHA256 context for this file */
ctx_ptr = &file_items[idx].ctx;
if (group_ranks[i] == 1) {
/*
* Only one file in this group,
* mfu_flist_file_name(flist, idx) is unique
*/
} else if (file_size <= (chunk_id * chunk_size)) {
/*
* We've run out of bytes to checksum, and we
* still have a group size > 1
* mfu_flist_file_name(flist, idx) is a
* duplicate with other files that also have
* matching group_id[i]
*/
unsigned char digest[SHA256_DIGEST_LENGTH];
SHA256_Final(digest, ctx_ptr);
char digest_string[SHA256_DIGEST_LENGTH * 2 + 1];
dump_sha256_digest(digest_string, digest);
printf("%s %s\n", fname, digest_string);
} else {
/* Have multiple files with the same checksum,
* but still have bytes left to read, so keep
* this file
*/
/* use new group ID to segregate files,
* this id will be unique for all files of the
* same size and having the same hash up to
* this point */
new_ptr[0] = group_id[i];
/* Copy over flist index into new list entry */
new_ptr[DDUP_KEY_SIZE] = idx;
/* got one more in the new list */
new_checking_files++;
/* move on to next item in new list */
new_ptr += DDUP_KEY_SIZE + 1;
MFU_LOG(MFU_LOG_DBG, "checking file "
"\"%s\" for chunk index %d of size %"
PRIu64"\n", fname, (int)chunk_id,
chunk_size);
}
/* move on to next file in the list */
ptr += DDUP_KEY_SIZE + 1;
}
/* Swap lists */
uint64_t* tmp_list;
tmp_list = list;
list = new_list;
new_list = tmp_list;
/* Update size of current list */
checking_files = new_checking_files;
/* Get new global list size */
MPI_Allreduce(&checking_files, &sum_checking_files, 1,
MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);
}
/* free the walk options */
mfu_walk_opts_delete(&walk_opts);
mfu_free(&group_rank);
mfu_free(&group_ranks);
mfu_free(&group_id);
mfu_free(&new_list);
mfu_free(&list);
mfu_free(&file_items);
mfu_free(&chunk_buf);
mfu_flist_free(&flist);
mtcmp_cmp_fini(&cmp);
mpi_type_fini(&key, &keysat);
status = 0;
out:
mfu_finalize();
MPI_Finalize();
return status;
}