int main(int argc, char* argv[])
{
MPI_Init(&argc, &argv);
mfu_init();
/* get our rank and number of ranks in the job */
int rank, ranks;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &ranks);
/* pointer to mfu_walk_opts */
mfu_walk_opts_t* walk_opts = mfu_walk_opts_new();
uint64_t idx;
int option_index = 0;
int usage = 0;
int report = 0;
unsigned int numpaths = 0;
mfu_param_path* paths = NULL;
unsigned long long bytes;
/* verbose by default */
mfu_debug_level = MFU_LOG_VERBOSE;
/* default to 1MB stripe size, stripe across all OSTs, and all files are candidates */
int stripes = -1;
uint64_t stripe_size = 1048576;
uint64_t min_size = 0;
static struct option long_options[] = {
{"count", 1, 0, 'c'},
{"size", 1, 0, 's'},
{"minsize", 1, 0, 'm'},
{"report", 0, 0, 'r'},
{"progress", 1, 0, 'P'},
{"verbose", 0, 0, 'v'},
{"quiet", 0, 0, 'q'},
{"help", 0, 0, 'h'},
{0, 0, 0, 0}
};
while (1) {
int c = getopt_long(argc, argv, "c:s:m:rvqh",
long_options, &option_index);
if (c == -1) {
break;
}
switch (c) {
case 'c':
/* stripe count */
stripes = atoi(optarg);
break;
case 's':
/* stripe size in bytes */
if (mfu_abtoull(optarg, &bytes) != MFU_SUCCESS) {
if (rank == 0) {
MFU_LOG(MFU_LOG_ERR, "Failed to parse stripe size: %s", optarg);
}
MPI_Abort(MPI_COMM_WORLD, 1);
}
stripe_size = (uint64_t)bytes;
break;
case 'm':
/* min file size in bytes */
if (mfu_abtoull(optarg, &bytes) != MFU_SUCCESS) {
if (rank == 0) {
MFU_LOG(MFU_LOG_ERR, "Failed to parse minimum file size: %s", optarg);
}
MPI_Abort(MPI_COMM_WORLD, 1);
}
min_size = (uint64_t)bytes;
break;
case 'r':
/* report striping info */
report = 1;
break;
case 'P':
mfu_progress_timeout = atoi(optarg);
break;
case 'v':
mfu_debug_level = MFU_LOG_VERBOSE;
break;
case 'q':
mfu_debug_level = MFU_LOG_NONE;
break;
case 'h':
/* display usage */
usage = 1;
break;
case '?':
/* display usage */
usage = 1;
break;
default:
if (rank == 0) {
printf("?? getopt returned character code 0%o ??\n", c);
}
}
}
/* check that we got a valid progress value */
if (mfu_progress_timeout < 0) {
if (rank == 0) {
MFU_LOG(MFU_LOG_ERR, "Seconds in --progress must be non-negative: %d invalid", mfu_progress_timeout);
}
usage = 1;
}
/* paths to walk come after the options */
if (optind < argc) {
/* determine number of paths specified by user */
numpaths = argc - optind;
/* allocate space for each path */
paths = (mfu_param_path*) MFU_MALLOC((size_t)numpaths * sizeof(mfu_param_path));
/* process each path */
char** p = &argv[optind];
mfu_param_path_set_all((uint64_t)numpaths, (const char**)p, paths);
optind += numpaths;
} else {
usage = 1;
}
/* if we need to print usage, print it and exit */
if (usage) {
if (rank == 0) {
print_usage();
}
mfu_finalize();
MPI_Finalize();
return 1;
}
/* nothing to do if lustre support is disabled */
#ifndef LUSTRE_SUPPORT
if (rank == 0) {
MFU_LOG(MFU_LOG_ERR, "Lustre support is disabled.");
}
MPI_Abort(MPI_COMM_WORLD, 1);
#endif
/* stripe count must be -1 for all available or greater than 0 */
if (stripes < -1) {
if (rank == 0) {
MFU_LOG(MFU_LOG_ERR, "Stripe count must be -1 for all servers, 0 for lustre file system default, or a positive value");
}
MPI_Abort(MPI_COMM_WORLD, 1);
}
/* lustre requires stripe sizes to be aligned */
if (stripe_size > 0 && stripe_size % 65536 != 0) {
if (rank == 0) {
MFU_LOG(MFU_LOG_ERR, "Stripe size must be a multiple of 65536");
}
MPI_Abort(MPI_COMM_WORLD, 1);
}
/* TODO: verify that source / target are on Lustre */
/* walk list of input paths and stat as we walk */
mfu_flist flist = mfu_flist_new();
mfu_flist_walk_param_paths(numpaths, paths, walk_opts, flist);
/* filter down our list to files which don't meet our striping requirements */
mfu_flist filtered = filter_list(flist, stripes, stripe_size, min_size, &create_prog_count_total, &stripe_prog_bytes_total);
mfu_flist_free(&flist);
MPI_Barrier(MPI_COMM_WORLD);
/* report the file size and stripe count of all files we found */
if (report) {
/* report the files in our filtered list */
stripe_info_report(filtered);
/* free the paths and our list */
mfu_flist_free(&filtered);
mfu_param_path_free_all(numpaths, paths);
mfu_free(&paths);
/* finalize */
mfu_finalize();
MPI_Finalize();
return 0;
}
/* generate a global suffix for our temp files and have each node check it's list */
char suffix[8];
uint64_t retry;
/* seed our random number generator */
srand(time(NULL));
/* keep trying to make a valid random suffix...*/
do {
uint64_t attempt = 0;
/* make rank 0 responsible for generating a random suffix */
if (rank == 0) {
generate_suffix(suffix, sizeof(suffix));
}
/* broadcast the random suffix to all ranks */
MPI_Bcast(suffix, sizeof(suffix), MPI_CHAR, 0, MPI_COMM_WORLD);
/* check that the file doesn't already exist */
uint64_t size = mfu_flist_size(filtered);
for (idx = 0; idx < size; idx++) {
char temp_path[PATH_MAX];
strcpy(temp_path, mfu_flist_file_get_name(filtered, idx));
strcat(temp_path, suffix);
if(!mfu_access(temp_path, F_OK)) {
/* the file already exists */
attempt = 1;
break;
}
}
/* do a reduce to figure out if a rank has a file collision */
MPI_Allreduce(&attempt, &retry, 1, MPI_UINT64_T, MPI_MAX, MPI_COMM_WORLD);
} while(retry != 0);
/* initialize progress messages while creating files */
create_prog_count = 0;
create_prog = mfu_progress_start(mfu_progress_timeout, 1, MPI_COMM_WORLD, create_progress_fn);
/* create new files so we can restripe */
uint64_t size = mfu_flist_size(filtered);
for (idx = 0; idx < size; idx++) {
char temp_path[PATH_MAX];
strcpy(temp_path, mfu_flist_file_get_name(filtered, idx));
strcat(temp_path, suffix);
/* create a striped file at the temp file path */
mfu_stripe_set(temp_path, stripe_size, stripes);
/* update our status for file create progress */
create_prog_count++;
mfu_progress_update(&create_prog_count, create_prog);
}
/* finalize file create progress messages */
mfu_progress_complete(&create_prog_count, &create_prog);
MPI_Barrier(MPI_COMM_WORLD);
/* initialize progress messages while copying data */
stripe_prog_bytes = 0;
stripe_prog = mfu_progress_start(mfu_progress_timeout, 1, MPI_COMM_WORLD, stripe_progress_fn);
/* found a suffix, now we need to break our files into chunks based on stripe size */
mfu_file_chunk* file_chunks = mfu_file_chunk_list_alloc(filtered, stripe_size);
mfu_file_chunk* p = file_chunks;
while (p != NULL) {
/* build path to temp file */
char temp_path[PATH_MAX];
strcpy(temp_path, p->name);
strcat(temp_path, suffix);
/* write each chunk in our list */
write_file_chunk(p, temp_path);
/* move on to next file chunk */
p = p->next;
}
mfu_file_chunk_list_free(&file_chunks);
/* finalize progress messages */
mfu_progress_complete(&stripe_prog_bytes, &stripe_prog);
MPI_Barrier(MPI_COMM_WORLD);
/* remove input file and rename temp file */
for (idx = 0; idx < size; idx++) {
/* build path to temp file */
const char *in_path = mfu_flist_file_get_name(filtered, idx);
char out_path[PATH_MAX];
strcpy(out_path, in_path);
strcat(out_path, suffix);
/* change the mode of the newly restriped file to be the same as the old one */
mode_t mode = (mode_t) mfu_flist_file_get_mode(filtered, idx);
if (mfu_chmod(out_path, mode) != 0) {
MFU_LOG(MFU_LOG_ERR, "Failed to chmod file %s (%s)", out_path, strerror(errno));
MPI_Abort(MPI_COMM_WORLD, 1);
}
/* rename the new, restriped file to the old name */
if (rename(out_path, in_path) != 0) {
MFU_LOG(MFU_LOG_ERR, "Failed to rename file %s to %s", out_path, in_path);
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
/* wait for everyone to finish */
MPI_Barrier(MPI_COMM_WORLD);
/* free the walk options */
mfu_walk_opts_delete(&walk_opts);
/* free filtered list, path parameters */
mfu_flist_free(&filtered);
mfu_param_path_free_all(numpaths, paths);
mfu_free(&paths);
mfu_finalize();
MPI_Finalize();
return 0;
}
void mfu_param_path_check_archive(int numparams, mfu_param_path* srcparams, mfu_param_path destparam, int* valid)
{
/* TODO: need to parallize this, rather than have every rank do the test */
/* assume paths are valid */
*valid = 1;
/* count number of source paths that we can read */
int i;
int num_readable = 0;
for (i = 0; i < numparams; i++) {
char* path = srcparams[i].path;
if (mfu_access(path, R_OK) == 0) {
/* found one that we can read */
num_readable++;
} else {
/* not readable */
char* orig = srcparams[i].orig;
MFU_LOG(MFU_LOG_ERR, "Could not read '%s' errno=%d %s",
orig, errno, strerror(errno));
}
}
/* verify we have at least one valid source */
if (num_readable < 1) {
MFU_LOG(MFU_LOG_ERR, "At least one valid source must be specified");
*valid = 0;
goto bcast;
}
/* copy destination to user opts structure */
DTAR_user_opts.dest_path = MFU_STRDUP(dest_param.path);
/* check destination */
if (destparam.path_stat_valid) {
if (DTAR_rank == 0) {
MFU_LOG(MFU_LOG_WARN, "Destination target exists, we will overwrite");
}
} else {
/* compute path to parent of destination archive */
mfu_path* parent = mfu_path_from_str(destparam.path);
mfu_path_dirname(parent);
char* parent_str = mfu_path_strdup(parent);
mfu_path_delete(&parent);
/* check if parent is writable */
if (mfu_access(parent_str, W_OK) < 0) {
MFU_LOG(MFU_LOG_ERR, "Destination parent directory is not wriable: '%s' ",
parent_str);
*valid = 0;
mfu_free(&parent_str);
goto bcast;
}
mfu_free(&parent_str);
}
/* at this point, we know
* (1) destination doesn't exist
* (2) parent directory is writable
*/
bcast:
MPI_Bcast(valid, 1, MPI_INT, 0, MPI_COMM_WORLD);
if (! *valid) {
if (DTAR_rank == 0) {
MFU_LOG(MFU_LOG_ERR, "Exiting run");
}
MPI_Barrier(MPI_COMM_WORLD);
DTAR_exit(EXIT_FAILURE);
}
}
/* check that source and destination paths are valid */
void mfu_param_path_check_copy(uint64_t num, const mfu_param_path* paths,
const mfu_param_path* destpath, int* flag_valid, int* flag_copy_into_dir)
{
/* initialize output params */
*flag_valid = 0;
*flag_copy_into_dir = 0;
/* need at least two paths to have a shot at being valid */
if (num < 1 || paths == NULL || destpath == NULL) {
return;
}
/* get current rank */
int rank;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
/* assume path parameters are valid */
int valid = 1;
/* just have rank 0 check */
if(rank == 0) {
/* count number of readable source paths */
uint64_t i;
int num_readable = 0;
for(i = 0; i < num; i++) {
const char* path = paths[i].path;
if(mfu_access(path, R_OK) == 0) {
num_readable++;
}
else {
/* found a source path that we can't read, not fatal,
* but print an error to notify user */
const char* orig = paths[i].orig;
MFU_LOG(MFU_LOG_ERR, "Could not read `%s' (errno=%d %s)",
orig, errno, strerror(errno));
}
}
/* verify that we have at least one source path */
if(num_readable < 1) {
MFU_LOG(MFU_LOG_ERR, "At least one valid source must be specified");
valid = 0;
goto bcast;
}
/*
* First we need to determine if the last argument is a file or a directory.
* We first attempt to see if the last argument already exists on disk. If it
* doesn't, we then look at the sources to see if we can determine what the
* last argument should be.
*/
bool dest_exists = false;
bool dest_is_dir = false;
bool dest_is_file = false;
bool dest_is_link_to_dir = false;
bool dest_is_link_to_file = false;
bool dest_required_to_be_dir = false;
/* check whether dest exists, its type, and whether it's writable */
if(destpath->path_stat_valid) {
/* we could stat dest path, so something is there */
dest_exists = true;
/* now determine its type */
if(S_ISDIR(destpath->path_stat.st_mode)) {
/* dest is a directory */
dest_is_dir = true;
}
else if(S_ISREG(destpath->path_stat.st_mode)) {
/* dest is a file */
dest_is_file = true;
}
else if(S_ISLNK(destpath->path_stat.st_mode)) {
/* dest is a symlink, but to what? */
if (destpath->target_stat_valid) {
/* target of the symlink exists, determine what it is */
if(S_ISDIR(destpath->target_stat.st_mode)) {
/* dest is link to a directory */
dest_is_link_to_dir = true;
}
else if(S_ISREG(destpath->target_stat.st_mode)) {
/* dest is link to a file */
dest_is_link_to_file = true;
}
else {
/* unsupported type */
MFU_LOG(MFU_LOG_ERR, "Unsupported filetype `%s' --> `%s'",
destpath->orig, destpath->target);
valid = 0;
goto bcast;
}
}
else {
/* dest is a link, but its target does not exist,
* consider this an error */
MFU_LOG(MFU_LOG_ERR, "Destination is broken symlink `%s'",
destpath->orig);
valid = 0;
goto bcast;
}
}
else {
/* unsupported type */
MFU_LOG(MFU_LOG_ERR, "Unsupported filetype `%s'",
destpath->orig);
valid = 0;
goto bcast;
}
/* check that dest is writable */
if(mfu_access(destpath->path, W_OK) < 0) {
MFU_LOG(MFU_LOG_ERR, "Destination is not writable `%s'",
destpath->path);
valid = 0;
goto bcast;
}
}
else {
/* destination does not exist, so we'll be creating it,
* check that its parent is writable */
/* compute parent path */
mfu_path* parent = mfu_path_from_str(destpath->path);
mfu_path_dirname(parent);
char* parent_str = mfu_path_strdup(parent);
mfu_path_delete(&parent);
/* check that parent is writable */
if(mfu_access(parent_str, W_OK) < 0) {
MFU_LOG(MFU_LOG_ERR, "Destination parent directory is not writable `%s'",
parent_str);
valid = 0;
mfu_free(&parent_str);
goto bcast;
}
mfu_free(&parent_str);
}
/* determine whether caller *requires* copy into dir */
/* TODO: if caller specifies dest/ or dest/. */
/* if caller specifies more than one source,
* then dest has to be a directory */
if(num > 1) {
dest_required_to_be_dir = true;
}
/* if caller requires dest to be a directory, and if dest does not
* exist or it does it exist but it's not a directory, then abort */
if(dest_required_to_be_dir &&
(!dest_exists || (!dest_is_dir && !dest_is_link_to_dir)))
{
MFU_LOG(MFU_LOG_ERR, "Destination is not a directory `%s'",
destpath->orig);
valid = 0;
goto bcast;
}
/* we copy into a directory if any of the following:
* 1) user specified more than one source
* 2) destination already exists and is a directory
* 3) destination already exists and is a link to a directory */
bool copy_into_dir = (dest_required_to_be_dir || dest_is_dir || dest_is_link_to_dir);
*flag_copy_into_dir = copy_into_dir ? 1 : 0;
}
bcast:
/* get status from rank 0 */
MPI_Bcast(&valid, 1, MPI_INT, 0, MPI_COMM_WORLD);
/* set valid flag */
*flag_valid = valid;
/* rank 0 broadcasts whether we're copying into a directory */
MPI_Bcast(flag_copy_into_dir, 1, MPI_INT, 0, MPI_COMM_WORLD);
return;
}
/* check that source and destination paths are valid */
static void DCOPY_check_paths(void)
{
/* assume path parameters are valid */
int valid = 1;
/* just have rank 0 check */
if(DCOPY_global_rank == 0) {
/* count number of readable source paths */
int i;
int num_readable = 0;
for(i = 0; i < num_src_params; i++) {
char* path = src_params[i].path;
if(mfu_access(path, R_OK) == 0) {
num_readable++;
}
else {
/* found a source path that we can't read, not fatal,
* but print an error to notify user */
char* orig = src_params[i].orig;
MFU_LOG(MFU_LOG_ERR, "Could not read `%s' errno=%d %s",
orig, errno, strerror(errno));
}
}
/* verify that we have at least one source path */
if(num_readable < 1) {
MFU_LOG(MFU_LOG_ERR, "At least one valid source must be specified");
valid = 0;
goto bcast;
}
/*
* First we need to determine if the last argument is a file or a directory.
* We first attempt to see if the last argument already exists on disk. If it
* doesn't, we then look at the sources to see if we can determine what the
* last argument should be.
*/
bool dest_exists = false;
bool dest_is_dir = false;
bool dest_is_file = false;
bool dest_is_link_to_dir = false;
bool dest_is_link_to_file = false;
bool dest_required_to_be_dir = false;
/* check whether dest exists, its type, and whether it's writable */
if(dest_param.path_stat_valid) {
/* we could stat dest path, so something is there */
dest_exists = true;
/* now determine its type */
if(S_ISDIR(dest_param.path_stat.st_mode)) {
/* dest is a directory */
dest_is_dir = true;
}
else if(S_ISREG(dest_param.path_stat.st_mode)) {
/* dest is a file */
dest_is_file = true;
}
else if(S_ISLNK(dest_param.path_stat.st_mode)) {
/* dest is a symlink, but to what? */
if (dest_param.target_stat_valid) {
/* target of the symlink exists, determine what it is */
if(S_ISDIR(dest_param.target_stat.st_mode)) {
/* dest is link to a directory */
dest_is_link_to_dir = true;
}
else if(S_ISREG(dest_param.target_stat.st_mode)) {
/* dest is link to a file */
dest_is_link_to_file = true;
}
else {
/* unsupported type */
MFU_LOG(MFU_LOG_ERR, "Unsupported filetype `%s' --> `%s'",
dest_param.orig, dest_param.target);
valid = 0;
goto bcast;
}
}
else {
/* dest is a link, but its target does not exist,
* consider this an error */
MFU_LOG(MFU_LOG_ERR, "Destination is broken symlink `%s'",
dest_param.orig);
valid = 0;
goto bcast;
}
}
else {
/* unsupported type */
MFU_LOG(MFU_LOG_ERR, "Unsupported filetype `%s'",
dest_param.orig);
valid = 0;
goto bcast;
}
/* check that dest is writable */
if(mfu_access(dest_param.path, W_OK) < 0) {
MFU_LOG(MFU_LOG_ERR, "Destination is not writable `%s'",
dest_param.path);
valid = 0;
goto bcast;
}
}
else {
/* destination does not exist, so we'll be creating it,
* check that its parent is writable */
/* compute parent path */
mfu_path* parent = mfu_path_from_str(dest_param.path);
mfu_path_dirname(parent);
char* parent_str = mfu_path_strdup(parent);
mfu_path_delete(&parent);
/* check that parent is writable */
if(mfu_access(parent_str, W_OK) < 0) {
MFU_LOG(MFU_LOG_ERR, "Destination parent directory is not writable `%s'",
parent_str);
valid = 0;
mfu_free(&parent_str);
goto bcast;
}
mfu_free(&parent_str);
}
/* determine whether caller *requires* copy into dir */
/* TODO: if caller specifies dest/ or dest/. */
/* if caller specifies more than one source,
* then dest has to be a directory */
if(num_src_params > 1) {
dest_required_to_be_dir = true;
}
/* if caller requires dest to be a directory, and if dest does not
* exist or it does it exist but it's not a directory, then abort */
if(dest_required_to_be_dir &&
(!dest_exists || (!dest_is_dir && !dest_is_link_to_dir)))
{
MFU_LOG(MFU_LOG_ERR, "Destination is not a directory `%s'",
dest_param.orig);
valid = 0;
goto bcast;
}
/* we copy into a directory if any of the following:
* 1) user specified more than one source
* 2) destination already exists and is a directory
* 3) destination already exists and is a link to a directory */
bool copy_into_dir = (dest_required_to_be_dir || dest_is_dir || dest_is_link_to_dir);
DCOPY_user_opts.copy_into_dir = copy_into_dir ? 1 : 0;
}
/* get status from rank 0 */
bcast:
MPI_Bcast(&valid, 1, MPI_INT, 0, MPI_COMM_WORLD);
/* exit job if we found a problem */
if(! valid) {
if(DCOPY_global_rank == 0) {
MFU_LOG(MFU_LOG_ERR, "Exiting run");
}
MPI_Barrier(MPI_COMM_WORLD);
DCOPY_exit(EXIT_FAILURE);
}
/* rank 0 broadcasts whether we're copying into a directory */
MPI_Bcast(&DCOPY_user_opts.copy_into_dir, 1, MPI_INT, 0, MPI_COMM_WORLD);
}