/* check that source and destination paths are valid */
void mfu_param_path_check_copy(uint64_t num, const mfu_param_path* paths,
const mfu_param_path* destpath, int* flag_valid, int* flag_copy_into_dir)
{
/* initialize output params */
*flag_valid = 0;
*flag_copy_into_dir = 0;
/* need at least two paths to have a shot at being valid */
if (num < 1 || paths == NULL || destpath == NULL) {
return;
}
/* get current rank */
int rank;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
/* assume path parameters are valid */
int valid = 1;
/* just have rank 0 check */
if(rank == 0) {
/* count number of readable source paths */
uint64_t i;
int num_readable = 0;
for(i = 0; i < num; i++) {
const char* path = paths[i].path;
if(mfu_access(path, R_OK) == 0) {
num_readable++;
}
else {
/* found a source path that we can't read, not fatal,
* but print an error to notify user */
const char* orig = paths[i].orig;
MFU_LOG(MFU_LOG_ERR, "Could not read `%s' (errno=%d %s)",
orig, errno, strerror(errno));
}
}
/* verify that we have at least one source path */
if(num_readable < 1) {
MFU_LOG(MFU_LOG_ERR, "At least one valid source must be specified");
valid = 0;
goto bcast;
}
/*
* First we need to determine if the last argument is a file or a directory.
* We first attempt to see if the last argument already exists on disk. If it
* doesn't, we then look at the sources to see if we can determine what the
* last argument should be.
*/
bool dest_exists = false;
bool dest_is_dir = false;
bool dest_is_file = false;
bool dest_is_link_to_dir = false;
bool dest_is_link_to_file = false;
bool dest_required_to_be_dir = false;
/* check whether dest exists, its type, and whether it's writable */
if(destpath->path_stat_valid) {
/* we could stat dest path, so something is there */
dest_exists = true;
/* now determine its type */
if(S_ISDIR(destpath->path_stat.st_mode)) {
/* dest is a directory */
dest_is_dir = true;
}
else if(S_ISREG(destpath->path_stat.st_mode)) {
/* dest is a file */
dest_is_file = true;
}
else if(S_ISLNK(destpath->path_stat.st_mode)) {
/* dest is a symlink, but to what? */
if (destpath->target_stat_valid) {
/* target of the symlink exists, determine what it is */
if(S_ISDIR(destpath->target_stat.st_mode)) {
/* dest is link to a directory */
dest_is_link_to_dir = true;
}
else if(S_ISREG(destpath->target_stat.st_mode)) {
/* dest is link to a file */
dest_is_link_to_file = true;
}
else {
/* unsupported type */
MFU_LOG(MFU_LOG_ERR, "Unsupported filetype `%s' --> `%s'",
destpath->orig, destpath->target);
valid = 0;
goto bcast;
}
}
else {
/* dest is a link, but its target does not exist,
* consider this an error */
MFU_LOG(MFU_LOG_ERR, "Destination is broken symlink `%s'",
destpath->orig);
valid = 0;
goto bcast;
}
}
else {
/* unsupported type */
MFU_LOG(MFU_LOG_ERR, "Unsupported filetype `%s'",
destpath->orig);
valid = 0;
goto bcast;
}
/* check that dest is writable */
if(mfu_access(destpath->path, W_OK) < 0) {
MFU_LOG(MFU_LOG_ERR, "Destination is not writable `%s'",
destpath->path);
valid = 0;
goto bcast;
}
}
else {
/* destination does not exist, so we'll be creating it,
* check that its parent is writable */
/* compute parent path */
mfu_path* parent = mfu_path_from_str(destpath->path);
mfu_path_dirname(parent);
char* parent_str = mfu_path_strdup(parent);
mfu_path_delete(&parent);
/* check that parent is writable */
if(mfu_access(parent_str, W_OK) < 0) {
MFU_LOG(MFU_LOG_ERR, "Destination parent directory is not writable `%s'",
parent_str);
valid = 0;
mfu_free(&parent_str);
goto bcast;
}
mfu_free(&parent_str);
}
/* determine whether caller *requires* copy into dir */
/* TODO: if caller specifies dest/ or dest/. */
/* if caller specifies more than one source,
* then dest has to be a directory */
if(num > 1) {
dest_required_to_be_dir = true;
}
/* if caller requires dest to be a directory, and if dest does not
* exist or it does it exist but it's not a directory, then abort */
if(dest_required_to_be_dir &&
(!dest_exists || (!dest_is_dir && !dest_is_link_to_dir)))
{
MFU_LOG(MFU_LOG_ERR, "Destination is not a directory `%s'",
destpath->orig);
valid = 0;
goto bcast;
}
/* we copy into a directory if any of the following:
* 1) user specified more than one source
* 2) destination already exists and is a directory
* 3) destination already exists and is a link to a directory */
bool copy_into_dir = (dest_required_to_be_dir || dest_is_dir || dest_is_link_to_dir);
*flag_copy_into_dir = copy_into_dir ? 1 : 0;
}
bcast:
/* get status from rank 0 */
MPI_Bcast(&valid, 1, MPI_INT, 0, MPI_COMM_WORLD);
/* set valid flag */
*flag_valid = valid;
/* rank 0 broadcasts whether we're copying into a directory */
MPI_Bcast(flag_copy_into_dir, 1, MPI_INT, 0, MPI_COMM_WORLD);
return;
}
/* given an item name, determine which source path this item
* is contained within, extract directory components from source
* path to this item and then prepend destination prefix. */
char* mfu_param_path_copy_dest(const char* name, int numpaths,
const mfu_param_path* paths, const mfu_param_path* destpath,
mfu_copy_opts_t* mfu_copy_opts)
{
/* identify which source directory this came from */
int i;
int idx = -1;
for (i = 0; i < numpaths; i++) {
/* get path for step */
const char* path = paths[i].path;
/* get length of source path */
size_t len = strlen(path);
/* see if name is a child of path */
if (strncmp(path, name, len) == 0) {
idx = i;
break;
}
}
/* this will happen if the named item is not a child of any
* source paths */
if (idx == -1) {
return NULL;
}
/* create path of item */
mfu_path* item = mfu_path_from_str(name);
/* get source directory */
mfu_path* src = mfu_path_from_str(paths[i].path);
/* get number of components in item */
int item_components = mfu_path_components(item);
/* get number of components in source path */
int src_components = mfu_path_components(src);
/* if copying into directory, keep last component,
* otherwise cut all components listed in source path */
int cut = src_components;
if (mfu_copy_opts->copy_into_dir && cut > 0) {
if ((mfu_copy_opts->do_sync != 1) &&
(paths[i].orig[strlen(paths[i].orig) - 1] != '/')) {
cut--;
}
}
/* compute number of components to keep */
int keep = item_components - cut;
/* chop prefix from item */
mfu_path_slice(item, cut, keep);
/* prepend destination path */
mfu_path_prepend_str(item, destpath->path);
/* convert to a NUL-terminated string */
char* dest = mfu_path_strdup(item);
/* free our temporary paths */
mfu_path_delete(&src);
mfu_path_delete(&item);
return dest;
}
void mfu_param_path_check_archive(int numparams, mfu_param_path* srcparams, mfu_param_path destparam, int* valid)
{
/* TODO: need to parallize this, rather than have every rank do the test */
/* assume paths are valid */
*valid = 1;
/* count number of source paths that we can read */
int i;
int num_readable = 0;
for (i = 0; i < numparams; i++) {
char* path = srcparams[i].path;
if (mfu_access(path, R_OK) == 0) {
/* found one that we can read */
num_readable++;
} else {
/* not readable */
char* orig = srcparams[i].orig;
MFU_LOG(MFU_LOG_ERR, "Could not read '%s' errno=%d %s",
orig, errno, strerror(errno));
}
}
/* verify we have at least one valid source */
if (num_readable < 1) {
MFU_LOG(MFU_LOG_ERR, "At least one valid source must be specified");
*valid = 0;
goto bcast;
}
/* copy destination to user opts structure */
DTAR_user_opts.dest_path = MFU_STRDUP(dest_param.path);
/* check destination */
if (destparam.path_stat_valid) {
if (DTAR_rank == 0) {
MFU_LOG(MFU_LOG_WARN, "Destination target exists, we will overwrite");
}
} else {
/* compute path to parent of destination archive */
mfu_path* parent = mfu_path_from_str(destparam.path);
mfu_path_dirname(parent);
char* parent_str = mfu_path_strdup(parent);
mfu_path_delete(&parent);
/* check if parent is writable */
if (mfu_access(parent_str, W_OK) < 0) {
MFU_LOG(MFU_LOG_ERR, "Destination parent directory is not wriable: '%s' ",
parent_str);
*valid = 0;
mfu_free(&parent_str);
goto bcast;
}
mfu_free(&parent_str);
}
/* at this point, we know
* (1) destination doesn't exist
* (2) parent directory is writable
*/
bcast:
MPI_Bcast(valid, 1, MPI_INT, 0, MPI_COMM_WORLD);
if (! *valid) {
if (DTAR_rank == 0) {
MFU_LOG(MFU_LOG_ERR, "Exiting run");
}
MPI_Barrier(MPI_COMM_WORLD);
DTAR_exit(EXIT_FAILURE);
}
}
/* given an input flist, return a newly allocated flist consisting of
* a filtered set by finding all items that match/don't match a given
* regular expression */
mfu_flist mfu_flist_filter_regex(mfu_flist flist, const char* regex_exp, int exclude, int name)
{
/* create our list to return */
mfu_flist dest = mfu_flist_subset(flist);
/* check if user passed in an expression, if so then filter the list */
if (regex_exp != NULL) {
/* compile regular expression, if it fails print error */
regex_t regex;
int regex_return = regcomp(®ex, regex_exp, 0);
if (regex_return) {
MFU_ABORT(-1, "Could not compile regex: `%s' rc=%d\n", regex_exp, regex_return);
}
/* copy the things that don't or do (based on input) match the regex into a
* filtered list */
uint64_t idx = 0;
uint64_t size = mfu_flist_size(flist);
while (idx < size) {
/* get full path of item */
const char* file_name = mfu_flist_file_get_name(flist, idx);
/* get basename of item (exclude the path) */
mfu_path* pathname = mfu_path_from_str(file_name);
mfu_path_basename(pathname);
char* base = mfu_path_strdup(pathname);
/* execute regex on item, either against the basename or
* the full path depending on name flag */
if (name) {
/* run regex on basename */
regex_return = regexec(®ex, base, 0, NULL, 0);
}
else {
/* run regex on full path */
regex_return = regexec(®ex, file_name, 0, NULL, 0);
}
/* copy item to the filtered list */
if (exclude) {
/* user wants to exclude items that match, so copy everything that
* does not match */
if (regex_return == REG_NOMATCH) {
mfu_flist_file_copy(flist, idx, dest);
}
}
else {
/* user wants to copy over any matching items */
if (regex_return == 0) {
mfu_flist_file_copy(flist, idx, dest);
}
}
/* free the basename */
mfu_free(&base);
mfu_path_delete(&pathname);
/* get next item in our list */
idx++;
}
/* summarize the filtered list */
mfu_flist_summarize(dest);
}
/* return the filtered list */
return dest;
}
/* check that source and destination paths are valid */
static void DCOPY_check_paths(void)
{
/* assume path parameters are valid */
int valid = 1;
/* just have rank 0 check */
if(DCOPY_global_rank == 0) {
/* count number of readable source paths */
int i;
int num_readable = 0;
for(i = 0; i < num_src_params; i++) {
char* path = src_params[i].path;
if(mfu_access(path, R_OK) == 0) {
num_readable++;
}
else {
/* found a source path that we can't read, not fatal,
* but print an error to notify user */
char* orig = src_params[i].orig;
MFU_LOG(MFU_LOG_ERR, "Could not read `%s' errno=%d %s",
orig, errno, strerror(errno));
}
}
/* verify that we have at least one source path */
if(num_readable < 1) {
MFU_LOG(MFU_LOG_ERR, "At least one valid source must be specified");
valid = 0;
goto bcast;
}
/*
* First we need to determine if the last argument is a file or a directory.
* We first attempt to see if the last argument already exists on disk. If it
* doesn't, we then look at the sources to see if we can determine what the
* last argument should be.
*/
bool dest_exists = false;
bool dest_is_dir = false;
bool dest_is_file = false;
bool dest_is_link_to_dir = false;
bool dest_is_link_to_file = false;
bool dest_required_to_be_dir = false;
/* check whether dest exists, its type, and whether it's writable */
if(dest_param.path_stat_valid) {
/* we could stat dest path, so something is there */
dest_exists = true;
/* now determine its type */
if(S_ISDIR(dest_param.path_stat.st_mode)) {
/* dest is a directory */
dest_is_dir = true;
}
else if(S_ISREG(dest_param.path_stat.st_mode)) {
/* dest is a file */
dest_is_file = true;
}
else if(S_ISLNK(dest_param.path_stat.st_mode)) {
/* dest is a symlink, but to what? */
if (dest_param.target_stat_valid) {
/* target of the symlink exists, determine what it is */
if(S_ISDIR(dest_param.target_stat.st_mode)) {
/* dest is link to a directory */
dest_is_link_to_dir = true;
}
else if(S_ISREG(dest_param.target_stat.st_mode)) {
/* dest is link to a file */
dest_is_link_to_file = true;
}
else {
/* unsupported type */
MFU_LOG(MFU_LOG_ERR, "Unsupported filetype `%s' --> `%s'",
dest_param.orig, dest_param.target);
valid = 0;
goto bcast;
}
}
else {
/* dest is a link, but its target does not exist,
* consider this an error */
MFU_LOG(MFU_LOG_ERR, "Destination is broken symlink `%s'",
dest_param.orig);
valid = 0;
goto bcast;
}
}
else {
/* unsupported type */
MFU_LOG(MFU_LOG_ERR, "Unsupported filetype `%s'",
dest_param.orig);
valid = 0;
goto bcast;
}
/* check that dest is writable */
if(mfu_access(dest_param.path, W_OK) < 0) {
MFU_LOG(MFU_LOG_ERR, "Destination is not writable `%s'",
dest_param.path);
valid = 0;
goto bcast;
}
}
else {
/* destination does not exist, so we'll be creating it,
* check that its parent is writable */
/* compute parent path */
mfu_path* parent = mfu_path_from_str(dest_param.path);
mfu_path_dirname(parent);
char* parent_str = mfu_path_strdup(parent);
mfu_path_delete(&parent);
/* check that parent is writable */
if(mfu_access(parent_str, W_OK) < 0) {
MFU_LOG(MFU_LOG_ERR, "Destination parent directory is not writable `%s'",
parent_str);
valid = 0;
mfu_free(&parent_str);
goto bcast;
}
mfu_free(&parent_str);
}
/* determine whether caller *requires* copy into dir */
/* TODO: if caller specifies dest/ or dest/. */
/* if caller specifies more than one source,
* then dest has to be a directory */
if(num_src_params > 1) {
dest_required_to_be_dir = true;
}
/* if caller requires dest to be a directory, and if dest does not
* exist or it does it exist but it's not a directory, then abort */
if(dest_required_to_be_dir &&
(!dest_exists || (!dest_is_dir && !dest_is_link_to_dir)))
{
MFU_LOG(MFU_LOG_ERR, "Destination is not a directory `%s'",
dest_param.orig);
valid = 0;
goto bcast;
}
/* we copy into a directory if any of the following:
* 1) user specified more than one source
* 2) destination already exists and is a directory
* 3) destination already exists and is a link to a directory */
bool copy_into_dir = (dest_required_to_be_dir || dest_is_dir || dest_is_link_to_dir);
DCOPY_user_opts.copy_into_dir = copy_into_dir ? 1 : 0;
}
/* get status from rank 0 */
bcast:
MPI_Bcast(&valid, 1, MPI_INT, 0, MPI_COMM_WORLD);
/* exit job if we found a problem */
if(! valid) {
if(DCOPY_global_rank == 0) {
MFU_LOG(MFU_LOG_ERR, "Exiting run");
}
MPI_Barrier(MPI_COMM_WORLD);
DCOPY_exit(EXIT_FAILURE);
}
/* rank 0 broadcasts whether we're copying into a directory */
MPI_Bcast(&DCOPY_user_opts.copy_into_dir, 1, MPI_INT, 0, MPI_COMM_WORLD);
}
