int DCOPY_input_flist_skip(const char* name, void* args) { /* create mfu_path from name */ mfu_path* path = mfu_path_from_str(name); /* iterate over each source path */ int i; for (i = 0; i < num_src_params; i++) { /* create mfu_path of source path */ char* src_name = src_params[i].path; const char* src_path = mfu_path_from_str(src_name); /* check whether path is contained within or equal to * source path and if so, we need to copy this file */ mfu_path_result result = mfu_path_cmp(path, src_path); if (result == MFU_PATH_SRC_CHILD || result == MFU_PATH_EQUAL) { MFU_LOG(MFU_LOG_INFO, "Need to copy %s because of %s.", name, src_name); mfu_path_delete(&src_path); mfu_path_delete(&path); return 0; } mfu_path_delete(&src_path); } /* the path in name is not a child of any source paths, * so skip this file */ MFU_LOG(MFU_LOG_INFO, "Skip %s.", name); mfu_path_delete(&path); return 1; }
/* check that source and destination paths are valid */ void mfu_param_path_check_copy(uint64_t num, const mfu_param_path* paths, const mfu_param_path* destpath, int* flag_valid, int* flag_copy_into_dir) { /* initialize output params */ *flag_valid = 0; *flag_copy_into_dir = 0; /* need at least two paths to have a shot at being valid */ if (num < 1 || paths == NULL || destpath == NULL) { return; } /* get current rank */ int rank; MPI_Comm_rank(MPI_COMM_WORLD, &rank); /* assume path parameters are valid */ int valid = 1; /* just have rank 0 check */ if(rank == 0) { /* count number of readable source paths */ uint64_t i; int num_readable = 0; for(i = 0; i < num; i++) { const char* path = paths[i].path; if(mfu_access(path, R_OK) == 0) { num_readable++; } else { /* found a source path that we can't read, not fatal, * but print an error to notify user */ const char* orig = paths[i].orig; MFU_LOG(MFU_LOG_ERR, "Could not read `%s' (errno=%d %s)", orig, errno, strerror(errno)); } } /* verify that we have at least one source path */ if(num_readable < 1) { MFU_LOG(MFU_LOG_ERR, "At least one valid source must be specified"); valid = 0; goto bcast; } /* * First we need to determine if the last argument is a file or a directory. * We first attempt to see if the last argument already exists on disk. If it * doesn't, we then look at the sources to see if we can determine what the * last argument should be. */ bool dest_exists = false; bool dest_is_dir = false; bool dest_is_file = false; bool dest_is_link_to_dir = false; bool dest_is_link_to_file = false; bool dest_required_to_be_dir = false; /* check whether dest exists, its type, and whether it's writable */ if(destpath->path_stat_valid) { /* we could stat dest path, so something is there */ dest_exists = true; /* now determine its type */ if(S_ISDIR(destpath->path_stat.st_mode)) { /* dest is a directory */ dest_is_dir = true; } else if(S_ISREG(destpath->path_stat.st_mode)) { /* dest is a file */ dest_is_file = true; } else if(S_ISLNK(destpath->path_stat.st_mode)) { /* dest is a symlink, but to what? */ if (destpath->target_stat_valid) { /* target of the symlink exists, determine what it is */ if(S_ISDIR(destpath->target_stat.st_mode)) { /* dest is link to a directory */ dest_is_link_to_dir = true; } else if(S_ISREG(destpath->target_stat.st_mode)) { /* dest is link to a file */ dest_is_link_to_file = true; } else { /* unsupported type */ MFU_LOG(MFU_LOG_ERR, "Unsupported filetype `%s' --> `%s'", destpath->orig, destpath->target); valid = 0; goto bcast; } } else { /* dest is a link, but its target does not exist, * consider this an error */ MFU_LOG(MFU_LOG_ERR, "Destination is broken symlink `%s'", destpath->orig); valid = 0; goto bcast; } } else { /* unsupported type */ MFU_LOG(MFU_LOG_ERR, "Unsupported filetype `%s'", destpath->orig); valid = 0; goto bcast; } /* check that dest is writable */ if(mfu_access(destpath->path, W_OK) < 0) { MFU_LOG(MFU_LOG_ERR, "Destination is not writable `%s'", destpath->path); valid = 0; goto bcast; } } else { /* destination does not exist, so we'll be creating it, * check that its parent is writable */ /* compute parent path */ mfu_path* parent = mfu_path_from_str(destpath->path); mfu_path_dirname(parent); char* parent_str = mfu_path_strdup(parent); mfu_path_delete(&parent); /* check that parent is writable */ if(mfu_access(parent_str, W_OK) < 0) { MFU_LOG(MFU_LOG_ERR, "Destination parent directory is not writable `%s'", parent_str); valid = 0; mfu_free(&parent_str); goto bcast; } mfu_free(&parent_str); } /* determine whether caller *requires* copy into dir */ /* TODO: if caller specifies dest/ or dest/. */ /* if caller specifies more than one source, * then dest has to be a directory */ if(num > 1) { dest_required_to_be_dir = true; } /* if caller requires dest to be a directory, and if dest does not * exist or it does it exist but it's not a directory, then abort */ if(dest_required_to_be_dir && (!dest_exists || (!dest_is_dir && !dest_is_link_to_dir))) { MFU_LOG(MFU_LOG_ERR, "Destination is not a directory `%s'", destpath->orig); valid = 0; goto bcast; } /* we copy into a directory if any of the following: * 1) user specified more than one source * 2) destination already exists and is a directory * 3) destination already exists and is a link to a directory */ bool copy_into_dir = (dest_required_to_be_dir || dest_is_dir || dest_is_link_to_dir); *flag_copy_into_dir = copy_into_dir ? 1 : 0; } bcast: /* get status from rank 0 */ MPI_Bcast(&valid, 1, MPI_INT, 0, MPI_COMM_WORLD); /* set valid flag */ *flag_valid = valid; /* rank 0 broadcasts whether we're copying into a directory */ MPI_Bcast(flag_copy_into_dir, 1, MPI_INT, 0, MPI_COMM_WORLD); return; }
/* given an item name, determine which source path this item * is contained within, extract directory components from source * path to this item and then prepend destination prefix. */ char* mfu_param_path_copy_dest(const char* name, int numpaths, const mfu_param_path* paths, const mfu_param_path* destpath, mfu_copy_opts_t* mfu_copy_opts) { /* identify which source directory this came from */ int i; int idx = -1; for (i = 0; i < numpaths; i++) { /* get path for step */ const char* path = paths[i].path; /* get length of source path */ size_t len = strlen(path); /* see if name is a child of path */ if (strncmp(path, name, len) == 0) { idx = i; break; } } /* this will happen if the named item is not a child of any * source paths */ if (idx == -1) { return NULL; } /* create path of item */ mfu_path* item = mfu_path_from_str(name); /* get source directory */ mfu_path* src = mfu_path_from_str(paths[i].path); /* get number of components in item */ int item_components = mfu_path_components(item); /* get number of components in source path */ int src_components = mfu_path_components(src); /* if copying into directory, keep last component, * otherwise cut all components listed in source path */ int cut = src_components; if (mfu_copy_opts->copy_into_dir && cut > 0) { if ((mfu_copy_opts->do_sync != 1) && (paths[i].orig[strlen(paths[i].orig) - 1] != '/')) { cut--; } } /* compute number of components to keep */ int keep = item_components - cut; /* chop prefix from item */ mfu_path_slice(item, cut, keep); /* prepend destination path */ mfu_path_prepend_str(item, destpath->path); /* convert to a NUL-terminated string */ char* dest = mfu_path_strdup(item); /* free our temporary paths */ mfu_path_delete(&src); mfu_path_delete(&item); return dest; }
/* given an input flist, return a newly allocated flist consisting of * a filtered set by finding all items that match/don't match a given * regular expression */ mfu_flist mfu_flist_filter_regex(mfu_flist flist, const char* regex_exp, int exclude, int name) { /* create our list to return */ mfu_flist dest = mfu_flist_subset(flist); /* check if user passed in an expression, if so then filter the list */ if (regex_exp != NULL) { /* compile regular expression, if it fails print error */ regex_t regex; int regex_return = regcomp(®ex, regex_exp, 0); if (regex_return) { MFU_ABORT(-1, "Could not compile regex: `%s' rc=%d\n", regex_exp, regex_return); } /* copy the things that don't or do (based on input) match the regex into a * filtered list */ uint64_t idx = 0; uint64_t size = mfu_flist_size(flist); while (idx < size) { /* get full path of item */ const char* file_name = mfu_flist_file_get_name(flist, idx); /* get basename of item (exclude the path) */ mfu_path* pathname = mfu_path_from_str(file_name); mfu_path_basename(pathname); char* base = mfu_path_strdup(pathname); /* execute regex on item, either against the basename or * the full path depending on name flag */ if (name) { /* run regex on basename */ regex_return = regexec(®ex, base, 0, NULL, 0); } else { /* run regex on full path */ regex_return = regexec(®ex, file_name, 0, NULL, 0); } /* copy item to the filtered list */ if (exclude) { /* user wants to exclude items that match, so copy everything that * does not match */ if (regex_return == REG_NOMATCH) { mfu_flist_file_copy(flist, idx, dest); } } else { /* user wants to copy over any matching items */ if (regex_return == 0) { mfu_flist_file_copy(flist, idx, dest); } } /* free the basename */ mfu_free(&base); mfu_path_delete(&pathname); /* get next item in our list */ idx++; } /* summarize the filtered list */ mfu_flist_summarize(dest); } /* return the filtered list */ return dest; }
void mfu_param_path_check_archive(int numparams, mfu_param_path* srcparams, mfu_param_path destparam, int* valid) { /* TODO: need to parallize this, rather than have every rank do the test */ /* assume paths are valid */ *valid = 1; /* count number of source paths that we can read */ int i; int num_readable = 0; for (i = 0; i < numparams; i++) { char* path = srcparams[i].path; if (mfu_access(path, R_OK) == 0) { /* found one that we can read */ num_readable++; } else { /* not readable */ char* orig = srcparams[i].orig; MFU_LOG(MFU_LOG_ERR, "Could not read '%s' errno=%d %s", orig, errno, strerror(errno)); } } /* verify we have at least one valid source */ if (num_readable < 1) { MFU_LOG(MFU_LOG_ERR, "At least one valid source must be specified"); *valid = 0; goto bcast; } /* copy destination to user opts structure */ DTAR_user_opts.dest_path = MFU_STRDUP(dest_param.path); /* check destination */ if (destparam.path_stat_valid) { if (DTAR_rank == 0) { MFU_LOG(MFU_LOG_WARN, "Destination target exists, we will overwrite"); } } else { /* compute path to parent of destination archive */ mfu_path* parent = mfu_path_from_str(destparam.path); mfu_path_dirname(parent); char* parent_str = mfu_path_strdup(parent); mfu_path_delete(&parent); /* check if parent is writable */ if (mfu_access(parent_str, W_OK) < 0) { MFU_LOG(MFU_LOG_ERR, "Destination parent directory is not wriable: '%s' ", parent_str); *valid = 0; mfu_free(&parent_str); goto bcast; } mfu_free(&parent_str); } /* at this point, we know * (1) destination doesn't exist * (2) parent directory is writable */ bcast: MPI_Bcast(valid, 1, MPI_INT, 0, MPI_COMM_WORLD); if (! *valid) { if (DTAR_rank == 0) { MFU_LOG(MFU_LOG_ERR, "Exiting run"); } MPI_Barrier(MPI_COMM_WORLD); DTAR_exit(EXIT_FAILURE); } }
/* check that source and destination paths are valid */ static void DCOPY_check_paths(void) { /* assume path parameters are valid */ int valid = 1; /* just have rank 0 check */ if(DCOPY_global_rank == 0) { /* count number of readable source paths */ int i; int num_readable = 0; for(i = 0; i < num_src_params; i++) { char* path = src_params[i].path; if(mfu_access(path, R_OK) == 0) { num_readable++; } else { /* found a source path that we can't read, not fatal, * but print an error to notify user */ char* orig = src_params[i].orig; MFU_LOG(MFU_LOG_ERR, "Could not read `%s' errno=%d %s", orig, errno, strerror(errno)); } } /* verify that we have at least one source path */ if(num_readable < 1) { MFU_LOG(MFU_LOG_ERR, "At least one valid source must be specified"); valid = 0; goto bcast; } /* * First we need to determine if the last argument is a file or a directory. * We first attempt to see if the last argument already exists on disk. If it * doesn't, we then look at the sources to see if we can determine what the * last argument should be. */ bool dest_exists = false; bool dest_is_dir = false; bool dest_is_file = false; bool dest_is_link_to_dir = false; bool dest_is_link_to_file = false; bool dest_required_to_be_dir = false; /* check whether dest exists, its type, and whether it's writable */ if(dest_param.path_stat_valid) { /* we could stat dest path, so something is there */ dest_exists = true; /* now determine its type */ if(S_ISDIR(dest_param.path_stat.st_mode)) { /* dest is a directory */ dest_is_dir = true; } else if(S_ISREG(dest_param.path_stat.st_mode)) { /* dest is a file */ dest_is_file = true; } else if(S_ISLNK(dest_param.path_stat.st_mode)) { /* dest is a symlink, but to what? */ if (dest_param.target_stat_valid) { /* target of the symlink exists, determine what it is */ if(S_ISDIR(dest_param.target_stat.st_mode)) { /* dest is link to a directory */ dest_is_link_to_dir = true; } else if(S_ISREG(dest_param.target_stat.st_mode)) { /* dest is link to a file */ dest_is_link_to_file = true; } else { /* unsupported type */ MFU_LOG(MFU_LOG_ERR, "Unsupported filetype `%s' --> `%s'", dest_param.orig, dest_param.target); valid = 0; goto bcast; } } else { /* dest is a link, but its target does not exist, * consider this an error */ MFU_LOG(MFU_LOG_ERR, "Destination is broken symlink `%s'", dest_param.orig); valid = 0; goto bcast; } } else { /* unsupported type */ MFU_LOG(MFU_LOG_ERR, "Unsupported filetype `%s'", dest_param.orig); valid = 0; goto bcast; } /* check that dest is writable */ if(mfu_access(dest_param.path, W_OK) < 0) { MFU_LOG(MFU_LOG_ERR, "Destination is not writable `%s'", dest_param.path); valid = 0; goto bcast; } } else { /* destination does not exist, so we'll be creating it, * check that its parent is writable */ /* compute parent path */ mfu_path* parent = mfu_path_from_str(dest_param.path); mfu_path_dirname(parent); char* parent_str = mfu_path_strdup(parent); mfu_path_delete(&parent); /* check that parent is writable */ if(mfu_access(parent_str, W_OK) < 0) { MFU_LOG(MFU_LOG_ERR, "Destination parent directory is not writable `%s'", parent_str); valid = 0; mfu_free(&parent_str); goto bcast; } mfu_free(&parent_str); } /* determine whether caller *requires* copy into dir */ /* TODO: if caller specifies dest/ or dest/. */ /* if caller specifies more than one source, * then dest has to be a directory */ if(num_src_params > 1) { dest_required_to_be_dir = true; } /* if caller requires dest to be a directory, and if dest does not * exist or it does it exist but it's not a directory, then abort */ if(dest_required_to_be_dir && (!dest_exists || (!dest_is_dir && !dest_is_link_to_dir))) { MFU_LOG(MFU_LOG_ERR, "Destination is not a directory `%s'", dest_param.orig); valid = 0; goto bcast; } /* we copy into a directory if any of the following: * 1) user specified more than one source * 2) destination already exists and is a directory * 3) destination already exists and is a link to a directory */ bool copy_into_dir = (dest_required_to_be_dir || dest_is_dir || dest_is_link_to_dir); DCOPY_user_opts.copy_into_dir = copy_into_dir ? 1 : 0; } /* get status from rank 0 */ bcast: MPI_Bcast(&valid, 1, MPI_INT, 0, MPI_COMM_WORLD); /* exit job if we found a problem */ if(! valid) { if(DCOPY_global_rank == 0) { MFU_LOG(MFU_LOG_ERR, "Exiting run"); } MPI_Barrier(MPI_COMM_WORLD); DCOPY_exit(EXIT_FAILURE); } /* rank 0 broadcasts whether we're copying into a directory */ MPI_Bcast(&DCOPY_user_opts.copy_into_dir, 1, MPI_INT, 0, MPI_COMM_WORLD); }