int MFU_PRED_EXEC (mfu_flist flist, uint64_t idx, void* arg)
{
int argmax = 1024*1024;;
int written = 0;
int ret;
char* command = MFU_STRDUP((char*) arg);
char* cmdline = (char*) MFU_MALLOC(argmax);
char* subst = strstr(command, "{}");
if (subst) {
subst[0] = '\0';
subst += 2; /* Point to the first char after '{}' */
}
const char* name = mfu_flist_file_get_name(flist, idx);
written = snprintf(cmdline, argmax/sizeof(char), "%s%s%s", command, name, subst);
if (written > argmax/sizeof(char)) {
fprintf(stderr, "argument %s to exec too long.\n", cmdline);
mfu_free(&cmdline);
mfu_free(&command);
return -1;
}
ret = system(cmdline);
mfu_free(&cmdline);
mfu_free(&command);
return ret ? 0 : 1;
}
/* insert copy of specified element into list */
static void list_insert_copy(flist_t* flist, elem_t* src)
{
/* create new element */
elem_t* elem = (elem_t*) MFU_MALLOC(sizeof(elem_t));
/* copy values from source */
elem->file = MFU_STRDUP(src->file);
elem->depth = src->depth;
elem->type = src->type;
elem->detail = src->detail;
elem->mode = src->mode;
elem->uid = src->uid;
elem->gid = src->gid;
elem->atime = src->atime;
elem->atime_nsec = src->atime_nsec;
elem->mtime = src->mtime;
elem->mtime_nsec = src->mtime_nsec;
elem->ctime = src->ctime;
elem->ctime_nsec = src->ctime_nsec;
elem->size = src->size;
/* append element to tail of linked list */
mfu_flist_insert_elem(flist, elem);
return;
}
/* unpack element from buffer and return number of bytes read */
static size_t list_elem_unpack2(const void* buf, elem_t* elem)
{
/* set pointer to start of buffer */
const char* start = (const char*) buf;
const char* ptr = start;
/* extract detail field */
uint32_t detail;
mfu_unpack_uint32(&ptr, &detail);
/* extract length of file name field */
uint32_t chars;
mfu_unpack_uint32(&ptr, &chars);
/* get name and advance pointer */
const char* file = ptr;
ptr += chars;
/* copy path */
elem->file = MFU_STRDUP(file);
/* set depth */
elem->depth = mfu_flist_compute_depth(file);
elem->detail = (int) detail;
if (detail) {
/* extract fields */
mfu_unpack_uint64(&ptr, &elem->mode);
mfu_unpack_uint64(&ptr, &elem->uid);
mfu_unpack_uint64(&ptr, &elem->gid);
mfu_unpack_uint64(&ptr, &elem->atime);
mfu_unpack_uint64(&ptr, &elem->atime_nsec);
mfu_unpack_uint64(&ptr, &elem->mtime);
mfu_unpack_uint64(&ptr, &elem->mtime_nsec);
mfu_unpack_uint64(&ptr, &elem->ctime);
mfu_unpack_uint64(&ptr, &elem->ctime_nsec);
mfu_unpack_uint64(&ptr, &elem->size);
/* use mode to set file type */
elem->type = mfu_flist_mode_to_filetype((mode_t)elem->mode);
}
else {
/* only have type */
uint32_t type;
mfu_unpack_uint32(&ptr, &type);
elem->type = (mfu_filetype) type;
}
size_t bytes = (size_t)(ptr - start);
return bytes;
}
static int distribution_parse(struct distribute_option *option, const char *string)
{
char *ptr;
char *next;
unsigned long long separator;
char *str;
int status = 0;
if (strncmp(string, "size", strlen("size")) != 0) {
return -1;
}
option->separator_number = 0;
if (strlen(string) == strlen("size")) {
return 0;
}
if (string[strlen("size")] != ':') {
return -1;
}
str = MFU_STRDUP(string);
/* Parse separators */
ptr = str + strlen("size:");
next = ptr;
while (ptr && ptr < str + strlen(string)) {
next = strchr(ptr, ',');
if (next != NULL) {
*next = '\0';
next++;
}
if (mfu_abtoull(ptr, &separator) != MFU_SUCCESS) {
printf("Invalid separator \"%s\"\n", ptr);
status = -1;
goto out;
}
if (distribute_separator_add(option, separator)) {
printf("Duplicated separator \"%llu\"\n", separator);
status = -1;
goto out;
}
ptr = next;
}
out:
mfu_free(&str);
return status;
}
/* we hash file names based on its parent directory to map all
* files in the same directory to the same process */
static int map_name(mfu_flist flist, uint64_t idx, int ranks, void* args)
{
/* get name of item */
const char* name = mfu_flist_file_get_name(flist, idx);
/* identify rank to send this file to */
char* dir = MFU_STRDUP(name);
dirname(dir);
size_t dir_len = strlen(dir);
uint32_t hash = mfu_hash_jenkins(dir, dir_len);
int rank = (int)(hash % (uint32_t)ranks);
mfu_free(&dir);
return rank;
}
/**
* Parse the source and destination paths that the user has provided.
*/
void DCOPY_parse_path_args(char** argv, \
int optind_local, \
int argc)
{
/* compute number of paths and index of last argument */
int num_args = argc - optind_local;
int last_arg_index = num_args + optind_local - 1;
/* we need to have at least two paths,
* one or more sources and one destination */
if(argv == NULL || num_args < 2) {
if(DCOPY_global_rank == 0) {
DCOPY_print_usage();
MFU_LOG(MFU_LOG_ERR, "You must specify a source and destination path");
}
MPI_Barrier(MPI_COMM_WORLD);
DCOPY_exit(EXIT_FAILURE);
}
/* determine number of source paths */
src_params = NULL;
num_src_params = last_arg_index - optind_local;
/* allocate space to record info about each source */
size_t src_params_bytes = ((size_t) num_src_params) * sizeof(mfu_param_path);
src_params = (mfu_param_path*) MFU_MALLOC(src_params_bytes);
/* record standardized paths and stat info for each source */
int opt_index;
for(opt_index = optind_local; opt_index < last_arg_index; opt_index++) {
char* path = argv[opt_index];
int idx = opt_index - optind_local;
mfu_param_path_set(path, &src_params[idx]);
}
/* standardize destination path */
const char* dstpath = argv[last_arg_index];
mfu_param_path_set(dstpath, &dest_param);
/* copy the destination path to user opts structure */
DCOPY_user_opts.dest_path = MFU_STRDUP(dest_param.path);
/* check that source and destinations are ok */
DCOPY_check_paths();
}
int DCOPY_open_file(const char* file, int read_flag, DCOPY_file_cache_t* cache)
{
int newfd = -1;
/* see if we have a cached file descriptor */
char* name = cache->name;
if (name != NULL) {
/* we have a cached file descriptor */
int fd = cache->fd;
if (strcmp(name, file) == 0 && cache->read == read_flag) {
/* the file we're trying to open matches name and read/write mode,
* so just return the cached descriptor */
return fd;
} else {
/* the file we're trying to open is different,
* close the old file and delete the name */
mfu_close(name, fd);
mfu_free(&cache->name);
}
}
/* open the new file */
if (read_flag) {
int flags = O_RDONLY;
if (DCOPY_user_opts.synchronous) {
flags |= O_DIRECT;
}
newfd = mfu_open(file, flags);
} else {
int flags = O_WRONLY | O_CREAT;
if (DCOPY_user_opts.synchronous) {
flags |= O_DIRECT;
}
newfd = mfu_open(file, flags, DCOPY_DEF_PERMS_FILE);
}
/* cache the file descriptor */
if (newfd != -1) {
cache->name = MFU_STRDUP(file);
cache->fd = newfd;
cache->read = read_flag;
}
return newfd;
}
/* unpack a string from a buffer, and return as newly allocated memory,
* caller must free returned string */
static void mfu_unpack_str(const char** pptr, char** pstr)
{
/* assume we don't have a string */
*pstr = NULL;
/* unpack the flag */
uint32_t flag;
mfu_unpack_uint32(pptr, &flag);
/* if we have a string, unpack the string */
if (flag) {
/* make a copy of the string to return to caller */
const char* str = *pptr;
*pstr = MFU_STRDUP(str);
/* advance caller's pointer */
size_t len = strlen(str) + 1;
*pptr += len;
}
return;
}
int DCOPY_open_file(const char* file, int read_flag, DCOPY_file_cache_t* cache)
{
int newfd = -1;
/* see if we have a cached file descriptor */
char* name = cache->name;
if (name != NULL) {
/* we have a cached file descriptor */
int fd = cache->fd;
if (strcmp(name, file) == 0 && cache->read == read_flag) {
/* the file we're trying to open matches name and read/write mode,
* so just return the cached descriptor */
return fd;
} else {
/* the file we're trying to open is different,
* close the old file and delete the name */
mfu_close(name, fd);
mfu_free(&cache->name);
}
}
/* open the new file */
if (read_flag) {
int flags = O_RDONLY;
if (DCOPY_user_opts.synchronous) {
flags |= O_DIRECT;
}
newfd = mfu_open(file, flags);
} else {
int flags = O_WRONLY | O_CREAT;
if (DCOPY_user_opts.synchronous) {
flags |= O_DIRECT;
}
newfd = mfu_open(file, flags, DCOPY_DEF_PERMS_FILE);
}
/* cache the file descriptor */
if (newfd != -1) {
cache->name = MFU_STRDUP(file);
cache->fd = newfd;
cache->read = read_flag;
#ifdef LUSTRE_SUPPORT
/* Zero is an invalid ID for grouplock. */
if (DCOPY_user_opts.grouplock_id != 0) {
int rc;
rc = ioctl(newfd, LL_IOC_GROUP_LOCK, DCOPY_user_opts.grouplock_id);
if (rc) {
MFU_LOG(MFU_LOG_ERR, "Failed to obtain grouplock with ID %d "
"on file `%s', ignoring this error: %s",
DCOPY_user_opts.grouplock_id,
file, strerror(errno));
} else {
MFU_LOG(MFU_LOG_INFO, "Obtained grouplock with ID %d "
"on file `%s', fd %d", DCOPY_user_opts.grouplock_id,
file, newfd);
}
}
#endif
}
return newfd;
}
int main(int argc, char** argv)
{
/* initialize MPI */
MPI_Init(&argc, &argv);
mfu_init();
/* get our rank and the size of comm_world */
int rank, ranks;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &ranks);
/* pointer to mfu_walk_opts */
mfu_walk_opts_t* walk_opts = mfu_walk_opts_new();
/* parse command line options */
char* inputname = NULL;
char* ownername = NULL;
char* groupname = NULL;
char* modestr = NULL;
char* regex_exp = NULL;
mfu_perms* head = NULL;
int walk = 0;
int exclude = 0;
int name = 0;
/* verbose by default */
mfu_debug_level = MFU_LOG_VERBOSE;
int option_index = 0;
static struct option long_options[] = {
{"input", 1, 0, 'i'},
{"owner", 1, 0, 'u'},
{"group", 1, 0, 'g'},
{"mode", 1, 0, 'm'},
{"exclude", 1, 0, 'e'},
{"match", 1, 0, 'a'},
{"name", 0, 0, 'n'},
{"progress", 1, 0, 'P'},
{"verbose", 0, 0, 'v'},
{"quiet", 0, 0, 'q'},
{"help", 0, 0, 'h'},
{0, 0, 0, 0}
};
int usage = 0;
while (1) {
int c = getopt_long(
argc, argv, "i:u:g:m:nvqh",
long_options, &option_index
);
if (c == -1) {
break;
}
switch (c) {
case 'i':
inputname = MFU_STRDUP(optarg);
break;
case 'u':
ownername = MFU_STRDUP(optarg);
break;
case 'g':
groupname = MFU_STRDUP(optarg);
break;
case 'm':
modestr = MFU_STRDUP(optarg);
break;
case 'e':
regex_exp = MFU_STRDUP(optarg);
exclude = 1;
break;
case 'a':
regex_exp = MFU_STRDUP(optarg);
exclude = 0;
break;
case 'n':
name = 1;
break;
case 'P':
mfu_progress_timeout = atoi(optarg);
break;
case 'v':
mfu_debug_level = MFU_LOG_VERBOSE;
break;
case 'q':
mfu_debug_level = MFU_LOG_NONE;
break;
case 'h':
usage = 1;
break;
case '?':
usage = 1;
break;
default:
if (rank == 0) {
printf("?? getopt returned character code 0%o ??\n", c);
}
}
}
/* check that we got a valid progress value */
if (mfu_progress_timeout < 0) {
if (rank == 0) {
MFU_LOG(MFU_LOG_ERR, "Seconds in --progress must be non-negative: %d invalid", mfu_progress_timeout);
}
usage = 1;
}
/* paths to walk come after the options */
int numpaths = 0;
mfu_param_path* paths = NULL;
if (optind < argc) {
/* got a path to walk */
walk = 1;
/* determine number of paths specified by user */
numpaths = argc - optind;
/* allocate space for each path */
paths = (mfu_param_path*) MFU_MALLOC((size_t)numpaths * sizeof(mfu_param_path));
/* process each path */
const char** argpaths = (const char**)(&argv[optind]);
mfu_param_path_set_all(numpaths, argpaths, paths);
/* advance to next set of options */
optind += numpaths;
/* don't allow input file and walk */
if (inputname != NULL) {
usage = 1;
}
}
else {
/* if we're not walking, we must be reading,
* and for that we need a file */
if (inputname == NULL) {
usage = 1;
}
}
/* check that our mode string parses correctly */
if (modestr != NULL) {
int valid = mfu_perms_parse(modestr, &head);
if (! valid) {
usage = 1;
if (rank == 0) {
printf("invalid mode string: %s\n", modestr);
}
/* free the head of the list */
mfu_perms_free(&head);
}
}
/* print usage if we need to */
if (usage) {
if (rank == 0) {
print_usage();
}
mfu_finalize();
MPI_Finalize();
return 1;
}
/* create an empty file list */
mfu_flist flist = mfu_flist_new();
/* flag used to check if permissions need to be
* set on the walk */
if (head != NULL) {
mfu_perms_need_dir_rx(head, walk_opts);
}
/* get our list of files, either by walking or reading an
* input file */
if (walk) {
/* if in octal mode set use_stat=0 to stat each file on walk */
if (head != NULL && head->octal && ownername == NULL && groupname == NULL) {
walk_opts->use_stat = 0;
}
/* walk list of input paths */
mfu_flist_walk_param_paths(numpaths, paths, walk_opts, flist);
}
else {
/* read list from file */
mfu_flist_read_cache(inputname, flist);
}
/* assume we'll use the full list */
mfu_flist srclist = flist;
/* filter the list if needed */
mfu_flist filtered_flist = MFU_FLIST_NULL;
if (regex_exp != NULL) {
/* filter the list based on regex */
filtered_flist = mfu_flist_filter_regex(flist, regex_exp, exclude, name);
/* update our source list to use the filtered list instead of the original */
srclist = filtered_flist;
}
/* change group and permissions */
mfu_flist_chmod(srclist, ownername, groupname, head);
/* free list if it was used */
if (filtered_flist != MFU_FLIST_NULL){
/* free the filtered flist (if any) */
mfu_flist_free(&filtered_flist);
}
/* free the file list */
mfu_flist_free(&flist);
/* free the path parameters */
mfu_param_path_free_all(numpaths, paths);
/* free memory allocated to hold params */
mfu_free(&paths);
/* free the owner and group names */
mfu_free(&ownername);
mfu_free(&groupname);
/* free the modestr */
mfu_free(&modestr);
/* free the match_pattern if it isn't null */
if (regex_exp != NULL) {
mfu_free(®ex_exp);
}
/* free the head of the list */
mfu_perms_free(&head);
/* free the input file name */
mfu_free(&inputname);
/* free the walk options */
mfu_walk_opts_delete(&walk_opts);
/* shut down MPI */
mfu_finalize();
MPI_Finalize();
return 0;
}
int main(int argc, char** argv)
{
int i;
/* initialize MPI */
MPI_Init(&argc, &argv);
mfu_init();
/* get our rank and the size of comm_world */
int rank, ranks;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &ranks);
/* parse command line options */
char* inputname = NULL;
char* regex_exp = NULL;
int walk = 0;
int exclude = 0;
int name = 0;
int dryrun = 0;
int option_index = 0;
static struct option long_options[] = {
{"input", 1, 0, 'i'},
{"lite", 0, 0, 'l'},
{"exclude", 1, 0, 'e'},
{"match", 1, 0, 'a'},
{"name", 0, 0, 'n'},
{"help", 0, 0, 'h'},
{"dryrun", 0, 0, 'd'},
{"verbose", 0, 0, 'v'},
{0, 0, 0, 0}
};
int usage = 0;
while (1) {
int c = getopt_long(
argc, argv, "i:nlhv",
long_options, &option_index
);
if (c == -1) {
break;
}
switch (c) {
case 'i':
inputname = MFU_STRDUP(optarg);
break;
case 'l':
walk_stat = 0;
break;
case 'e':
regex_exp = MFU_STRDUP(optarg);
exclude = 1;
break;
case 'a':
regex_exp = MFU_STRDUP(optarg);
exclude = 0;
break;
case 'n':
name = 1;
break;
case 'h':
usage = 1;
break;
case 'd':
dryrun = 1;
break;
case 'v':
mfu_debug_level = MFU_LOG_VERBOSE;
break;
case '?':
usage = 1;
break;
default:
if (rank == 0) {
printf("?? getopt returned character code 0%o ??\n", c);
}
}
}
/* paths to walk come after the options */
int numpaths = 0;
mfu_param_path* paths = NULL;
if (optind < argc) {
/* got a path to walk */
walk = 1;
/* determine number of paths specified by user */
numpaths = argc - optind;
/* allocate space for each path */
paths = (mfu_param_path*) MFU_MALLOC((size_t)numpaths * sizeof(mfu_param_path));
/* process each path */
char** argpaths = &argv[optind];
mfu_param_path_set_all(numpaths, argpaths, paths);
/* advance to next set of options */
optind += numpaths;
/* don't allow input file and walk */
if (inputname != NULL) {
usage = 1;
}
}
else {
/* if we're not walking, we must be reading,
* and for that we need a file */
if (inputname == NULL) {
usage = 1;
}
}
/* print usage if we need to */
if (usage) {
if (rank == 0) {
print_usage();
}
mfu_finalize();
MPI_Finalize();
return 1;
}
/* create an empty file list */
mfu_flist flist = mfu_flist_new();
/* get our list of files, either by walking or reading an
* input file */
if (walk) {
/* walk list of input paths */
mfu_param_path_walk(numpaths, paths, walk_stat, flist, dir_perm);
}
else {
/* read list from file */
mfu_flist_read_cache(inputname, flist);
}
/* assume we'll use the full list */
mfu_flist srclist = flist;
/* filter the list if needed */
mfu_flist filtered_flist = MFU_FLIST_NULL;
if (regex_exp != NULL) {
/* filter the list based on regex */
filtered_flist = mfu_flist_filter_regex(flist, regex_exp, exclude, name);
/* update our source list to use the filtered list instead of the original */
srclist = filtered_flist;
}
/* only actually delete files if the user wasn't doing a dry run */
if (dryrun) {
/* just print what we would delete without actually doing anything,
* this is useful if the user is trying to get a regex right */
mfu_flist_print(srclist);
} else {
/* remove files */
mfu_flist_unlink(srclist);
}
/* free list if it was used */
if (filtered_flist != MFU_FLIST_NULL){
/* free the filtered flist (if any) */
mfu_flist_free(&filtered_flist);
}
/* free the file list */
mfu_flist_free(&flist);
/* free the path parameters */
mfu_param_path_free_all(numpaths, paths);
/* free memory allocated to hold params */
mfu_free(&paths);
/* free the regex string if we have one */
mfu_free(®ex_exp);
/* free the input file name */
mfu_free(&inputname);
/* shut down MPI */
mfu_finalize();
MPI_Finalize();
return 0;
}
int main (int argc, char** argv)
{
/* initialize MPI */
MPI_Init(&argc, &argv);
mfu_init();
/* get our rank and the size of comm_world */
int rank, ranks;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &ranks);
/* pointer to mfu_walk_opts */
mfu_walk_opts_t* walk_opts = mfu_walk_opts_new();
/* capture current time for any time based queries,
* to get a consistent value, capture and bcast from rank 0 */
mfu_pred_times* now_t = mfu_pred_now();
int ch;
mfu_pred* pred_head = mfu_pred_new();
char* inputname = NULL;
char* outputname = NULL;
int walk = 0;
int text = 0;
static struct option long_options[] = {
{"input", 1, 0, 'i'},
{"output", 1, 0, 'o'},
{"verbose", 0, 0, 'v'},
{"quiet", 0, 0, 'q'},
{"help", 0, 0, 'h'},
{ "maxdepth", required_argument, NULL, 'd' },
{ "amin", required_argument, NULL, 'a' },
{ "anewer", required_argument, NULL, 'B' },
{ "atime", required_argument, NULL, 'A' },
{ "cmin", required_argument, NULL, 'c' },
{ "cnewer", required_argument, NULL, 'D' },
{ "ctime", required_argument, NULL, 'C' },
{ "mmin", required_argument, NULL, 'm' },
{ "newer", required_argument, NULL, 'N' },
{ "mtime", required_argument, NULL, 'M' },
{ "gid", required_argument, NULL, 'g' },
{ "group", required_argument, NULL, 'G' },
{ "uid", required_argument, NULL, 'u' },
{ "user", required_argument, NULL, 'U' },
{ "name", required_argument, NULL, 'n' },
{ "path", required_argument, NULL, 'P' },
{ "regex", required_argument, NULL, 'r' },
{ "size", required_argument, NULL, 's' },
{ "type", required_argument, NULL, 't' },
{ "print", no_argument, NULL, 'p' },
{ "exec", required_argument, NULL, 'e' },
{ NULL, 0, NULL, 0 },
};
options.maxdepth = INT_MAX;
int usage = 0;
while (1) {
int c = getopt_long(
argc, argv, "i:o:vqh",
long_options, NULL
);
if (c == -1) {
break;
}
int i;
int space;
char* buf;
mfu_pred_times* t;
mfu_pred_times_rel* tr;
regex_t* r;
int ret;
/* verbose by default */
mfu_debug_level = MFU_LOG_VERBOSE;
switch (c) {
case 'e':
space = 1024 * 1024;
buf = (char *)MFU_MALLOC(space);
for (i = optind-1; strcmp(";", argv[i]); i++) {
if (i > argc) {
if (rank == 0) {
printf("%s: exec missing terminating ';'\n", argv[0]);
}
exit(1);
}
strncat(buf, argv[i], space);
space -= strlen(argv[i]) + 1; /* save room for space or null */
if (space <= 0) {
if (rank == 0) {
printf("%s: exec argument list too long.\n", argv[0]);
}
mfu_free(&buf);
continue;
}
strcat(buf, " ");
optind++;
}
buf[strlen(buf)] = '\0'; /* clobbers trailing space */
mfu_pred_add(pred_head, MFU_PRED_EXEC, buf);
break;
case 'd':
options.maxdepth = atoi(optarg);
break;
case 'g':
/* TODO: error check argument */
buf = MFU_STRDUP(optarg);
mfu_pred_add(pred_head, MFU_PRED_GID, (void *)buf);
break;
case 'G':
buf = MFU_STRDUP(optarg);
mfu_pred_add(pred_head, MFU_PRED_GROUP, (void *)buf);
break;
case 'u':
/* TODO: error check argument */
buf = MFU_STRDUP(optarg);
mfu_pred_add(pred_head, MFU_PRED_UID, (void *)buf);
break;
case 'U':
buf = MFU_STRDUP(optarg);
mfu_pred_add(pred_head, MFU_PRED_USER, (void *)buf);
break;
case 's':
buf = MFU_STRDUP(optarg);
mfu_pred_add(pred_head, MFU_PRED_SIZE, (void *)buf);
break;
case 'n':
mfu_pred_add(pred_head, MFU_PRED_NAME, MFU_STRDUP(optarg));
break;
case 'P':
mfu_pred_add(pred_head, MFU_PRED_PATH, MFU_STRDUP(optarg));
break;
case 'r':
r = (regex_t*) MFU_MALLOC(sizeof(regex_t));
ret = regcomp(r, optarg, 0);
if (ret) {
MFU_ABORT(-1, "Could not compile regex: `%s' rc=%d\n", optarg, ret);
}
mfu_pred_add(pred_head, MFU_PRED_REGEX, (void*)r);
break;
case 'a':
tr = mfu_pred_relative(optarg, now_t);
mfu_pred_add(pred_head, MFU_PRED_AMIN, (void *)tr);
break;
case 'm':
tr = mfu_pred_relative(optarg, now_t);
mfu_pred_add(pred_head, MFU_PRED_MMIN, (void *)tr);
break;
case 'c':
tr = mfu_pred_relative(optarg, now_t);
mfu_pred_add(pred_head, MFU_PRED_CMIN, (void *)tr);
break;
case 'A':
tr = mfu_pred_relative(optarg, now_t);
mfu_pred_add(pred_head, MFU_PRED_ATIME, (void *)tr);
break;
case 'M':
tr = mfu_pred_relative(optarg, now_t);
mfu_pred_add(pred_head, MFU_PRED_MTIME, (void *)tr);
break;
case 'C':
tr = mfu_pred_relative(optarg, now_t);
mfu_pred_add(pred_head, MFU_PRED_CTIME, (void *)tr);
break;
case 'B':
t = get_mtimes(optarg);
if (t == NULL) {
if (rank == 0) {
printf("%s: can't find file %s\n", argv[0], optarg);
}
exit(1);
}
mfu_pred_add(pred_head, MFU_PRED_ANEWER, (void *)t);
break;
case 'N':
t = get_mtimes(optarg);
if (t == NULL) {
if (rank == 0) {
printf("%s: can't find file %s\n", argv[0], optarg);
}
exit(1);
}
mfu_pred_add(pred_head, MFU_PRED_MNEWER, (void *)t);
break;
case 'D':
t = get_mtimes(optarg);
if (t == NULL) {
if (rank == 0) {
printf("%s: can't find file %s\n", argv[0], optarg);
}
exit(1);
}
mfu_pred_add(pred_head, MFU_PRED_CNEWER, (void *)t);
break;
case 'p':
mfu_pred_add(pred_head, MFU_PRED_PRINT, NULL);
break;
case 't':
ret = add_type(pred_head, *optarg);
if (ret != 1) {
if (rank == 0) {
printf("%s: unsupported file type %s\n", argv[0], optarg);
}
exit(1);
}
break;
case 'i':
inputname = MFU_STRDUP(optarg);
break;
case 'o':
outputname = MFU_STRDUP(optarg);
break;
case 'v':
mfu_debug_level = MFU_LOG_VERBOSE;
break;
case 'q':
mfu_debug_level = MFU_LOG_NONE;
break;
case 'h':
usage = 1;
break;
case '?':
usage = 1;
break;
default:
if (rank == 0) {
printf("?? getopt returned character code 0%o ??\n", c);
}
}
}
pred_commit(pred_head);
/* paths to walk come after the options */
int numpaths = 0;
mfu_param_path* paths = NULL;
if (optind < argc) {
/* got a path to walk */
walk = 1;
/* determine number of paths specified by user */
numpaths = argc - optind;
/* allocate space for each path */
paths = (mfu_param_path*) MFU_MALLOC((size_t)numpaths * sizeof(mfu_param_path));
/* process each path */
char** p = &argv[optind];
mfu_param_path_set_all((uint64_t)numpaths, (const char**)p, paths);
optind += numpaths;
/* don't allow user to specify input file with walk */
if (inputname != NULL) {
usage = 1;
}
}
else {
/* if we're not walking, we must be reading,
* and for that we need a file */
if (inputname == NULL) {
usage = 1;
}
}
if (usage) {
if (rank == 0) {
print_usage();
}
mfu_finalize();
MPI_Finalize();
return 0;
}
/* create an empty file list */
mfu_flist flist = mfu_flist_new();
if (walk) {
/* walk list of input paths */
mfu_flist_walk_param_paths(numpaths, paths, walk_opts, flist);
}
else {
/* read data from cache file */
mfu_flist_read_cache(inputname, flist);
}
/* apply predicates to each item in list */
mfu_flist flist2 = mfu_flist_filter_pred(flist, pred_head);
/* write data to cache file */
if (outputname != NULL) {
if (!text) {
mfu_flist_write_cache(outputname, flist2);
} else {
mfu_flist_write_text(outputname, flist2);
}
}
/* free off the filtered list */
mfu_flist_free(&flist2);
/* free users, groups, and files objects */
mfu_flist_free(&flist);
/* free predicate list */
mfu_pred_free(&pred_head);
/* free memory allocated for options */
mfu_free(&outputname);
mfu_free(&inputname);
/* free the path parameters */
mfu_param_path_free_all(numpaths, paths);
/* free memory allocated to hold params */
mfu_free(&paths);
/* free structure holding current time */
mfu_free(&now_t);
/* free the walk options */
mfu_walk_opts_delete(&walk_opts);
/* shut down MPI */
mfu_finalize();
MPI_Finalize();
return 0;
}
int main(int argc, char** argv)
{
int i;
/* initialize MPI */
MPI_Init(&argc, &argv);
mfu_init();
/* get our rank and the size of comm_world */
int rank, ranks;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &ranks);
/* pointer to mfu_walk_opts */
mfu_walk_opts_t* walk_opts = mfu_walk_opts_new();
/* TODO: extend options
* - allow user to cache scan result in file
* - allow user to load cached scan as input
*
* - allow user to filter by user, group, or filename using keyword or regex
* - allow user to specify time window
* - allow user to specify file sizes
*
* - allow user to sort by different fields
* - allow user to group output (sum all bytes, group by user) */
char* inputname = NULL;
char* outputname = NULL;
char* sortfields = NULL;
char* distribution = NULL;
int file_histogram = 0;
int walk = 0;
int print = 0;
int text = 0;
struct distribute_option option;
/* verbose by default */
mfu_debug_level = MFU_LOG_VERBOSE;
int option_index = 0;
static struct option long_options[] = {
{"input", 1, 0, 'i'},
{"output", 1, 0, 'o'},
{"text", 0, 0, 't'},
{"lite", 0, 0, 'l'},
{"sort", 1, 0, 's'},
{"distribution", 1, 0, 'd'},
{"file_histogram", 0, 0, 'f'},
{"print", 0, 0, 'p'},
{"verbose", 0, 0, 'v'},
{"quiet", 0, 0, 'q'},
{"help", 0, 0, 'h'},
{0, 0, 0, 0}
};
int usage = 0;
while (1) {
int c = getopt_long(
argc, argv, "i:o:tls:d:fpvqh",
long_options, &option_index
);
if (c == -1) {
break;
}
switch (c) {
case 'i':
inputname = MFU_STRDUP(optarg);
break;
case 'o':
outputname = MFU_STRDUP(optarg);
break;
case 'l':
/* don't stat each file on the walk */
walk_opts->use_stat = 0;
break;
case 's':
sortfields = MFU_STRDUP(optarg);
break;
case 'd':
distribution = MFU_STRDUP(optarg);
break;
case 'f':
file_histogram = 1;
break;
case 'p':
print = 1;
break;
case 'v':
mfu_debug_level = MFU_LOG_VERBOSE;
break;
case 'q':
mfu_debug_level = 0;
break;
case 't':
text = 1;
break;
case 'h':
usage = 1;
break;
case '?':
usage = 1;
break;
default:
if (rank == 0) {
printf("?? getopt returned character code 0%o ??\n", c);
}
}
}
/* paths to walk come after the options */
int numpaths = 0;
mfu_param_path* paths = NULL;
if (optind < argc) {
/* got a path to walk */
walk = 1;
/* determine number of paths specified by user */
numpaths = argc - optind;
/* allocate space for each path */
paths = (mfu_param_path*) MFU_MALLOC((size_t)numpaths * sizeof(mfu_param_path));
/* process each path */
char** p = &argv[optind];
mfu_param_path_set_all((uint64_t)numpaths, (const char**)p, paths);
optind += numpaths;
/* don't allow user to specify input file with walk */
if (inputname != NULL) {
usage = 1;
}
}
else {
/* if we're not walking, we must be reading,
* and for that we need a file */
if (inputname == NULL) {
usage = 1;
}
}
/* if user is trying to sort, verify the sort fields are valid */
if (sortfields != NULL) {
int maxfields;
int nfields = 0;
char* sortfields_copy = MFU_STRDUP(sortfields);
if (walk_opts->use_stat) {
maxfields = 7;
char* token = strtok(sortfields_copy, ",");
while (token != NULL) {
if (strcmp(token, "name") != 0 &&
strcmp(token, "-name") != 0 &&
strcmp(token, "user") != 0 &&
strcmp(token, "-user") != 0 &&
strcmp(token, "group") != 0 &&
strcmp(token, "-group") != 0 &&
strcmp(token, "uid") != 0 &&
strcmp(token, "-uid") != 0 &&
strcmp(token, "gid") != 0 &&
strcmp(token, "-gid") != 0 &&
strcmp(token, "atime") != 0 &&
strcmp(token, "-atime") != 0 &&
strcmp(token, "mtime") != 0 &&
strcmp(token, "-mtime") != 0 &&
strcmp(token, "ctime") != 0 &&
strcmp(token, "-ctime") != 0 &&
strcmp(token, "size") != 0 &&
strcmp(token, "-size") != 0) {
/* invalid token */
if (rank == 0) {
printf("Invalid sort field: %s\n", token);
}
usage = 1;
}
nfields++;
token = strtok(NULL, ",");
}
}
else {
maxfields = 1;
char* token = strtok(sortfields_copy, ",");
while (token != NULL) {
if (strcmp(token, "name") != 0 &&
strcmp(token, "-name") != 0) {
/* invalid token */
if (rank == 0) {
printf("Invalid sort field: %s\n", token);
}
usage = 1;
}
nfields++;
token = strtok(NULL, ",");
}
}
if (nfields > maxfields) {
if (rank == 0) {
printf("Exceeded maximum number of sort fields: %d\n", maxfields);
}
usage = 1;
}
mfu_free(&sortfields_copy);
}
if (distribution != NULL) {
if (distribution_parse(&option, distribution) != 0) {
if (rank == 0) {
printf("Invalid distribution argument: %s\n", distribution);
}
usage = 1;
} else if (rank == 0 && option.separator_number != 0) {
printf("Separators: ");
for (i = 0; i < option.separator_number; i++) {
if (i != 0) {
printf(", ");
}
printf("%"PRIu64, option.separators[i]);
}
printf("\n");
}
}
if (usage) {
if (rank == 0) {
print_usage();
}
MPI_Finalize();
return 0;
}
/* TODO: check stat fields fit within MPI types */
// if (sizeof(st_uid) > uint64_t) error(); etc...
/* create an empty file list with default values */
mfu_flist flist = mfu_flist_new();
if (walk) {
/* walk list of input paths */
mfu_flist_walk_param_paths(numpaths, paths, walk_opts, flist);
}
else {
/* read data from cache file */
mfu_flist_read_cache(inputname, flist);
}
/* TODO: filter files */
//filter_files(&flist);
/* sort files */
if (sortfields != NULL) {
/* TODO: don't sort unless all_count > 0 */
mfu_flist_sort(sortfields, &flist);
}
/* print details for individual files */
if (print) {
mfu_flist_print(flist);
}
/* print summary statistics of flist */
mfu_flist_print_summary(flist);
/* print distribution if user specified this option */
if (distribution != NULL || file_histogram) {
print_flist_distribution(file_histogram, &option, &flist, rank);
}
/* write data to cache file */
if (outputname != NULL) {
if (!text) {
mfu_flist_write_cache(outputname, flist);
} else {
mfu_flist_write_text(outputname, flist);
}
}
/* free users, groups, and files objects */
mfu_flist_free(&flist);
/* free memory allocated for options */
mfu_free(&distribution);
mfu_free(&sortfields);
mfu_free(&outputname);
mfu_free(&inputname);
/* free the path parameters */
mfu_param_path_free_all(numpaths, paths);
/* free memory allocated to hold params */
mfu_free(&paths);
/* free the walk options */
mfu_walk_opts_delete(&walk_opts);
/* shut down MPI */
mfu_finalize();
MPI_Finalize();
return 0;
}
static int sort_files_readdir(const char* sortfields, mfu_flist* pflist)
{
/* get list from caller */
mfu_flist flist = *pflist;
/* create a new list as subset of original list */
mfu_flist flist2 = mfu_flist_subset(flist);
uint64_t incount = mfu_flist_size(flist);
uint64_t chars = mfu_flist_file_max_name(flist);
/* create datatype for packed file list element */
MPI_Datatype dt_sat;
size_t bytes = mfu_flist_file_pack_size(flist);
MPI_Type_contiguous((int)bytes, MPI_BYTE, &dt_sat);
/* get our rank and the size of comm_world */
int rank, ranks;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &ranks);
/* build type for file path */
MPI_Datatype dt_filepath;
MPI_Type_contiguous((int)chars, MPI_CHAR, &dt_filepath);
MPI_Type_commit(&dt_filepath);
/* build comparison op for filenames */
DTCMP_Op op_filepath;
if (DTCMP_Op_create(dt_filepath, my_strcmp, &op_filepath) != DTCMP_SUCCESS) {
MFU_ABORT(1, "Failed to create sorting operation for filepath");
}
/* build comparison op for filenames */
DTCMP_Op op_filepath_rev;
if (DTCMP_Op_create(dt_filepath, my_strcmp_rev, &op_filepath_rev) != DTCMP_SUCCESS) {
MFU_ABORT(1, "Failed to create reverse sorting operation for filepath");
}
/* TODO: process sort fields */
const int MAXFIELDS = 1;
MPI_Datatype types[MAXFIELDS];
DTCMP_Op ops[MAXFIELDS];
sort_field fields[MAXFIELDS];
size_t lengths[MAXFIELDS];
int nfields = 0;
for (nfields = 0; nfields < MAXFIELDS; nfields++) {
types[nfields] = MPI_DATATYPE_NULL;
ops[nfields] = DTCMP_OP_NULL;
}
nfields = 0;
char* sortfields_copy = MFU_STRDUP(sortfields);
char* token = strtok(sortfields_copy, ",");
while (token != NULL) {
int valid = 1;
if (strcmp(token, "name") == 0) {
types[nfields] = dt_filepath;
ops[nfields] = op_filepath;
fields[nfields] = FILENAME;
lengths[nfields] = chars;
}
else if (strcmp(token, "-name") == 0) {
types[nfields] = dt_filepath;
ops[nfields] = op_filepath_rev;
fields[nfields] = FILENAME;
lengths[nfields] = chars;
}
else {
/* invalid token */
valid = 0;
if (rank == 0) {
MFU_LOG(MFU_LOG_ERR, "Invalid sort field: %s\n", token);
}
}
if (valid) {
nfields++;
}
if (nfields > MAXFIELDS) {
/* TODO: print warning if we have too many fields */
break;
}
token = strtok(NULL, ",");
}
mfu_free(&sortfields_copy);
/* build key type */
MPI_Datatype dt_key;
if (DTCMP_Type_create_series(nfields, types, &dt_key) != DTCMP_SUCCESS) {
MFU_ABORT(1, "Failed to create type for key");
}
/* create sort op */
DTCMP_Op op_key;
if (DTCMP_Op_create_series(nfields, ops, &op_key) != DTCMP_SUCCESS) {
MFU_ABORT(1, "Failed to create sorting operation for key");
}
/* build keysat type */
MPI_Datatype dt_keysat, keysat_types[2];
keysat_types[0] = dt_key;
keysat_types[1] = dt_sat;
if (DTCMP_Type_create_series(2, keysat_types, &dt_keysat) != DTCMP_SUCCESS) {
MFU_ABORT(1, "Failed to create type for keysat");
}
/* get extent of key type */
MPI_Aint key_lb, key_extent;
MPI_Type_get_extent(dt_key, &key_lb, &key_extent);
/* get extent of keysat type */
MPI_Aint keysat_lb, keysat_extent;
MPI_Type_get_extent(dt_keysat, &keysat_lb, &keysat_extent);
/* get extent of sat type */
MPI_Aint sat_lb, sat_extent;
MPI_Type_get_extent(dt_sat, &sat_lb, &sat_extent);
/* compute size of sort element and allocate buffer */
size_t sortbufsize = (size_t)keysat_extent * incount;
void* sortbuf = MFU_MALLOC(sortbufsize);
/* copy data into sort elements */
uint64_t idx = 0;
char* sortptr = (char*) sortbuf;
while (idx < incount) {
/* copy in access time */
int i;
for (i = 0; i < nfields; i++) {
if (fields[i] == FILENAME) {
const char* name = mfu_flist_file_get_name(flist, idx);
strcpy(sortptr, name);
}
sortptr += lengths[i];
}
/* pack file element */
sortptr += mfu_flist_file_pack(sortptr, flist, idx);
idx++;
}
/* sort data */
void* outsortbuf;
int outsortcount;
DTCMP_Handle handle;
int sort_rc = DTCMP_Sortz(
sortbuf, (int)incount, &outsortbuf, &outsortcount,
dt_key, dt_keysat, op_key, DTCMP_FLAG_NONE,
MPI_COMM_WORLD, &handle
);
if (sort_rc != DTCMP_SUCCESS) {
MFU_ABORT(1, "Failed to sort data");
}
/* step through sorted data filenames */
idx = 0;
sortptr = (char*) outsortbuf;
while (idx < (uint64_t)outsortcount) {
sortptr += key_extent;
sortptr += mfu_flist_file_unpack(sortptr, flist2);
idx++;
}
/* build summary of new list */
mfu_flist_summarize(flist2);
/* free memory */
DTCMP_Free(&handle);
/* free ops */
DTCMP_Op_free(&op_key);
DTCMP_Op_free(&op_filepath_rev);
DTCMP_Op_free(&op_filepath);
/* free types */
MPI_Type_free(&dt_keysat);
MPI_Type_free(&dt_key);
MPI_Type_free(&dt_filepath);
/* free input buffer holding sort elements */
mfu_free(&sortbuf);
/* free the satellite type */
MPI_Type_free(&dt_sat);
/* return new list and free old one */
*pflist = flist2;
mfu_flist_free(&flist);
return MFU_SUCCESS;
}
void mfu_param_path_set_all(uint64_t num, const char** paths, mfu_param_path* params)
{
/* get our rank and number of ranks */
int rank, ranks;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &ranks);
/* determine number we should look up */
uint64_t count = num / (uint64_t) ranks;
uint64_t extra = num - count * (uint64_t) ranks;
if (rank < (int) extra) {
/* procs whose rank is less than extra each
* handle one extra param than those whose
* rank is equal or greater than extra */
count++;
}
/* determine our starting point */
uint64_t start = 0;
if (rank < (int) extra) {
/* for these procs, count is one more than procs with ranks >= extra */
start = (uint64_t)rank * count;
} else {
/* for these procs, count is one less than procs with ranks < extra */
start = extra * (count + 1) + ((uint64_t)rank - extra) * count;
}
/* TODO: allocate temporary params */
mfu_param_path* p = MFU_MALLOC(count * sizeof(mfu_param_path));
/* track maximum path length */
uint64_t bytes = 0;
/* process each path we're responsible for */
uint64_t i;
for (i = 0; i < count; i++) {
/* get pointer to param structure */
mfu_param_path* param = &p[i];
/* initialize all fields */
mfu_param_path_init(param);
/* lookup the path */
uint64_t path_idx = start + i;
const char* path = paths[path_idx];
/* set param fields for path */
if (path != NULL) {
/* make a copy of original path */
param->orig = MFU_STRDUP(path);
/* get absolute path and remove ".", "..", consecutive "/",
* and trailing "/" characters */
param->path = mfu_path_strdup_abs_reduce_str(path);
/* get stat info for simplified path */
if (mfu_lstat(param->path, ¶m->path_stat) == 0) {
param->path_stat_valid = 1;
}
/* TODO: we use realpath below, which is nice since it takes out
* ".", "..", symlinks, and adds the absolute path, however, it
* fails if the file/directory does not already exist, which is
* often the case for dest path. */
/* resolve any symlinks */
char target[PATH_MAX];
if (realpath(path, target) != NULL) {
/* make a copy of resolved name */
param->target = MFU_STRDUP(target);
/* get stat info for resolved path */
if (mfu_lstat(param->target, ¶m->target_stat) == 0) {
param->target_stat_valid = 1;
}
}
/* add in bytes needed to pack this param */
bytes += (uint64_t) mfu_pack_param_size(param);
}
}
/* TODO: eventually it would be nice to leave this data distributed,
* however for now some tools expect all params to be defined */
/* allgather to get bytes on each process */
int* recvcounts = (int*) MFU_MALLOC(ranks * sizeof(int));
int* recvdispls = (int*) MFU_MALLOC(ranks * sizeof(int));
int sendcount = (int) bytes;
MPI_Allgather(&sendcount, 1, MPI_INT, recvcounts, 1, MPI_INT, MPI_COMM_WORLD);
/* compute displacements and total number of bytes that we'll receive */
uint64_t allbytes = 0;
int disp = 0;
for (i = 0; i < (uint64_t) ranks; i++) {
recvdispls[i] = disp;
disp += recvcounts[i];
allbytes += (uint64_t) recvcounts[i];
}
/* allocate memory for send and recv buffers */
char* sendbuf = MFU_MALLOC(bytes);
char* recvbuf = MFU_MALLOC(allbytes);
/* pack send buffer */
char* ptr = sendbuf;
for (i = 0; i < count; i++) {
mfu_pack_param(&ptr, &p[i]);
}
/* allgatherv to collect data */
MPI_Allgatherv(sendbuf, sendcount, MPI_BYTE, recvbuf, recvcounts, recvdispls, MPI_BYTE, MPI_COMM_WORLD);
/* unpack recv buffer into caller's params */
ptr = recvbuf;
for (i = 0; i < num; i++) {
mfu_unpack_param((const char**)(&ptr), ¶ms[i]);
}
/* Loop through the list of files &/or directories, and check the params
* struct to see if all of them are valid file names. If one is not, let
* the user know by printing a warning */
if (rank == 0) {
for (i = 0; i < num; i++) {
/* get pointer to param structure */
mfu_param_path* param = ¶ms[i];
if (param->path_stat_valid == 0) {
/* failed to find a file at this location, let user know (may be a typo) */
MFU_LOG(MFU_LOG_WARN, "Warning: `%s' does not exist", param->orig);
}
}
}
/* free message buffers */
mfu_free(&recvbuf);
mfu_free(&sendbuf);
/* free arrays for recv counts and displacements */
mfu_free(&recvdispls);
mfu_free(&recvcounts);
/* free temporary params */
mfu_param_path_free_list(count, p);
mfu_free(&p);
return;
}
/**
* Decode the operation code from a message on the distributed queue structure.
*/
DCOPY_operation_t* DCOPY_decode_operation(char* op)
{
DCOPY_operation_t* ret = (DCOPY_operation_t*) malloc(sizeof(DCOPY_operation_t));
if(sscanf(strtok(op, ":"), "%" SCNd64, &(ret->file_size)) != 1) {
MFU_LOG(MFU_LOG_ERR, "Could not decode file size attribute");
DCOPY_abort(EXIT_FAILURE);
}
if(sscanf(strtok(NULL, ":"), "%" SCNd64, &(ret->chunk)) != 1) {
MFU_LOG(MFU_LOG_ERR, "Could not decode chunk index attribute");
DCOPY_abort(EXIT_FAILURE);
}
if(sscanf(strtok(NULL, ":"), "%" SCNu16, &(ret->source_base_offset)) != 1) {
MFU_LOG(MFU_LOG_ERR, "Could not decode source base offset attribute");
DCOPY_abort(EXIT_FAILURE);
}
if(sscanf(strtok(NULL, ":"), "%d", (int*) &(ret->code)) != 1) {
MFU_LOG(MFU_LOG_ERR, "Could not decode stage code attribute");
DCOPY_abort(EXIT_FAILURE);
}
/* get number of characters in operand string */
int op_len;
char* str = strtok(NULL, ":");
if(sscanf(str, "%d", &op_len) != 1) {
MFU_LOG(MFU_LOG_ERR, "Could not decode operand string length");
DCOPY_abort(EXIT_FAILURE);
}
/* skip over digits and trailing ':' to get pointer to operand */
char* operand = str + strlen(str) + 1;
ret->operand = operand;
/* if operand ends with ':', then the dest_base_appendix is next */
int dest_base_exists = 0;
if(operand[op_len] == ':') {
dest_base_exists = 1;
}
/* NUL-terminate the operand string */
operand[op_len] = '\0';
ret->dest_base_appendix = NULL;
if(dest_base_exists) {
/* get pointer to first character of dest_base_len */
str = operand + op_len + 1;
/* tokenize length and scan it in */
int dest_len;
str = strtok(str, ":");
if(sscanf(str, "%d", &dest_len) != 1) {
MFU_LOG(MFU_LOG_ERR, "Could not decode destination base appendix string length");
DCOPY_abort(EXIT_FAILURE);
}
/* skip over digits and trailing ':' to get pointer to
* destination base, and NUL-terminate the string */
char* base = str + strlen(str) + 1;
ret->dest_base_appendix = base;
base[dest_len] = '\0';
}
/* get pointer to first character past source base path, if one exists */
const char* last_component = NULL;
if(ret->source_base_offset < op_len) {
last_component = ret->operand + ret->source_base_offset + 1;
}
/* build destination object name */
int written;
char dest_path_recursive[PATH_MAX];
if(ret->dest_base_appendix == NULL) {
if (last_component == NULL) {
written = snprintf(dest_path_recursive, sizeof(dest_path_recursive),
"%s", DCOPY_user_opts.dest_path);
} else {
written = snprintf(dest_path_recursive, sizeof(dest_path_recursive),
"%s/%s", DCOPY_user_opts.dest_path, last_component);
}
}
else {
if (last_component == NULL) {
written = snprintf(dest_path_recursive, sizeof(dest_path_recursive),
"%s/%s", DCOPY_user_opts.dest_path, ret->dest_base_appendix);
} else {
written = snprintf(dest_path_recursive, sizeof(dest_path_recursive),
"%s/%s/%s", DCOPY_user_opts.dest_path, ret->dest_base_appendix, last_component);
}
}
/* fail if we would have overwritten the buffer */
if((size_t)written >= sizeof(dest_path_recursive)) {
MFU_LOG(MFU_LOG_ERR, "Destination path buffer too small");
DCOPY_abort(EXIT_FAILURE);
}
/* record destination path in operation descriptor */
ret->dest_full_path = MFU_STRDUP(dest_path_recursive);
if(ret->dest_full_path == NULL) {
MFU_LOG(MFU_LOG_ERR, "Failed to allocate full destination path");
DCOPY_abort(EXIT_FAILURE);
}
return ret;
}
void mfu_param_path_check_archive(int numparams, mfu_param_path* srcparams, mfu_param_path destparam, int* valid)
{
/* TODO: need to parallize this, rather than have every rank do the test */
/* assume paths are valid */
*valid = 1;
/* count number of source paths that we can read */
int i;
int num_readable = 0;
for (i = 0; i < numparams; i++) {
char* path = srcparams[i].path;
if (mfu_access(path, R_OK) == 0) {
/* found one that we can read */
num_readable++;
} else {
/* not readable */
char* orig = srcparams[i].orig;
MFU_LOG(MFU_LOG_ERR, "Could not read '%s' errno=%d %s",
orig, errno, strerror(errno));
}
}
/* verify we have at least one valid source */
if (num_readable < 1) {
MFU_LOG(MFU_LOG_ERR, "At least one valid source must be specified");
*valid = 0;
goto bcast;
}
/* copy destination to user opts structure */
DTAR_user_opts.dest_path = MFU_STRDUP(dest_param.path);
/* check destination */
if (destparam.path_stat_valid) {
if (DTAR_rank == 0) {
MFU_LOG(MFU_LOG_WARN, "Destination target exists, we will overwrite");
}
} else {
/* compute path to parent of destination archive */
mfu_path* parent = mfu_path_from_str(destparam.path);
mfu_path_dirname(parent);
char* parent_str = mfu_path_strdup(parent);
mfu_path_delete(&parent);
/* check if parent is writable */
if (mfu_access(parent_str, W_OK) < 0) {
MFU_LOG(MFU_LOG_ERR, "Destination parent directory is not wriable: '%s' ",
parent_str);
*valid = 0;
mfu_free(&parent_str);
goto bcast;
}
mfu_free(&parent_str);
}
/* at this point, we know
* (1) destination doesn't exist
* (2) parent directory is writable
*/
bcast:
MPI_Bcast(valid, 1, MPI_INT, 0, MPI_COMM_WORLD);
if (! *valid) {
if (DTAR_rank == 0) {
MFU_LOG(MFU_LOG_ERR, "Exiting run");
}
MPI_Barrier(MPI_COMM_WORLD);
DTAR_exit(EXIT_FAILURE);
}
}
