/* TODO: condionally set setuid and setgid bits? */
void DCOPY_copy_permissions(
const struct stat64* statbuf,
const char* dest_path)
{
/* change mode */
if(! S_ISLNK(statbuf->st_mode)) {
if(mfu_chmod(dest_path, statbuf->st_mode) != 0) {
MFU_LOG(MFU_LOG_ERR, "Failed to change permissions on %s chmod() errno=%d %s",
dest_path, errno, strerror(errno)
);
}
}
return;
}
/** Callback given to process the dataset. */
static void walk_stat_process(CIRCLE_handle* handle)
{
/* get path from queue */
char path[CIRCLE_MAX_STRING_LEN];
handle->dequeue(path);
/* stat item */
struct stat st;
int status = mfu_lstat(path, &st);
if (status != 0) {
/* print error */
return;
}
/* increment our item count */
reduce_items++;
/* TODO: filter items by stat info */
if (REMOVE_FILES && !S_ISDIR(st.st_mode)) {
mfu_unlink(path);
} else {
/* record info for item in list */
mfu_flist_insert_stat(CURRENT_LIST, path, st.st_mode, &st);
}
/* recurse into directory */
if (S_ISDIR(st.st_mode)) {
/* before more processing check if SET_DIR_PERMS is set,
* and set usr read and execute bits if need be */
if (SET_DIR_PERMS) {
/* use masks to check if usr_r and usr_x are already on */
long usr_r_mask = 1 << 8;
long usr_x_mask = 1 << 6;
/* turn on the usr read & execute bits if they are not already on*/
if (!((usr_r_mask & st.st_mode) && (usr_x_mask & st.st_mode))) {
st.st_mode |= S_IRUSR;
st.st_mode |= S_IXUSR;
mfu_chmod(path, st.st_mode);
}
}
/* TODO: check that we can recurse into directory */
walk_stat_process_dir(path, handle);
}
return;
}
/* TODO: condionally set setuid and setgid bits? */
void DCOPY_copy_permissions(
mfu_flist flist,
uint64_t idx,
const char* dest_path)
{
/* get mode and type */
mfu_filetype type = mfu_flist_file_get_type(flist, idx);
mode_t mode = (mode_t) mfu_flist_file_get_mode(flist, idx);
/* change mode */
if(type != MFU_TYPE_LINK) {
if(mfu_chmod(dest_path, mode) != 0) {
MFU_LOG(MFU_LOG_ERR, "Failed to change permissions on %s chmod() errno=%d %s",
dest_path, errno, strerror(errno)
);
}
}
return;
}
static void walk_readdir_process_dir(const char* dir, CIRCLE_handle* handle)
{
/* TODO: may need to try these functions multiple times */
DIR* dirp = mfu_opendir(dir);
/* if there is a permissions error and the usr read & execute are being turned
* on when walk_stat=0 then catch the permissions error and turn the bits on */
if (dirp == NULL) {
if (errno == EACCES && SET_DIR_PERMS) {
struct stat st;
mfu_lstat(dir, &st);
// turn on the usr read & execute bits
st.st_mode |= S_IRUSR;
st.st_mode |= S_IXUSR;
mfu_chmod(dir, st.st_mode);
dirp = mfu_opendir(dir);
if (dirp == NULL) {
if (errno == EACCES) {
MFU_LOG(MFU_LOG_ERR, "Failed to open directory with opendir: `%s' (errno=%d %s)", dir, errno, strerror(errno));
}
}
}
}
if (! dirp) {
/* TODO: print error */
}
else {
/* Read all directory entries */
while (1) {
/* read next directory entry */
struct dirent* entry = mfu_readdir(dirp);
if (entry == NULL) {
break;
}
/* process component, unless it's "." or ".." */
char* name = entry->d_name;
if ((strncmp(name, ".", 2)) && (strncmp(name, "..", 3))) {
/* <dir> + '/' + <name> + '/0' */
char newpath[CIRCLE_MAX_STRING_LEN];
size_t len = strlen(dir) + 1 + strlen(name) + 1;
if (len < sizeof(newpath)) {
/* build full path to item */
strcpy(newpath, dir);
strcat(newpath, "/");
strcat(newpath, name);
#ifdef _DIRENT_HAVE_D_TYPE
/* record info for item */
mode_t mode;
int have_mode = 0;
if (entry->d_type != DT_UNKNOWN) {
/* unlink files here if remove option is on,
* and dtype is known without a stat */
if (REMOVE_FILES && (entry->d_type != DT_DIR)) {
mfu_unlink(newpath);
} else {
/* we can read object type from directory entry */
have_mode = 1;
mode = DTTOIF(entry->d_type);
mfu_flist_insert_stat(CURRENT_LIST, newpath, mode, NULL);
}
}
else {
/* type is unknown, we need to stat it */
struct stat st;
int status = mfu_lstat(newpath, &st);
if (status == 0) {
have_mode = 1;
mode = st.st_mode;
/* unlink files here if remove option is on,
* and stat was necessary to get type */
if (REMOVE_FILES && !S_ISDIR(st.st_mode)) {
mfu_unlink(newpath);
} else {
mfu_flist_insert_stat(CURRENT_LIST, newpath, mode, &st);
}
}
else {
/* error */
}
}
/* increment our item count */
reduce_items++;
/* recurse into directories */
if (have_mode && S_ISDIR(mode)) {
handle->enqueue(newpath);
}
#endif
}
else {
/* TODO: print error in correct format */
/* name is too long */
MFU_LOG(MFU_LOG_ERR, "Path name is too long: %lu chars exceeds limit %lu", len, sizeof(newpath));
}
}
}
}
mfu_closedir(dirp);
return;
}
int main(int argc, char* argv[])
{
MPI_Init(&argc, &argv);
mfu_init();
/* get our rank and number of ranks in the job */
int rank, ranks;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &ranks);
/* pointer to mfu_walk_opts */
mfu_walk_opts_t* walk_opts = mfu_walk_opts_new();
uint64_t idx;
int option_index = 0;
int usage = 0;
int report = 0;
unsigned int numpaths = 0;
mfu_param_path* paths = NULL;
unsigned long long bytes;
/* verbose by default */
mfu_debug_level = MFU_LOG_VERBOSE;
/* default to 1MB stripe size, stripe across all OSTs, and all files are candidates */
int stripes = -1;
uint64_t stripe_size = 1048576;
uint64_t min_size = 0;
static struct option long_options[] = {
{"count", 1, 0, 'c'},
{"size", 1, 0, 's'},
{"minsize", 1, 0, 'm'},
{"report", 0, 0, 'r'},
{"progress", 1, 0, 'P'},
{"verbose", 0, 0, 'v'},
{"quiet", 0, 0, 'q'},
{"help", 0, 0, 'h'},
{0, 0, 0, 0}
};
while (1) {
int c = getopt_long(argc, argv, "c:s:m:rvqh",
long_options, &option_index);
if (c == -1) {
break;
}
switch (c) {
case 'c':
/* stripe count */
stripes = atoi(optarg);
break;
case 's':
/* stripe size in bytes */
if (mfu_abtoull(optarg, &bytes) != MFU_SUCCESS) {
if (rank == 0) {
MFU_LOG(MFU_LOG_ERR, "Failed to parse stripe size: %s", optarg);
}
MPI_Abort(MPI_COMM_WORLD, 1);
}
stripe_size = (uint64_t)bytes;
break;
case 'm':
/* min file size in bytes */
if (mfu_abtoull(optarg, &bytes) != MFU_SUCCESS) {
if (rank == 0) {
MFU_LOG(MFU_LOG_ERR, "Failed to parse minimum file size: %s", optarg);
}
MPI_Abort(MPI_COMM_WORLD, 1);
}
min_size = (uint64_t)bytes;
break;
case 'r':
/* report striping info */
report = 1;
break;
case 'P':
mfu_progress_timeout = atoi(optarg);
break;
case 'v':
mfu_debug_level = MFU_LOG_VERBOSE;
break;
case 'q':
mfu_debug_level = MFU_LOG_NONE;
break;
case 'h':
/* display usage */
usage = 1;
break;
case '?':
/* display usage */
usage = 1;
break;
default:
if (rank == 0) {
printf("?? getopt returned character code 0%o ??\n", c);
}
}
}
/* check that we got a valid progress value */
if (mfu_progress_timeout < 0) {
if (rank == 0) {
MFU_LOG(MFU_LOG_ERR, "Seconds in --progress must be non-negative: %d invalid", mfu_progress_timeout);
}
usage = 1;
}
/* paths to walk come after the options */
if (optind < argc) {
/* determine number of paths specified by user */
numpaths = argc - optind;
/* allocate space for each path */
paths = (mfu_param_path*) MFU_MALLOC((size_t)numpaths * sizeof(mfu_param_path));
/* process each path */
char** p = &argv[optind];
mfu_param_path_set_all((uint64_t)numpaths, (const char**)p, paths);
optind += numpaths;
} else {
usage = 1;
}
/* if we need to print usage, print it and exit */
if (usage) {
if (rank == 0) {
print_usage();
}
mfu_finalize();
MPI_Finalize();
return 1;
}
/* nothing to do if lustre support is disabled */
#ifndef LUSTRE_SUPPORT
if (rank == 0) {
MFU_LOG(MFU_LOG_ERR, "Lustre support is disabled.");
}
MPI_Abort(MPI_COMM_WORLD, 1);
#endif
/* stripe count must be -1 for all available or greater than 0 */
if (stripes < -1) {
if (rank == 0) {
MFU_LOG(MFU_LOG_ERR, "Stripe count must be -1 for all servers, 0 for lustre file system default, or a positive value");
}
MPI_Abort(MPI_COMM_WORLD, 1);
}
/* lustre requires stripe sizes to be aligned */
if (stripe_size > 0 && stripe_size % 65536 != 0) {
if (rank == 0) {
MFU_LOG(MFU_LOG_ERR, "Stripe size must be a multiple of 65536");
}
MPI_Abort(MPI_COMM_WORLD, 1);
}
/* TODO: verify that source / target are on Lustre */
/* walk list of input paths and stat as we walk */
mfu_flist flist = mfu_flist_new();
mfu_flist_walk_param_paths(numpaths, paths, walk_opts, flist);
/* filter down our list to files which don't meet our striping requirements */
mfu_flist filtered = filter_list(flist, stripes, stripe_size, min_size, &create_prog_count_total, &stripe_prog_bytes_total);
mfu_flist_free(&flist);
MPI_Barrier(MPI_COMM_WORLD);
/* report the file size and stripe count of all files we found */
if (report) {
/* report the files in our filtered list */
stripe_info_report(filtered);
/* free the paths and our list */
mfu_flist_free(&filtered);
mfu_param_path_free_all(numpaths, paths);
mfu_free(&paths);
/* finalize */
mfu_finalize();
MPI_Finalize();
return 0;
}
/* generate a global suffix for our temp files and have each node check it's list */
char suffix[8];
uint64_t retry;
/* seed our random number generator */
srand(time(NULL));
/* keep trying to make a valid random suffix...*/
do {
uint64_t attempt = 0;
/* make rank 0 responsible for generating a random suffix */
if (rank == 0) {
generate_suffix(suffix, sizeof(suffix));
}
/* broadcast the random suffix to all ranks */
MPI_Bcast(suffix, sizeof(suffix), MPI_CHAR, 0, MPI_COMM_WORLD);
/* check that the file doesn't already exist */
uint64_t size = mfu_flist_size(filtered);
for (idx = 0; idx < size; idx++) {
char temp_path[PATH_MAX];
strcpy(temp_path, mfu_flist_file_get_name(filtered, idx));
strcat(temp_path, suffix);
if(!mfu_access(temp_path, F_OK)) {
/* the file already exists */
attempt = 1;
break;
}
}
/* do a reduce to figure out if a rank has a file collision */
MPI_Allreduce(&attempt, &retry, 1, MPI_UINT64_T, MPI_MAX, MPI_COMM_WORLD);
} while(retry != 0);
/* initialize progress messages while creating files */
create_prog_count = 0;
create_prog = mfu_progress_start(mfu_progress_timeout, 1, MPI_COMM_WORLD, create_progress_fn);
/* create new files so we can restripe */
uint64_t size = mfu_flist_size(filtered);
for (idx = 0; idx < size; idx++) {
char temp_path[PATH_MAX];
strcpy(temp_path, mfu_flist_file_get_name(filtered, idx));
strcat(temp_path, suffix);
/* create a striped file at the temp file path */
mfu_stripe_set(temp_path, stripe_size, stripes);
/* update our status for file create progress */
create_prog_count++;
mfu_progress_update(&create_prog_count, create_prog);
}
/* finalize file create progress messages */
mfu_progress_complete(&create_prog_count, &create_prog);
MPI_Barrier(MPI_COMM_WORLD);
/* initialize progress messages while copying data */
stripe_prog_bytes = 0;
stripe_prog = mfu_progress_start(mfu_progress_timeout, 1, MPI_COMM_WORLD, stripe_progress_fn);
/* found a suffix, now we need to break our files into chunks based on stripe size */
mfu_file_chunk* file_chunks = mfu_file_chunk_list_alloc(filtered, stripe_size);
mfu_file_chunk* p = file_chunks;
while (p != NULL) {
/* build path to temp file */
char temp_path[PATH_MAX];
strcpy(temp_path, p->name);
strcat(temp_path, suffix);
/* write each chunk in our list */
write_file_chunk(p, temp_path);
/* move on to next file chunk */
p = p->next;
}
mfu_file_chunk_list_free(&file_chunks);
/* finalize progress messages */
mfu_progress_complete(&stripe_prog_bytes, &stripe_prog);
MPI_Barrier(MPI_COMM_WORLD);
/* remove input file and rename temp file */
for (idx = 0; idx < size; idx++) {
/* build path to temp file */
const char *in_path = mfu_flist_file_get_name(filtered, idx);
char out_path[PATH_MAX];
strcpy(out_path, in_path);
strcat(out_path, suffix);
/* change the mode of the newly restriped file to be the same as the old one */
mode_t mode = (mode_t) mfu_flist_file_get_mode(filtered, idx);
if (mfu_chmod(out_path, mode) != 0) {
MFU_LOG(MFU_LOG_ERR, "Failed to chmod file %s (%s)", out_path, strerror(errno));
MPI_Abort(MPI_COMM_WORLD, 1);
}
/* rename the new, restriped file to the old name */
if (rename(out_path, in_path) != 0) {
MFU_LOG(MFU_LOG_ERR, "Failed to rename file %s to %s", out_path, in_path);
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
/* wait for everyone to finish */
MPI_Barrier(MPI_COMM_WORLD);
/* free the walk options */
mfu_walk_opts_delete(&walk_opts);
/* free filtered list, path parameters */
mfu_flist_free(&filtered);
mfu_param_path_free_all(numpaths, paths);
mfu_free(&paths);
mfu_finalize();
MPI_Finalize();
return 0;
}
