/** Call back given to initialize the dataset. */
static void walk_readdir_create(CIRCLE_handle* handle)
{
uint64_t i;
for (i = 0; i < CURRENT_NUM_DIRS; i++) {
const char* path = CURRENT_DIRS[i];
/* stat top level item */
struct stat st;
int status = mfu_lstat(path, &st);
if (status != 0) {
/* TODO: print error */
return;
}
/* increment our item count */
reduce_items++;
/* record item info */
mfu_flist_insert_stat(CURRENT_LIST, path, st.st_mode, &st);
/* recurse into directory */
if (S_ISDIR(st.st_mode)) {
walk_readdir_process_dir(path, handle);
}
}
return;
}
static void DTAR_write_header(struct archive* ar, uint64_t idx, uint64_t offset)
{
/* allocate and entry for this item */
struct archive_entry* entry = archive_entry_new();
/* get file name for this item */
/* fill up entry, FIXME: the uglyness of removing leading slash */
const char* fname = mfu_flist_file_get_name(DTAR_flist, idx);
archive_entry_copy_pathname(entry, &fname[1]);
if (DTAR_user_opts.preserve) {
struct archive* source = archive_read_disk_new();
archive_read_disk_set_standard_lookup(source);
int fd = open(fname, O_RDONLY);
if (archive_read_disk_entry_from_file(source, entry, fd, NULL) != ARCHIVE_OK) {
MFU_LOG(MFU_LOG_ERR, "archive_read_disk_entry_from_file(): %s", archive_error_string(ar));
}
archive_read_free(source);
close(fd);
} else {
/* TODO: read stat info from mfu_flist */
struct stat stbuf;
mfu_lstat(fname, &stbuf);
archive_entry_copy_stat(entry, &stbuf);
/* set user name of owner */
const char* uname = mfu_flist_file_get_username(DTAR_flist, idx);
archive_entry_set_uname(entry, uname);
/* set group name */
const char* gname = mfu_flist_file_get_groupname(DTAR_flist, idx);
archive_entry_set_gname(entry, gname);
}
/* TODO: Seems to be a bug here potentially leading to corrupted
* archive files. archive_write_free also writes two blocks of
* NULL bytes at the end of an archive file, however, each rank
* will have a different view of the length of the file, so one
* rank may write its NULL blocks over top of the actual data
* written by another rank */
/* write entry info to archive */
struct archive* dest = archive_write_new();
archive_write_set_format_pax(dest);
if (archive_write_open_fd(dest, DTAR_writer.fd_tar) != ARCHIVE_OK) {
MFU_LOG(MFU_LOG_ERR, "archive_write_open_fd(): %s", archive_error_string(ar));
}
/* seek to offset in tar archive for this file */
lseek(DTAR_writer.fd_tar, offset, SEEK_SET);
/* write header for this item */
if (archive_write_header(dest, entry) != ARCHIVE_OK) {
MFU_LOG(MFU_LOG_ERR, "archive_write_header(): %s", archive_error_string(ar));
}
archive_entry_free(entry);
archive_write_free(dest);
}
/* Given an input file list, stat each file and enqueue details
* in output file list, skip entries excluded by skip function
* and skip args */
void mfu_flist_stat(
mfu_flist input_flist,
mfu_flist flist,
mfu_flist_skip_fn skip_fn,
void *skip_args)
{
flist_t* file_list = (flist_t*)flist;
/* we will stat all items in output list, so set detail to 1 */
file_list->detail = 1;
/* get user data if needed */
if (file_list->have_users == 0) {
mfu_flist_usrgrp_get_users(flist);
}
/* get groups data if needed */
if (file_list->have_groups == 0) {
mfu_flist_usrgrp_get_groups(flist);
}
/* step through each item in input list and stat it */
uint64_t idx;
uint64_t size = mfu_flist_size(input_flist);
for (idx = 0; idx < size; idx++) {
/* get name of item */
const char* name = mfu_flist_file_get_name(input_flist, idx);
/* check whether we should skip this item */
if (skip_fn != NULL && skip_fn(name, skip_args)) {
/* skip this file, don't include it in new list */
MFU_LOG(MFU_LOG_INFO, "skip %s");
continue;
}
/* stat the item */
struct stat st;
int status = mfu_lstat(name, &st);
if (status != 0) {
MFU_LOG(MFU_LOG_ERR, "mfu_lstat() failed: `%s' rc=%d (errno=%d %s)", name, status, errno, strerror(errno));
continue;
}
/* insert item into output list */
mfu_flist_insert_stat(flist, name, st.st_mode, &st);
}
/* compute global summary */
mfu_flist_summarize(flist);
}
/** Callback given to process the dataset. */
static void walk_stat_process(CIRCLE_handle* handle)
{
/* get path from queue */
char path[CIRCLE_MAX_STRING_LEN];
handle->dequeue(path);
/* stat item */
struct stat st;
int status = mfu_lstat(path, &st);
if (status != 0) {
/* print error */
return;
}
/* increment our item count */
reduce_items++;
/* TODO: filter items by stat info */
if (REMOVE_FILES && !S_ISDIR(st.st_mode)) {
mfu_unlink(path);
} else {
/* record info for item in list */
mfu_flist_insert_stat(CURRENT_LIST, path, st.st_mode, &st);
}
/* recurse into directory */
if (S_ISDIR(st.st_mode)) {
/* before more processing check if SET_DIR_PERMS is set,
* and set usr read and execute bits if need be */
if (SET_DIR_PERMS) {
/* use masks to check if usr_r and usr_x are already on */
long usr_r_mask = 1 << 8;
long usr_x_mask = 1 << 6;
/* turn on the usr read & execute bits if they are not already on*/
if (!((usr_r_mask & st.st_mode) && (usr_x_mask & st.st_mode))) {
st.st_mode |= S_IRUSR;
st.st_mode |= S_IXUSR;
mfu_chmod(path, st.st_mode);
}
}
/* TODO: check that we can recurse into directory */
walk_stat_process_dir(path, handle);
}
return;
}
void DTAR_write_header(struct archive *ar, uint64_t idx, uint64_t offset)
{
const char * fname = mfu_flist_file_get_name(DTAR_flist, idx);
/* fill up entry, FIXME: the uglyness of removing leading slash */
struct archive_entry *entry = archive_entry_new();
archive_entry_copy_pathname(entry, &fname[1]);
if (DTAR_user_opts.preserve) {
struct archive * source = archive_read_disk_new();
archive_read_disk_set_standard_lookup(source);
int fd = open(fname, O_RDONLY);
if (archive_read_disk_entry_from_file(source, entry, fd, NULL) != ARCHIVE_OK) {
MFU_LOG(MFU_LOG_ERR, "archive_read_disk_entry_from_file(): %s", archive_error_string(ar));
}
archive_read_free(source);
} else {
/* read stat info from mfu_flist */
struct stat stbuf;
mfu_lstat(fname, &stbuf);
archive_entry_copy_stat(entry, &stbuf);
const char* uname = mfu_flist_file_get_username(DTAR_flist, idx);
archive_entry_set_uname(entry, uname);
const char* gname = mfu_flist_file_get_groupname(DTAR_flist, idx);
archive_entry_set_gname(entry, gname);
}
/* write entry info to archive */
struct archive* dest = archive_write_new();
archive_write_set_format_pax(dest);
if (archive_write_open_fd(dest, DTAR_writer.fd_tar) != ARCHIVE_OK) {
MFU_LOG(MFU_LOG_ERR, "archive_write_open_fd(): %s", archive_error_string(ar));
}
lseek64(DTAR_writer.fd_tar, offset, SEEK_SET);
if (archive_write_header(dest, entry) != ARCHIVE_OK) {
MFU_LOG(MFU_LOG_ERR, "archive_write_header(): %s", archive_error_string(ar));
}
archive_entry_free(entry);
archive_write_free(dest);
}
static void walk_readdir_process_dir(const char* dir, CIRCLE_handle* handle)
{
/* TODO: may need to try these functions multiple times */
DIR* dirp = mfu_opendir(dir);
/* if there is a permissions error and the usr read & execute are being turned
* on when walk_stat=0 then catch the permissions error and turn the bits on */
if (dirp == NULL) {
if (errno == EACCES && SET_DIR_PERMS) {
struct stat st;
mfu_lstat(dir, &st);
// turn on the usr read & execute bits
st.st_mode |= S_IRUSR;
st.st_mode |= S_IXUSR;
mfu_chmod(dir, st.st_mode);
dirp = mfu_opendir(dir);
if (dirp == NULL) {
if (errno == EACCES) {
MFU_LOG(MFU_LOG_ERR, "Failed to open directory with opendir: `%s' (errno=%d %s)", dir, errno, strerror(errno));
}
}
}
}
if (! dirp) {
/* TODO: print error */
}
else {
/* Read all directory entries */
while (1) {
/* read next directory entry */
struct dirent* entry = mfu_readdir(dirp);
if (entry == NULL) {
break;
}
/* process component, unless it's "." or ".." */
char* name = entry->d_name;
if ((strncmp(name, ".", 2)) && (strncmp(name, "..", 3))) {
/* <dir> + '/' + <name> + '/0' */
char newpath[CIRCLE_MAX_STRING_LEN];
size_t len = strlen(dir) + 1 + strlen(name) + 1;
if (len < sizeof(newpath)) {
/* build full path to item */
strcpy(newpath, dir);
strcat(newpath, "/");
strcat(newpath, name);
#ifdef _DIRENT_HAVE_D_TYPE
/* record info for item */
mode_t mode;
int have_mode = 0;
if (entry->d_type != DT_UNKNOWN) {
/* unlink files here if remove option is on,
* and dtype is known without a stat */
if (REMOVE_FILES && (entry->d_type != DT_DIR)) {
mfu_unlink(newpath);
} else {
/* we can read object type from directory entry */
have_mode = 1;
mode = DTTOIF(entry->d_type);
mfu_flist_insert_stat(CURRENT_LIST, newpath, mode, NULL);
}
}
else {
/* type is unknown, we need to stat it */
struct stat st;
int status = mfu_lstat(newpath, &st);
if (status == 0) {
have_mode = 1;
mode = st.st_mode;
/* unlink files here if remove option is on,
* and stat was necessary to get type */
if (REMOVE_FILES && !S_ISDIR(st.st_mode)) {
mfu_unlink(newpath);
} else {
mfu_flist_insert_stat(CURRENT_LIST, newpath, mode, &st);
}
}
else {
/* error */
}
}
/* increment our item count */
reduce_items++;
/* recurse into directories */
if (have_mode && S_ISDIR(mode)) {
handle->enqueue(newpath);
}
#endif
}
else {
/* TODO: print error in correct format */
/* name is too long */
MFU_LOG(MFU_LOG_ERR, "Path name is too long: %lu chars exceeds limit %lu", len, sizeof(newpath));
}
}
}
}
mfu_closedir(dirp);
return;
}
void mfu_param_path_set_all(uint64_t num, const char** paths, mfu_param_path* params)
{
/* get our rank and number of ranks */
int rank, ranks;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &ranks);
/* determine number we should look up */
uint64_t count = num / (uint64_t) ranks;
uint64_t extra = num - count * (uint64_t) ranks;
if (rank < (int) extra) {
/* procs whose rank is less than extra each
* handle one extra param than those whose
* rank is equal or greater than extra */
count++;
}
/* determine our starting point */
uint64_t start = 0;
if (rank < (int) extra) {
/* for these procs, count is one more than procs with ranks >= extra */
start = (uint64_t)rank * count;
} else {
/* for these procs, count is one less than procs with ranks < extra */
start = extra * (count + 1) + ((uint64_t)rank - extra) * count;
}
/* TODO: allocate temporary params */
mfu_param_path* p = MFU_MALLOC(count * sizeof(mfu_param_path));
/* track maximum path length */
uint64_t bytes = 0;
/* process each path we're responsible for */
uint64_t i;
for (i = 0; i < count; i++) {
/* get pointer to param structure */
mfu_param_path* param = &p[i];
/* initialize all fields */
mfu_param_path_init(param);
/* lookup the path */
uint64_t path_idx = start + i;
const char* path = paths[path_idx];
/* set param fields for path */
if (path != NULL) {
/* make a copy of original path */
param->orig = MFU_STRDUP(path);
/* get absolute path and remove ".", "..", consecutive "/",
* and trailing "/" characters */
param->path = mfu_path_strdup_abs_reduce_str(path);
/* get stat info for simplified path */
if (mfu_lstat(param->path, ¶m->path_stat) == 0) {
param->path_stat_valid = 1;
}
/* TODO: we use realpath below, which is nice since it takes out
* ".", "..", symlinks, and adds the absolute path, however, it
* fails if the file/directory does not already exist, which is
* often the case for dest path. */
/* resolve any symlinks */
char target[PATH_MAX];
if (realpath(path, target) != NULL) {
/* make a copy of resolved name */
param->target = MFU_STRDUP(target);
/* get stat info for resolved path */
if (mfu_lstat(param->target, ¶m->target_stat) == 0) {
param->target_stat_valid = 1;
}
}
/* add in bytes needed to pack this param */
bytes += (uint64_t) mfu_pack_param_size(param);
}
}
/* TODO: eventually it would be nice to leave this data distributed,
* however for now some tools expect all params to be defined */
/* allgather to get bytes on each process */
int* recvcounts = (int*) MFU_MALLOC(ranks * sizeof(int));
int* recvdispls = (int*) MFU_MALLOC(ranks * sizeof(int));
int sendcount = (int) bytes;
MPI_Allgather(&sendcount, 1, MPI_INT, recvcounts, 1, MPI_INT, MPI_COMM_WORLD);
/* compute displacements and total number of bytes that we'll receive */
uint64_t allbytes = 0;
int disp = 0;
for (i = 0; i < (uint64_t) ranks; i++) {
recvdispls[i] = disp;
disp += recvcounts[i];
allbytes += (uint64_t) recvcounts[i];
}
/* allocate memory for send and recv buffers */
char* sendbuf = MFU_MALLOC(bytes);
char* recvbuf = MFU_MALLOC(allbytes);
/* pack send buffer */
char* ptr = sendbuf;
for (i = 0; i < count; i++) {
mfu_pack_param(&ptr, &p[i]);
}
/* allgatherv to collect data */
MPI_Allgatherv(sendbuf, sendcount, MPI_BYTE, recvbuf, recvcounts, recvdispls, MPI_BYTE, MPI_COMM_WORLD);
/* unpack recv buffer into caller's params */
ptr = recvbuf;
for (i = 0; i < num; i++) {
mfu_unpack_param((const char**)(&ptr), ¶ms[i]);
}
/* Loop through the list of files &/or directories, and check the params
* struct to see if all of them are valid file names. If one is not, let
* the user know by printing a warning */
if (rank == 0) {
for (i = 0; i < num; i++) {
/* get pointer to param structure */
mfu_param_path* param = ¶ms[i];
if (param->path_stat_valid == 0) {
/* failed to find a file at this location, let user know (may be a typo) */
MFU_LOG(MFU_LOG_WARN, "Warning: `%s' does not exist", param->orig);
}
}
}
/* free message buffers */
mfu_free(&recvbuf);
mfu_free(&sendbuf);
/* free arrays for recv counts and displacements */
mfu_free(&recvdispls);
mfu_free(&recvcounts);
/* free temporary params */
mfu_param_path_free_list(count, p);
mfu_free(&p);
return;
}
