static void DTAR_write_header(struct archive* ar, uint64_t idx, uint64_t offset)
{
/* allocate and entry for this item */
struct archive_entry* entry = archive_entry_new();
/* get file name for this item */
/* fill up entry, FIXME: the uglyness of removing leading slash */
const char* fname = mfu_flist_file_get_name(DTAR_flist, idx);
archive_entry_copy_pathname(entry, &fname[1]);
if (DTAR_user_opts.preserve) {
struct archive* source = archive_read_disk_new();
archive_read_disk_set_standard_lookup(source);
int fd = open(fname, O_RDONLY);
if (archive_read_disk_entry_from_file(source, entry, fd, NULL) != ARCHIVE_OK) {
MFU_LOG(MFU_LOG_ERR, "archive_read_disk_entry_from_file(): %s", archive_error_string(ar));
}
archive_read_free(source);
close(fd);
} else {
/* TODO: read stat info from mfu_flist */
struct stat stbuf;
mfu_lstat(fname, &stbuf);
archive_entry_copy_stat(entry, &stbuf);
/* set user name of owner */
const char* uname = mfu_flist_file_get_username(DTAR_flist, idx);
archive_entry_set_uname(entry, uname);
/* set group name */
const char* gname = mfu_flist_file_get_groupname(DTAR_flist, idx);
archive_entry_set_gname(entry, gname);
}
/* TODO: Seems to be a bug here potentially leading to corrupted
* archive files. archive_write_free also writes two blocks of
* NULL bytes at the end of an archive file, however, each rank
* will have a different view of the length of the file, so one
* rank may write its NULL blocks over top of the actual data
* written by another rank */
/* write entry info to archive */
struct archive* dest = archive_write_new();
archive_write_set_format_pax(dest);
if (archive_write_open_fd(dest, DTAR_writer.fd_tar) != ARCHIVE_OK) {
MFU_LOG(MFU_LOG_ERR, "archive_write_open_fd(): %s", archive_error_string(ar));
}
/* seek to offset in tar archive for this file */
lseek(DTAR_writer.fd_tar, offset, SEEK_SET);
/* write header for this item */
if (archive_write_header(dest, entry) != ARCHIVE_OK) {
MFU_LOG(MFU_LOG_ERR, "archive_write_header(): %s", archive_error_string(ar));
}
archive_entry_free(entry);
archive_write_free(dest);
}
void DTAR_write_header(struct archive *ar, uint64_t idx, uint64_t offset)
{
const char * fname = mfu_flist_file_get_name(DTAR_flist, idx);
/* fill up entry, FIXME: the uglyness of removing leading slash */
struct archive_entry *entry = archive_entry_new();
archive_entry_copy_pathname(entry, &fname[1]);
if (DTAR_user_opts.preserve) {
struct archive * source = archive_read_disk_new();
archive_read_disk_set_standard_lookup(source);
int fd = open(fname, O_RDONLY);
if (archive_read_disk_entry_from_file(source, entry, fd, NULL) != ARCHIVE_OK) {
MFU_LOG(MFU_LOG_ERR, "archive_read_disk_entry_from_file(): %s", archive_error_string(ar));
}
archive_read_free(source);
} else {
/* read stat info from mfu_flist */
struct stat stbuf;
mfu_lstat(fname, &stbuf);
archive_entry_copy_stat(entry, &stbuf);
const char* uname = mfu_flist_file_get_username(DTAR_flist, idx);
archive_entry_set_uname(entry, uname);
const char* gname = mfu_flist_file_get_groupname(DTAR_flist, idx);
archive_entry_set_gname(entry, gname);
}
/* write entry info to archive */
struct archive* dest = archive_write_new();
archive_write_set_format_pax(dest);
if (archive_write_open_fd(dest, DTAR_writer.fd_tar) != ARCHIVE_OK) {
MFU_LOG(MFU_LOG_ERR, "archive_write_open_fd(): %s", archive_error_string(ar));
}
lseek64(DTAR_writer.fd_tar, offset, SEEK_SET);
if (archive_write_header(dest, entry) != ARCHIVE_OK) {
MFU_LOG(MFU_LOG_ERR, "archive_write_header(): %s", archive_error_string(ar));
}
archive_entry_free(entry);
archive_write_free(dest);
}
static int sort_files_stat(const char* sortfields, mfu_flist* pflist)
{
/* get list from caller */
mfu_flist flist = *pflist;
/* create a new list as subset of original list */
mfu_flist flist2 = mfu_flist_subset(flist);
uint64_t incount = mfu_flist_size(flist);
uint64_t chars = mfu_flist_file_max_name(flist);
uint64_t chars_user = mfu_flist_user_max_name(flist);
uint64_t chars_group = mfu_flist_group_max_name(flist);
/* create datatype for packed file list element */
MPI_Datatype dt_sat;
size_t bytes = mfu_flist_file_pack_size(flist);
MPI_Type_contiguous((int)bytes, MPI_BYTE, &dt_sat);
/* get our rank and the size of comm_world */
int rank, ranks;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &ranks);
/* build type for file path */
MPI_Datatype dt_filepath, dt_user, dt_group;
MPI_Type_contiguous((int)chars, MPI_CHAR, &dt_filepath);
MPI_Type_contiguous((int)chars_user, MPI_CHAR, &dt_user);
MPI_Type_contiguous((int)chars_group, MPI_CHAR, &dt_group);
MPI_Type_commit(&dt_filepath);
MPI_Type_commit(&dt_user);
MPI_Type_commit(&dt_group);
/* build comparison op for filenames */
DTCMP_Op op_filepath, op_user, op_group;
if (DTCMP_Op_create(dt_filepath, my_strcmp, &op_filepath) != DTCMP_SUCCESS) {
MFU_ABORT(1, "Failed to create sorting operation for filepath");
}
if (DTCMP_Op_create(dt_user, my_strcmp, &op_user) != DTCMP_SUCCESS) {
MFU_ABORT(1, "Failed to create sorting operation for username");
}
if (DTCMP_Op_create(dt_group, my_strcmp, &op_group) != DTCMP_SUCCESS) {
MFU_ABORT(1, "Failed to create sorting operation for groupname");
}
/* build comparison op for filenames */
DTCMP_Op op_filepath_rev, op_user_rev, op_group_rev;
if (DTCMP_Op_create(dt_filepath, my_strcmp_rev, &op_filepath_rev) != DTCMP_SUCCESS) {
MFU_ABORT(1, "Failed to create reverse sorting operation for groupname");
}
if (DTCMP_Op_create(dt_user, my_strcmp_rev, &op_user_rev) != DTCMP_SUCCESS) {
MFU_ABORT(1, "Failed to create reverse sorting operation for groupname");
}
if (DTCMP_Op_create(dt_group, my_strcmp_rev, &op_group_rev) != DTCMP_SUCCESS) {
MFU_ABORT(1, "Failed to create reverse sorting operation for groupname");
}
/* TODO: process sort fields */
const int MAXFIELDS = 7;
MPI_Datatype types[MAXFIELDS];
DTCMP_Op ops[MAXFIELDS];
sort_field fields[MAXFIELDS];
size_t lengths[MAXFIELDS];
int nfields = 0;
for (nfields = 0; nfields < MAXFIELDS; nfields++) {
types[nfields] = MPI_DATATYPE_NULL;
ops[nfields] = DTCMP_OP_NULL;
}
nfields = 0;
char* sortfields_copy = MFU_STRDUP(sortfields);
char* token = strtok(sortfields_copy, ",");
while (token != NULL) {
int valid = 1;
if (strcmp(token, "name") == 0) {
types[nfields] = dt_filepath;
ops[nfields] = op_filepath;
fields[nfields] = FILENAME;
lengths[nfields] = chars;
}
else if (strcmp(token, "-name") == 0) {
types[nfields] = dt_filepath;
ops[nfields] = op_filepath_rev;
fields[nfields] = FILENAME;
lengths[nfields] = chars;
}
else if (strcmp(token, "user") == 0) {
types[nfields] = dt_user;
ops[nfields] = op_user;
fields[nfields] = USERNAME;
lengths[nfields] = chars_user;
}
else if (strcmp(token, "-user") == 0) {
types[nfields] = dt_user;
ops[nfields] = op_user_rev;
fields[nfields] = USERNAME;
lengths[nfields] = chars_user;
}
else if (strcmp(token, "group") == 0) {
types[nfields] = dt_group;
ops[nfields] = op_group;
fields[nfields] = GROUPNAME;
lengths[nfields] = chars_group;
}
else if (strcmp(token, "-group") == 0) {
types[nfields] = dt_group;
ops[nfields] = op_group_rev;
fields[nfields] = GROUPNAME;
lengths[nfields] = chars_group;
}
else if (strcmp(token, "uid") == 0) {
types[nfields] = MPI_UINT32_T;
ops[nfields] = DTCMP_OP_UINT32T_ASCEND;
fields[nfields] = USERID;
lengths[nfields] = 4;
}
else if (strcmp(token, "-uid") == 0) {
types[nfields] = MPI_UINT32_T;
ops[nfields] = DTCMP_OP_UINT32T_DESCEND;
fields[nfields] = USERID;
lengths[nfields] = 4;
}
else if (strcmp(token, "gid") == 0) {
types[nfields] = MPI_UINT32_T;
ops[nfields] = DTCMP_OP_UINT32T_ASCEND;
fields[nfields] = GROUPID;
lengths[nfields] = 4;
}
else if (strcmp(token, "-gid") == 0) {
types[nfields] = MPI_UINT32_T;
ops[nfields] = DTCMP_OP_UINT32T_DESCEND;
fields[nfields] = GROUPID;
lengths[nfields] = 4;
}
else if (strcmp(token, "atime") == 0) {
types[nfields] = MPI_UINT32_T;
ops[nfields] = DTCMP_OP_UINT32T_ASCEND;
fields[nfields] = ATIME;
lengths[nfields] = 4;
}
else if (strcmp(token, "-atime") == 0) {
types[nfields] = MPI_UINT32_T;
ops[nfields] = DTCMP_OP_UINT32T_DESCEND;
fields[nfields] = ATIME;
lengths[nfields] = 4;
}
else if (strcmp(token, "mtime") == 0) {
types[nfields] = MPI_UINT32_T;
ops[nfields] = DTCMP_OP_UINT32T_ASCEND;
fields[nfields] = MTIME;
lengths[nfields] = 4;
}
else if (strcmp(token, "-mtime") == 0) {
types[nfields] = MPI_UINT32_T;
ops[nfields] = DTCMP_OP_UINT32T_DESCEND;
fields[nfields] = MTIME;
lengths[nfields] = 4;
}
else if (strcmp(token, "ctime") == 0) {
types[nfields] = MPI_UINT32_T;
ops[nfields] = DTCMP_OP_UINT32T_ASCEND;
fields[nfields] = CTIME;
lengths[nfields] = 4;
}
else if (strcmp(token, "-ctime") == 0) {
types[nfields] = MPI_UINT32_T;
ops[nfields] = DTCMP_OP_UINT32T_DESCEND;
fields[nfields] = CTIME;
lengths[nfields] = 4;
}
else if (strcmp(token, "size") == 0) {
types[nfields] = MPI_UINT64_T;
ops[nfields] = DTCMP_OP_UINT64T_ASCEND;
fields[nfields] = FILESIZE;
lengths[nfields] = 8;
}
else if (strcmp(token, "-size") == 0) {
types[nfields] = MPI_UINT64_T;
ops[nfields] = DTCMP_OP_UINT64T_DESCEND;
fields[nfields] = FILESIZE;
lengths[nfields] = 8;
}
else {
/* invalid token */
valid = 0;
if (rank == 0) {
MFU_LOG(MFU_LOG_ERR, "Invalid sort field: %s\n", token);
}
}
if (valid) {
nfields++;
}
if (nfields > MAXFIELDS) {
/* TODO: print warning if we have too many fields */
break;
}
token = strtok(NULL, ",");
}
mfu_free(&sortfields_copy);
/* build key type */
MPI_Datatype dt_key;
if (DTCMP_Type_create_series(nfields, types, &dt_key) != DTCMP_SUCCESS) {
MFU_ABORT(1, "Failed to create key type");
}
/* create op to sort by access time, then filename */
DTCMP_Op op_key;
if (DTCMP_Op_create_series(nfields, ops, &op_key) != DTCMP_SUCCESS) {
MFU_ABORT(1, "Failed to create sorting operation for key");
}
/* build keysat type */
MPI_Datatype dt_keysat, keysat_types[2];
keysat_types[0] = dt_key;
keysat_types[1] = dt_sat;
if (DTCMP_Type_create_series(2, keysat_types, &dt_keysat) != DTCMP_SUCCESS) {
MFU_ABORT(1, "Failed to create keysat type");
}
/* get extent of key type */
MPI_Aint key_lb, key_extent;
MPI_Type_get_extent(dt_key, &key_lb, &key_extent);
/* get extent of keysat type */
MPI_Aint keysat_lb, keysat_extent;
MPI_Type_get_extent(dt_keysat, &keysat_lb, &keysat_extent);
/* get extent of sat type */
MPI_Aint stat_lb, stat_extent;
MPI_Type_get_extent(dt_sat, &stat_lb, &stat_extent);
/* compute size of sort element and allocate buffer */
size_t sortbufsize = (size_t)keysat_extent * incount;
void* sortbuf = MFU_MALLOC(sortbufsize);
/* copy data into sort elements */
uint64_t idx = 0;
char* sortptr = (char*) sortbuf;
while (idx < incount) {
/* copy in access time */
int i;
for (i = 0; i < nfields; i++) {
if (fields[i] == FILENAME) {
const char* name = mfu_flist_file_get_name(flist, idx);
strcpy(sortptr, name);
}
else if (fields[i] == USERNAME) {
const char* name = mfu_flist_file_get_username(flist, idx);
strcpy(sortptr, name);
}
else if (fields[i] == GROUPNAME) {
const char* name = mfu_flist_file_get_groupname(flist, idx);
strcpy(sortptr, name);
}
else if (fields[i] == USERID) {
uint32_t val32 = (uint32_t) mfu_flist_file_get_uid(flist, idx);
memcpy(sortptr, &val32, 4);
}
else if (fields[i] == GROUPID) {
uint32_t val32 = (uint32_t) mfu_flist_file_get_gid(flist, idx);
memcpy(sortptr, &val32, 4);
}
else if (fields[i] == ATIME) {
uint32_t val32 = (uint32_t) mfu_flist_file_get_atime(flist, idx);
memcpy(sortptr, &val32, 4);
}
else if (fields[i] == MTIME) {
uint32_t val32 = (uint32_t) mfu_flist_file_get_mtime(flist, idx);
memcpy(sortptr, &val32, 4);
}
else if (fields[i] == CTIME) {
uint32_t val32 = (uint32_t) mfu_flist_file_get_ctime(flist, idx);
memcpy(sortptr, &val32, 4);
}
else if (fields[i] == FILESIZE) {
uint64_t val64 = mfu_flist_file_get_size(flist, idx);
memcpy(sortptr, &val64, 8);
}
sortptr += lengths[i];
}
/* pack file element */
sortptr += mfu_flist_file_pack(sortptr, flist, idx);
idx++;
}
/* sort data */
void* outsortbuf;
int outsortcount;
DTCMP_Handle handle;
int sort_rc = DTCMP_Sortz(
sortbuf, (int)incount, &outsortbuf, &outsortcount,
dt_key, dt_keysat, op_key, DTCMP_FLAG_NONE,
MPI_COMM_WORLD, &handle
);
if (sort_rc != DTCMP_SUCCESS) {
MFU_ABORT(1, "Failed to sort data");
}
/* step through sorted data filenames */
idx = 0;
sortptr = (char*) outsortbuf;
while (idx < (uint64_t)outsortcount) {
sortptr += key_extent;
sortptr += mfu_flist_file_unpack(sortptr, flist2);
idx++;
}
/* build summary of new list */
mfu_flist_summarize(flist2);
/* free memory */
DTCMP_Free(&handle);
/* free ops */
DTCMP_Op_free(&op_key);
DTCMP_Op_free(&op_group_rev);
DTCMP_Op_free(&op_user_rev);
DTCMP_Op_free(&op_filepath_rev);
DTCMP_Op_free(&op_group);
DTCMP_Op_free(&op_user);
DTCMP_Op_free(&op_filepath);
/* free types */
MPI_Type_free(&dt_keysat);
MPI_Type_free(&dt_key);
MPI_Type_free(&dt_group);
MPI_Type_free(&dt_user);
MPI_Type_free(&dt_filepath);
/* free input buffer holding sort elements */
mfu_free(&sortbuf);
/* free the satellite type */
MPI_Type_free(&dt_sat);
/* return new list and free old one */
*pflist = flist2;
mfu_flist_free(&flist);
return MFU_SUCCESS;
}
/* print information about a file given the index and rank (used in print_files) */
static void print_file(mfu_flist flist, uint64_t idx)
{
/* store types as strings for print_file */
char type_str_unknown[] = "UNK";
char type_str_dir[] = "DIR";
char type_str_file[] = "REG";
char type_str_link[] = "LNK";
/* get filename */
const char* file = mfu_flist_file_get_name(flist, idx);
if (mfu_flist_have_detail(flist)) {
/* get mode */
mode_t mode = (mode_t) mfu_flist_file_get_mode(flist, idx);
//uint32_t uid = (uint32_t) mfu_flist_file_get_uid(flist, idx);
//uint32_t gid = (uint32_t) mfu_flist_file_get_gid(flist, idx);
uint64_t acc = mfu_flist_file_get_atime(flist, idx);
uint64_t mod = mfu_flist_file_get_mtime(flist, idx);
uint64_t cre = mfu_flist_file_get_ctime(flist, idx);
uint64_t size = mfu_flist_file_get_size(flist, idx);
const char* username = mfu_flist_file_get_username(flist, idx);
const char* groupname = mfu_flist_file_get_groupname(flist, idx);
char access_s[30];
char modify_s[30];
char create_s[30];
time_t access_t = (time_t) acc;
time_t modify_t = (time_t) mod;
time_t create_t = (time_t) cre;
size_t access_rc = strftime(access_s, sizeof(access_s) - 1, "%FT%T", localtime(&access_t));
//size_t modify_rc = strftime(modify_s, sizeof(modify_s) - 1, "%FT%T", localtime(&modify_t));
size_t modify_rc = strftime(modify_s, sizeof(modify_s) - 1, "%b %e %Y %H:%M", localtime(&modify_t));
size_t create_rc = strftime(create_s, sizeof(create_s) - 1, "%FT%T", localtime(&create_t));
if (access_rc == 0 || modify_rc == 0 || create_rc == 0) {
/* error */
access_s[0] = '\0';
modify_s[0] = '\0';
create_s[0] = '\0';
}
char mode_format[11];
mfu_format_mode(mode, mode_format);
double size_tmp;
const char* size_units;
mfu_format_bytes(size, &size_tmp, &size_units);
printf("%s %s %s %7.3f %2s %s %s\n",
mode_format, username, groupname,
size_tmp, size_units, modify_s, file
);
#if 0
printf("%s %s %s A%s M%s C%s %lu %s\n",
mode_format, username, groupname,
access_s, modify_s, create_s, (unsigned long)size, file
);
printf("Mode=%lx(%s) UID=%d(%s) GUI=%d(%s) Access=%s Modify=%s Create=%s Size=%lu File=%s\n",
(unsigned long)mode, mode_format, uid, username, gid, groupname,
access_s, modify_s, create_s, (unsigned long)size, file
);
#endif
}
else {
/* get type */
mfu_filetype type = mfu_flist_file_get_type(flist, idx);
char* type_str = type_str_unknown;
if (type == MFU_TYPE_DIR) {
type_str = type_str_dir;
}
else if (type == MFU_TYPE_FILE) {
type_str = type_str_file;
}
else if (type == MFU_TYPE_LINK) {
type_str = type_str_link;
}
printf("Type=%s File=%s\n",
type_str, file
);
}
}
