/* free object referenced by comparison operation handle */
int DTCMP_Op_free(DTCMP_Op* cmp)
{
/* check that we're initialized */
if (dtcmp_init_count <= 0) {
return DTCMP_FAILURE;
}
if (cmp != NULL && *cmp != DTCMP_OP_NULL) {
dtcmp_op_handle_t* c = (dtcmp_op_handle_t*)(*cmp);
MPI_Type_free(&(c->key));
if (c->series != DTCMP_OP_NULL) {
DTCMP_Op_free(&(c->series));
}
free(*cmp);
*cmp = DTCMP_OP_NULL;
return DTCMP_SUCCESS;
} else {
return DTCMP_FAILURE;
}
}
/* for each depth, sort files by filename and then remove, to test
* whether it matters to limit the number of directories each process
* has to reference (e.g., locking) */
static void remove_sort(mfu_flist list, uint64_t* rmcount)
{
/* bail out if total count is 0 */
uint64_t all_count = mfu_flist_global_size(list);
if (all_count == 0) {
return;
}
/* get maximum file name and number of items */
int chars = (int) mfu_flist_file_max_name(list);
uint64_t my_count = mfu_flist_size(list);
/* create key datatype (filename) and comparison op */
MPI_Datatype dt_key;
DTCMP_Op op_str;
DTCMP_Str_create_ascend(chars, &dt_key, &op_str);
/* create keysat datatype (filename + type) */
MPI_Datatype types[2], dt_keysat;
types[0] = dt_key;
types[1] = MPI_CHAR;
DTCMP_Type_create_series(2, types, &dt_keysat);
/* allocate send buffer */
int sendcount = (int) my_count;
size_t sendbufsize = (size_t)(sendcount * (chars + 1));
char* sendbuf = (char*) MFU_MALLOC(sendbufsize);
/* copy data into buffer */
char* ptr = sendbuf;
uint64_t idx;
for (idx = 0; idx < my_count; idx++) {
/* encode the filename first */
const char* name = mfu_flist_file_get_name(list, idx);
strcpy(ptr, name);
ptr += chars;
/* last character encodes item type */
mfu_filetype type = mfu_flist_file_get_type(list, idx);
if (type == MFU_TYPE_DIR) {
ptr[0] = 'd';
}
else if (type == MFU_TYPE_FILE || type == MFU_TYPE_LINK) {
ptr[0] = 'f';
}
else {
ptr[0] = 'u';
}
ptr++;
}
/* sort items */
void* recvbuf;
int recvcount;
DTCMP_Handle handle;
DTCMP_Sortz(
sendbuf, sendcount, &recvbuf, &recvcount,
dt_key, dt_keysat, op_str, DTCMP_FLAG_NONE, MPI_COMM_WORLD, &handle
);
/* delete data */
int delcount = 0;
ptr = (char*)recvbuf;
while (delcount < recvcount) {
/* get item name */
char* name = ptr;
ptr += chars;
/* get item type */
char type = ptr[0];
ptr++;
/* delete item */
remove_type(type, name);
delcount++;
}
/* record number of items we deleted */
*rmcount = (uint64_t) delcount;
/* free output data */
DTCMP_Free(&handle);
/* free our send buffer */
mfu_free(&sendbuf);
/* free key comparison operation */
DTCMP_Op_free(&op_str);
/* free datatypes */
MPI_Type_free(&dt_keysat);
MPI_Type_free(&dt_key);
return;
}
static int sort_files_readdir(const char* sortfields, mfu_flist* pflist)
{
/* get list from caller */
mfu_flist flist = *pflist;
/* create a new list as subset of original list */
mfu_flist flist2 = mfu_flist_subset(flist);
uint64_t incount = mfu_flist_size(flist);
uint64_t chars = mfu_flist_file_max_name(flist);
/* create datatype for packed file list element */
MPI_Datatype dt_sat;
size_t bytes = mfu_flist_file_pack_size(flist);
MPI_Type_contiguous((int)bytes, MPI_BYTE, &dt_sat);
/* get our rank and the size of comm_world */
int rank, ranks;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &ranks);
/* build type for file path */
MPI_Datatype dt_filepath;
MPI_Type_contiguous((int)chars, MPI_CHAR, &dt_filepath);
MPI_Type_commit(&dt_filepath);
/* build comparison op for filenames */
DTCMP_Op op_filepath;
if (DTCMP_Op_create(dt_filepath, my_strcmp, &op_filepath) != DTCMP_SUCCESS) {
MFU_ABORT(1, "Failed to create sorting operation for filepath");
}
/* build comparison op for filenames */
DTCMP_Op op_filepath_rev;
if (DTCMP_Op_create(dt_filepath, my_strcmp_rev, &op_filepath_rev) != DTCMP_SUCCESS) {
MFU_ABORT(1, "Failed to create reverse sorting operation for filepath");
}
/* TODO: process sort fields */
const int MAXFIELDS = 1;
MPI_Datatype types[MAXFIELDS];
DTCMP_Op ops[MAXFIELDS];
sort_field fields[MAXFIELDS];
size_t lengths[MAXFIELDS];
int nfields = 0;
for (nfields = 0; nfields < MAXFIELDS; nfields++) {
types[nfields] = MPI_DATATYPE_NULL;
ops[nfields] = DTCMP_OP_NULL;
}
nfields = 0;
char* sortfields_copy = MFU_STRDUP(sortfields);
char* token = strtok(sortfields_copy, ",");
while (token != NULL) {
int valid = 1;
if (strcmp(token, "name") == 0) {
types[nfields] = dt_filepath;
ops[nfields] = op_filepath;
fields[nfields] = FILENAME;
lengths[nfields] = chars;
}
else if (strcmp(token, "-name") == 0) {
types[nfields] = dt_filepath;
ops[nfields] = op_filepath_rev;
fields[nfields] = FILENAME;
lengths[nfields] = chars;
}
else {
/* invalid token */
valid = 0;
if (rank == 0) {
MFU_LOG(MFU_LOG_ERR, "Invalid sort field: %s\n", token);
}
}
if (valid) {
nfields++;
}
if (nfields > MAXFIELDS) {
/* TODO: print warning if we have too many fields */
break;
}
token = strtok(NULL, ",");
}
mfu_free(&sortfields_copy);
/* build key type */
MPI_Datatype dt_key;
if (DTCMP_Type_create_series(nfields, types, &dt_key) != DTCMP_SUCCESS) {
MFU_ABORT(1, "Failed to create type for key");
}
/* create sort op */
DTCMP_Op op_key;
if (DTCMP_Op_create_series(nfields, ops, &op_key) != DTCMP_SUCCESS) {
MFU_ABORT(1, "Failed to create sorting operation for key");
}
/* build keysat type */
MPI_Datatype dt_keysat, keysat_types[2];
keysat_types[0] = dt_key;
keysat_types[1] = dt_sat;
if (DTCMP_Type_create_series(2, keysat_types, &dt_keysat) != DTCMP_SUCCESS) {
MFU_ABORT(1, "Failed to create type for keysat");
}
/* get extent of key type */
MPI_Aint key_lb, key_extent;
MPI_Type_get_extent(dt_key, &key_lb, &key_extent);
/* get extent of keysat type */
MPI_Aint keysat_lb, keysat_extent;
MPI_Type_get_extent(dt_keysat, &keysat_lb, &keysat_extent);
/* get extent of sat type */
MPI_Aint sat_lb, sat_extent;
MPI_Type_get_extent(dt_sat, &sat_lb, &sat_extent);
/* compute size of sort element and allocate buffer */
size_t sortbufsize = (size_t)keysat_extent * incount;
void* sortbuf = MFU_MALLOC(sortbufsize);
/* copy data into sort elements */
uint64_t idx = 0;
char* sortptr = (char*) sortbuf;
while (idx < incount) {
/* copy in access time */
int i;
for (i = 0; i < nfields; i++) {
if (fields[i] == FILENAME) {
const char* name = mfu_flist_file_get_name(flist, idx);
strcpy(sortptr, name);
}
sortptr += lengths[i];
}
/* pack file element */
sortptr += mfu_flist_file_pack(sortptr, flist, idx);
idx++;
}
/* sort data */
void* outsortbuf;
int outsortcount;
DTCMP_Handle handle;
int sort_rc = DTCMP_Sortz(
sortbuf, (int)incount, &outsortbuf, &outsortcount,
dt_key, dt_keysat, op_key, DTCMP_FLAG_NONE,
MPI_COMM_WORLD, &handle
);
if (sort_rc != DTCMP_SUCCESS) {
MFU_ABORT(1, "Failed to sort data");
}
/* step through sorted data filenames */
idx = 0;
sortptr = (char*) outsortbuf;
while (idx < (uint64_t)outsortcount) {
sortptr += key_extent;
sortptr += mfu_flist_file_unpack(sortptr, flist2);
idx++;
}
/* build summary of new list */
mfu_flist_summarize(flist2);
/* free memory */
DTCMP_Free(&handle);
/* free ops */
DTCMP_Op_free(&op_key);
DTCMP_Op_free(&op_filepath_rev);
DTCMP_Op_free(&op_filepath);
/* free types */
MPI_Type_free(&dt_keysat);
MPI_Type_free(&dt_key);
MPI_Type_free(&dt_filepath);
/* free input buffer holding sort elements */
mfu_free(&sortbuf);
/* free the satellite type */
MPI_Type_free(&dt_sat);
/* return new list and free old one */
*pflist = flist2;
mfu_flist_free(&flist);
return MFU_SUCCESS;
}
/* finalize the sorting library and set static values back
* to their pre-init state */
int DTCMP_Finalize()
{
/* if we're going to 0, free everything off */
if (dtcmp_init_count == 1) {
/* free off predefined cmp handles */
DTCMP_Op_free(&DTCMP_OP_LONGDOUBLE_DESCEND);
DTCMP_Op_free(&DTCMP_OP_LONGDOUBLE_ASCEND);
DTCMP_Op_free(&DTCMP_OP_DOUBLE_DESCEND);
DTCMP_Op_free(&DTCMP_OP_DOUBLE_ASCEND);
DTCMP_Op_free(&DTCMP_OP_FLOAT_DESCEND);
DTCMP_Op_free(&DTCMP_OP_FLOAT_ASCEND);
DTCMP_Op_free(&DTCMP_OP_UINT64T_DESCEND);
DTCMP_Op_free(&DTCMP_OP_UINT64T_ASCEND);
DTCMP_Op_free(&DTCMP_OP_UINT32T_DESCEND);
DTCMP_Op_free(&DTCMP_OP_UINT32T_ASCEND);
DTCMP_Op_free(&DTCMP_OP_UINT16T_DESCEND);
DTCMP_Op_free(&DTCMP_OP_UINT16T_ASCEND);
DTCMP_Op_free(&DTCMP_OP_UINT8T_DESCEND);
DTCMP_Op_free(&DTCMP_OP_UINT8T_ASCEND);
DTCMP_Op_free(&DTCMP_OP_INT64T_DESCEND);
DTCMP_Op_free(&DTCMP_OP_INT64T_ASCEND);
DTCMP_Op_free(&DTCMP_OP_INT32T_DESCEND);
DTCMP_Op_free(&DTCMP_OP_INT32T_ASCEND);
DTCMP_Op_free(&DTCMP_OP_INT16T_DESCEND);
DTCMP_Op_free(&DTCMP_OP_INT16T_ASCEND);
DTCMP_Op_free(&DTCMP_OP_INT8T_DESCEND);
DTCMP_Op_free(&DTCMP_OP_INT8T_ASCEND);
DTCMP_Op_free(&DTCMP_OP_UNSIGNEDLONGLONG_DESCEND);
DTCMP_Op_free(&DTCMP_OP_UNSIGNEDLONGLONG_ASCEND);
DTCMP_Op_free(&DTCMP_OP_UNSIGNEDLONG_DESCEND);
DTCMP_Op_free(&DTCMP_OP_UNSIGNEDLONG_ASCEND);
DTCMP_Op_free(&DTCMP_OP_UNSIGNEDINT_DESCEND);
DTCMP_Op_free(&DTCMP_OP_UNSIGNEDINT_ASCEND);
DTCMP_Op_free(&DTCMP_OP_UNSIGNEDSHORT_DESCEND);
DTCMP_Op_free(&DTCMP_OP_UNSIGNEDSHORT_ASCEND);
DTCMP_Op_free(&DTCMP_OP_LONGLONG_DESCEND);
DTCMP_Op_free(&DTCMP_OP_LONGLONG_ASCEND);
DTCMP_Op_free(&DTCMP_OP_LONG_DESCEND);
DTCMP_Op_free(&DTCMP_OP_LONG_ASCEND);
DTCMP_Op_free(&DTCMP_OP_INT_DESCEND);
DTCMP_Op_free(&DTCMP_OP_INT_ASCEND);
DTCMP_Op_free(&DTCMP_OP_SHORT_DESCEND);
DTCMP_Op_free(&DTCMP_OP_SHORT_ASCEND);
if (dtcmp_reduceop_randroot != MPI_OP_NULL) {
MPI_Op_free(&dtcmp_reduceop_randroot);
dtcmp_reduceop_randroot = MPI_OP_NULL;
}
if (dtcmp_reduceop_mms_3uint64t != MPI_OP_NULL) {
MPI_Op_free(&dtcmp_reduceop_mms_3uint64t);
dtcmp_reduceop_mms_3uint64t = MPI_OP_NULL;
}
if (dtcmp_type_3int != MPI_DATATYPE_NULL) {
MPI_Type_free(&dtcmp_type_3int);
dtcmp_type_3int = MPI_DATATYPE_NULL;
}
if (dtcmp_type_3uint64t != MPI_DATATYPE_NULL) {
MPI_Type_free(&dtcmp_type_3uint64t);
dtcmp_type_3uint64t = MPI_DATATYPE_NULL;
}
/* free our copy of comm_self */
if (dtcmp_comm_self != MPI_COMM_NULL) {
MPI_Comm_free(&dtcmp_comm_self);
dtcmp_comm_self = MPI_COMM_NULL;
}
}
/* decrement our reference count for number of init calls */
dtcmp_init_count--;
return DTCMP_SUCCESS;
}
/* free the comparison operation */
static void mtcmp_cmp_fini(DTCMP_Op* cmp)
{
DTCMP_Op_free(cmp);
}
