Beispiel #1
0
/* same as above but comparison op sorts in reverse order */
int DTCMP_Str_create_descend(
    int chars,
    MPI_Datatype* type,
    DTCMP_Op* cmp)
{
    if (chars <= 0 || type == NULL || cmp == NULL) {
        return DTCMP_FAILURE;
    }

    MPI_Datatype dt;
    MPI_Type_contiguous(chars, MPI_CHAR, &dt);
    MPI_Type_commit(&dt);

    DTCMP_Op op;
    DTCMP_Op_create(dt, dtcmp_op_fn_strcmp_descend, &op);

    *type = dt;
    *cmp  = op;

    return DTCMP_SUCCESS;
}
Beispiel #2
0
static int sort_files_readdir(const char* sortfields, mfu_flist* pflist)
{
    /* get list from caller */
    mfu_flist flist = *pflist;

    /* create a new list as subset of original list */
    mfu_flist flist2 = mfu_flist_subset(flist);

    uint64_t incount = mfu_flist_size(flist);
    uint64_t chars   = mfu_flist_file_max_name(flist);

    /* create datatype for packed file list element */
    MPI_Datatype dt_sat;
    size_t bytes = mfu_flist_file_pack_size(flist);
    MPI_Type_contiguous((int)bytes, MPI_BYTE, &dt_sat);

    /* get our rank and the size of comm_world */
    int rank, ranks;
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &ranks);

    /* build type for file path */
    MPI_Datatype dt_filepath;
    MPI_Type_contiguous((int)chars, MPI_CHAR, &dt_filepath);
    MPI_Type_commit(&dt_filepath);

    /* build comparison op for filenames */
    DTCMP_Op op_filepath;
    if (DTCMP_Op_create(dt_filepath, my_strcmp, &op_filepath) != DTCMP_SUCCESS) {
        MFU_ABORT(1, "Failed to create sorting operation for filepath");
    }

    /* build comparison op for filenames */
    DTCMP_Op op_filepath_rev;
    if (DTCMP_Op_create(dt_filepath, my_strcmp_rev, &op_filepath_rev) != DTCMP_SUCCESS) {
        MFU_ABORT(1, "Failed to create reverse sorting operation for filepath");
    }

    /* TODO: process sort fields */
    const int MAXFIELDS = 1;
    MPI_Datatype types[MAXFIELDS];
    DTCMP_Op ops[MAXFIELDS];
    sort_field fields[MAXFIELDS];
    size_t lengths[MAXFIELDS];
    int nfields = 0;
    for (nfields = 0; nfields < MAXFIELDS; nfields++) {
        types[nfields]   = MPI_DATATYPE_NULL;
        ops[nfields]     = DTCMP_OP_NULL;
    }
    nfields = 0;
    char* sortfields_copy = MFU_STRDUP(sortfields);
    char* token = strtok(sortfields_copy, ",");
    while (token != NULL) {
        int valid = 1;
        if (strcmp(token, "name") == 0) {
            types[nfields]   = dt_filepath;
            ops[nfields]     = op_filepath;
            fields[nfields]  = FILENAME;
            lengths[nfields] = chars;
        }
        else if (strcmp(token, "-name") == 0) {
            types[nfields]   = dt_filepath;
            ops[nfields]     = op_filepath_rev;
            fields[nfields]  = FILENAME;
            lengths[nfields] = chars;
        }
        else {
            /* invalid token */
            valid = 0;
            if (rank == 0) {
                MFU_LOG(MFU_LOG_ERR, "Invalid sort field: %s\n", token);
            }
        }
        if (valid) {
            nfields++;
        }
        if (nfields > MAXFIELDS) {
            /* TODO: print warning if we have too many fields */
            break;
        }
        token = strtok(NULL, ",");
    }
    mfu_free(&sortfields_copy);

    /* build key type */
    MPI_Datatype dt_key;
    if (DTCMP_Type_create_series(nfields, types, &dt_key) != DTCMP_SUCCESS) {
        MFU_ABORT(1, "Failed to create type for key");
    }

    /* create sort op */
    DTCMP_Op op_key;
    if (DTCMP_Op_create_series(nfields, ops, &op_key) != DTCMP_SUCCESS) {
        MFU_ABORT(1, "Failed to create sorting operation for key");
    }

    /* build keysat type */
    MPI_Datatype dt_keysat, keysat_types[2];
    keysat_types[0] = dt_key;
    keysat_types[1] = dt_sat;
    if (DTCMP_Type_create_series(2, keysat_types, &dt_keysat) != DTCMP_SUCCESS) {
        MFU_ABORT(1, "Failed to create type for keysat");
    }

    /* get extent of key type */
    MPI_Aint key_lb, key_extent;
    MPI_Type_get_extent(dt_key, &key_lb, &key_extent);

    /* get extent of keysat type */
    MPI_Aint keysat_lb, keysat_extent;
    MPI_Type_get_extent(dt_keysat, &keysat_lb, &keysat_extent);

    /* get extent of sat type */
    MPI_Aint sat_lb, sat_extent;
    MPI_Type_get_extent(dt_sat, &sat_lb, &sat_extent);

    /* compute size of sort element and allocate buffer */
    size_t sortbufsize = (size_t)keysat_extent * incount;
    void* sortbuf = MFU_MALLOC(sortbufsize);

    /* copy data into sort elements */
    uint64_t idx = 0;
    char* sortptr = (char*) sortbuf;
    while (idx < incount) {
        /* copy in access time */
        int i;
        for (i = 0; i < nfields; i++) {
            if (fields[i] == FILENAME) {
                const char* name = mfu_flist_file_get_name(flist, idx);
                strcpy(sortptr, name);
            }
            sortptr += lengths[i];
        }

        /* pack file element */
        sortptr += mfu_flist_file_pack(sortptr, flist, idx);

        idx++;
    }

    /* sort data */
    void* outsortbuf;
    int outsortcount;
    DTCMP_Handle handle;
    int sort_rc = DTCMP_Sortz(
                      sortbuf, (int)incount, &outsortbuf, &outsortcount,
                      dt_key, dt_keysat, op_key, DTCMP_FLAG_NONE,
                      MPI_COMM_WORLD, &handle
                  );
    if (sort_rc != DTCMP_SUCCESS) {
        MFU_ABORT(1, "Failed to sort data");
    }

    /* step through sorted data filenames */
    idx = 0;
    sortptr = (char*) outsortbuf;
    while (idx < (uint64_t)outsortcount) {
        sortptr += key_extent;
        sortptr += mfu_flist_file_unpack(sortptr, flist2);
        idx++;
    }

    /* build summary of new list */
    mfu_flist_summarize(flist2);

    /* free memory */
    DTCMP_Free(&handle);

    /* free ops */
    DTCMP_Op_free(&op_key);
    DTCMP_Op_free(&op_filepath_rev);
    DTCMP_Op_free(&op_filepath);

    /* free types */
    MPI_Type_free(&dt_keysat);
    MPI_Type_free(&dt_key);
    MPI_Type_free(&dt_filepath);

    /* free input buffer holding sort elements */
    mfu_free(&sortbuf);

    /* free the satellite type */
    MPI_Type_free(&dt_sat);

    /* return new list and free old one */
    *pflist = flist2;
    mfu_flist_free(&flist);

    return MFU_SUCCESS;
}
Beispiel #3
0
/* initialize the sorting library */
int DTCMP_Init()
{
    /* increment the number of times init has been called */
    dtcmp_init_count++;

    /* initialize our values if init has not already been called */
    if (dtcmp_init_count == 1) {
        /* copy comm_self */
        MPI_Comm_dup(MPI_COMM_SELF, &dtcmp_comm_self);

        /* intialize our random seed */
        int rank;
        MPI_Comm_rank(MPI_COMM_WORLD, &rank);
        dtcmp_rand_seed = (unsigned) rank;

        /* set up a datatype for min/max/sum reduction */
        MPI_Type_contiguous(3, MPI_UINT64_T, &dtcmp_type_3uint64t);
        MPI_Type_commit(&dtcmp_type_3uint64t);

        /* set up our user-defined op for min/max/sum reduction,
         * just integer min/max/addition of non-negative values,
         * so assume this is commutative */
        int commutative = 1;
        MPI_Op_create(dtcmp_reducefn_uint64t_min_max_sum, commutative, &dtcmp_reduceop_mms_3uint64t);

        /* set up a datatype for max rand/count/rank reduction */
        MPI_Type_contiguous(3, MPI_INT, &dtcmp_type_3int);
        MPI_Type_commit(&dtcmp_type_3int);

        /* set up a reduction op to select the rank having a positive
         * count and the max random value (break ties with count then
         * rank), this is also commutative */
        MPI_Op_create(dtcmp_reducefn_randroot, commutative, &dtcmp_reduceop_randroot);

        /* setup predefined cmp handles */
        DTCMP_Op_create(MPI_SHORT, dtcmp_op_fn_short_ascend,  &DTCMP_OP_SHORT_ASCEND);
        DTCMP_Op_create(MPI_SHORT, dtcmp_op_fn_short_descend, &DTCMP_OP_SHORT_DESCEND);
        DTCMP_Op_create(MPI_INT, dtcmp_op_fn_int_ascend,  &DTCMP_OP_INT_ASCEND);
        DTCMP_Op_create(MPI_INT, dtcmp_op_fn_int_descend, &DTCMP_OP_INT_DESCEND);
        DTCMP_Op_create(MPI_LONG, dtcmp_op_fn_long_ascend,  &DTCMP_OP_LONG_ASCEND);
        DTCMP_Op_create(MPI_LONG, dtcmp_op_fn_long_descend, &DTCMP_OP_LONG_DESCEND);
        DTCMP_Op_create(MPI_LONG_LONG, dtcmp_op_fn_longlong_ascend,  &DTCMP_OP_LONGLONG_ASCEND);
        DTCMP_Op_create(MPI_LONG_LONG, dtcmp_op_fn_longlong_descend, &DTCMP_OP_LONGLONG_DESCEND);
        DTCMP_Op_create(MPI_UNSIGNED_SHORT, dtcmp_op_fn_unsignedshort_ascend,  &DTCMP_OP_UNSIGNEDSHORT_ASCEND);
        DTCMP_Op_create(MPI_UNSIGNED_SHORT, dtcmp_op_fn_unsignedshort_descend, &DTCMP_OP_UNSIGNEDSHORT_DESCEND);
        DTCMP_Op_create(MPI_UNSIGNED, dtcmp_op_fn_unsignedint_ascend,  &DTCMP_OP_UNSIGNEDINT_ASCEND);
        DTCMP_Op_create(MPI_UNSIGNED, dtcmp_op_fn_unsignedint_descend, &DTCMP_OP_UNSIGNEDINT_DESCEND);
        DTCMP_Op_create(MPI_UNSIGNED_LONG, dtcmp_op_fn_unsignedlong_ascend,  &DTCMP_OP_UNSIGNEDLONG_ASCEND);
        DTCMP_Op_create(MPI_UNSIGNED_LONG, dtcmp_op_fn_unsignedlong_descend, &DTCMP_OP_UNSIGNEDLONG_DESCEND);
        DTCMP_Op_create(MPI_UNSIGNED_LONG_LONG, dtcmp_op_fn_unsignedlonglong_ascend,  &DTCMP_OP_UNSIGNEDLONGLONG_ASCEND);
        DTCMP_Op_create(MPI_UNSIGNED_LONG_LONG, dtcmp_op_fn_unsignedlonglong_descend, &DTCMP_OP_UNSIGNEDLONGLONG_DESCEND);
        DTCMP_Op_create(MPI_INT8_T,  dtcmp_op_fn_int8t_ascend,   &DTCMP_OP_INT8T_ASCEND);
        DTCMP_Op_create(MPI_INT8_T,  dtcmp_op_fn_int8t_descend,  &DTCMP_OP_INT8T_DESCEND);
        DTCMP_Op_create(MPI_INT16_T, dtcmp_op_fn_int16t_ascend,  &DTCMP_OP_INT16T_ASCEND);
        DTCMP_Op_create(MPI_INT16_T, dtcmp_op_fn_int16t_descend, &DTCMP_OP_INT16T_DESCEND);
        DTCMP_Op_create(MPI_INT32_T, dtcmp_op_fn_int32t_ascend,  &DTCMP_OP_INT32T_ASCEND);
        DTCMP_Op_create(MPI_INT32_T, dtcmp_op_fn_int32t_descend, &DTCMP_OP_INT32T_DESCEND);
        DTCMP_Op_create(MPI_INT64_T, dtcmp_op_fn_int64t_ascend,  &DTCMP_OP_INT64T_ASCEND);
        DTCMP_Op_create(MPI_INT64_T, dtcmp_op_fn_int64t_descend, &DTCMP_OP_INT64T_DESCEND);
        DTCMP_Op_create(MPI_UINT8_T,  dtcmp_op_fn_uint8t_ascend,   &DTCMP_OP_UINT8T_ASCEND);
        DTCMP_Op_create(MPI_UINT8_T,  dtcmp_op_fn_uint8t_descend,  &DTCMP_OP_UINT8T_DESCEND);
        DTCMP_Op_create(MPI_UINT16_T, dtcmp_op_fn_uint16t_ascend,  &DTCMP_OP_UINT16T_ASCEND);
        DTCMP_Op_create(MPI_UINT16_T, dtcmp_op_fn_uint16t_descend, &DTCMP_OP_UINT16T_DESCEND);
        DTCMP_Op_create(MPI_UINT32_T, dtcmp_op_fn_uint32t_ascend,  &DTCMP_OP_UINT32T_ASCEND);
        DTCMP_Op_create(MPI_UINT32_T, dtcmp_op_fn_uint32t_descend, &DTCMP_OP_UINT32T_DESCEND);
        DTCMP_Op_create(MPI_UINT64_T, dtcmp_op_fn_uint64t_ascend,  &DTCMP_OP_UINT64T_ASCEND);
        DTCMP_Op_create(MPI_UINT64_T, dtcmp_op_fn_uint64t_descend, &DTCMP_OP_UINT64T_DESCEND);
        DTCMP_Op_create(MPI_FLOAT, dtcmp_op_fn_float_ascend,  &DTCMP_OP_FLOAT_ASCEND);
        DTCMP_Op_create(MPI_FLOAT, dtcmp_op_fn_float_descend, &DTCMP_OP_FLOAT_DESCEND);
        DTCMP_Op_create(MPI_DOUBLE, dtcmp_op_fn_double_ascend,  &DTCMP_OP_DOUBLE_ASCEND);
        DTCMP_Op_create(MPI_DOUBLE, dtcmp_op_fn_double_descend, &DTCMP_OP_DOUBLE_DESCEND);
        DTCMP_Op_create(MPI_LONG_DOUBLE, dtcmp_op_fn_longdouble_ascend,  &DTCMP_OP_LONGDOUBLE_ASCEND);
        DTCMP_Op_create(MPI_LONG_DOUBLE, dtcmp_op_fn_longdouble_descend, &DTCMP_OP_LONGDOUBLE_DESCEND);
    }

    return DTCMP_SUCCESS;
}