Exemple #1
0
/* given an input list, split items into separate lists depending
 * on their depth, returns number of levels, minimum depth, and
 * array of lists as output */
void mfu_flist_array_by_depth(
    mfu_flist srclist,
    int* outlevels,
    int* outmin,
    mfu_flist** outlists)
{
    /* check that our pointers are valid */
    if (outlevels == NULL || outmin == NULL || outlists == NULL) {
        return;
    }

    /* initialize return values */
    *outlevels = 0;
    *outmin    = -1;
    *outlists  = NULL;

    /* get total file count */
    uint64_t total = mfu_flist_global_size(srclist);
    if (total == 0) {
        return;
    }

    /* get min and max depths, determine number of levels,
     * allocate array of lists */
    int min = mfu_flist_min_depth(srclist);
    int max = mfu_flist_max_depth(srclist);
    int levels = max - min + 1;
    mfu_flist* lists = (mfu_flist*) MFU_MALLOC((size_t)levels * sizeof(mfu_flist));

    /* create a list for each level */
    int i;
    for (i = 0; i < levels; i++) {
        lists[i] = mfu_flist_subset(srclist);
    }

    /* copy each item from source list to its corresponding level */
    uint64_t idx = 0;
    uint64_t size = mfu_flist_size(srclist);
    while (idx < size) {
        int depth = mfu_flist_file_get_depth(srclist, idx);
        int depth_index = depth - min;
        mfu_flist dstlist = lists[depth_index];
        mfu_flist_file_copy(srclist, idx, dstlist);
        idx++;
    }

    /* summarize each list */
    for (i = 0; i < levels; i++) {
        mfu_flist_summarize(lists[i]);
    }

    /* set return parameters */
    *outlevels = levels;
    *outmin    = min;
    *outlists  = lists;

    return;
}
Exemple #2
0
/* filter the list of files down based on the current stripe size and stripe count */
static mfu_flist filter_list(mfu_flist list, int stripe_count, uint64_t stripe_size, uint64_t min_size, uint64_t* total_count, uint64_t* total_size)
{
    /* initialize counters for file and byte count */
    uint64_t my_count = 0;
    uint64_t my_size  = 0;

    /* this is going to be a subset of the full file list */
    mfu_flist filtered = mfu_flist_subset(list);

    uint64_t idx;
    uint64_t size = mfu_flist_size(list);
    for (idx = 0; idx < size; idx++) {
        /* we only care about regular files */
        mfu_filetype type = mfu_flist_file_get_type(list, idx);
        if (type == MFU_TYPE_FILE) {
            /* if our file is below the minimum file size, skip it */
            uint64_t filesize = mfu_flist_file_get_size(list, idx);
            if (filesize < min_size) {
                continue;
            }

            const char* in_path = mfu_flist_file_get_name(list, idx);
            uint64_t curr_stripe_size = 0;
            uint64_t curr_stripe_count = 0;

            /*
             * attempt to get striping info,
             * skip the file if we can't get the striping info we seek
             */
            if (mfu_stripe_get(in_path, &curr_stripe_size, &curr_stripe_count) != 0) {
                continue;
            }

            /* TODO: this should probably be better */
            /* if the current stripe size or stripe count doesn't match, then a restripe the file */
            if (curr_stripe_count != stripe_count || curr_stripe_size != stripe_size) {
                mfu_flist_file_copy(list, idx, filtered);

                /* increment file count and add file size to our running total */
                my_count += 1;
                my_size  += filesize;
            }
        }
    }

    /* summarize and return the new list */
    mfu_flist_summarize(filtered);

    /* get sum of count and size */
    MPI_Allreduce(&my_count, total_count, 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);
    MPI_Allreduce(&my_size,  total_size,  1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);

    return filtered;
}
Exemple #3
0
static int sort_files_readdir(const char* sortfields, mfu_flist* pflist)
{
    /* get list from caller */
    mfu_flist flist = *pflist;

    /* create a new list as subset of original list */
    mfu_flist flist2 = mfu_flist_subset(flist);

    uint64_t incount = mfu_flist_size(flist);
    uint64_t chars   = mfu_flist_file_max_name(flist);

    /* create datatype for packed file list element */
    MPI_Datatype dt_sat;
    size_t bytes = mfu_flist_file_pack_size(flist);
    MPI_Type_contiguous((int)bytes, MPI_BYTE, &dt_sat);

    /* get our rank and the size of comm_world */
    int rank, ranks;
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &ranks);

    /* build type for file path */
    MPI_Datatype dt_filepath;
    MPI_Type_contiguous((int)chars, MPI_CHAR, &dt_filepath);
    MPI_Type_commit(&dt_filepath);

    /* build comparison op for filenames */
    DTCMP_Op op_filepath;
    if (DTCMP_Op_create(dt_filepath, my_strcmp, &op_filepath) != DTCMP_SUCCESS) {
        MFU_ABORT(1, "Failed to create sorting operation for filepath");
    }

    /* build comparison op for filenames */
    DTCMP_Op op_filepath_rev;
    if (DTCMP_Op_create(dt_filepath, my_strcmp_rev, &op_filepath_rev) != DTCMP_SUCCESS) {
        MFU_ABORT(1, "Failed to create reverse sorting operation for filepath");
    }

    /* TODO: process sort fields */
    const int MAXFIELDS = 1;
    MPI_Datatype types[MAXFIELDS];
    DTCMP_Op ops[MAXFIELDS];
    sort_field fields[MAXFIELDS];
    size_t lengths[MAXFIELDS];
    int nfields = 0;
    for (nfields = 0; nfields < MAXFIELDS; nfields++) {
        types[nfields]   = MPI_DATATYPE_NULL;
        ops[nfields]     = DTCMP_OP_NULL;
    }
    nfields = 0;
    char* sortfields_copy = MFU_STRDUP(sortfields);
    char* token = strtok(sortfields_copy, ",");
    while (token != NULL) {
        int valid = 1;
        if (strcmp(token, "name") == 0) {
            types[nfields]   = dt_filepath;
            ops[nfields]     = op_filepath;
            fields[nfields]  = FILENAME;
            lengths[nfields] = chars;
        }
        else if (strcmp(token, "-name") == 0) {
            types[nfields]   = dt_filepath;
            ops[nfields]     = op_filepath_rev;
            fields[nfields]  = FILENAME;
            lengths[nfields] = chars;
        }
        else {
            /* invalid token */
            valid = 0;
            if (rank == 0) {
                MFU_LOG(MFU_LOG_ERR, "Invalid sort field: %s\n", token);
            }
        }
        if (valid) {
            nfields++;
        }
        if (nfields > MAXFIELDS) {
            /* TODO: print warning if we have too many fields */
            break;
        }
        token = strtok(NULL, ",");
    }
    mfu_free(&sortfields_copy);

    /* build key type */
    MPI_Datatype dt_key;
    if (DTCMP_Type_create_series(nfields, types, &dt_key) != DTCMP_SUCCESS) {
        MFU_ABORT(1, "Failed to create type for key");
    }

    /* create sort op */
    DTCMP_Op op_key;
    if (DTCMP_Op_create_series(nfields, ops, &op_key) != DTCMP_SUCCESS) {
        MFU_ABORT(1, "Failed to create sorting operation for key");
    }

    /* build keysat type */
    MPI_Datatype dt_keysat, keysat_types[2];
    keysat_types[0] = dt_key;
    keysat_types[1] = dt_sat;
    if (DTCMP_Type_create_series(2, keysat_types, &dt_keysat) != DTCMP_SUCCESS) {
        MFU_ABORT(1, "Failed to create type for keysat");
    }

    /* get extent of key type */
    MPI_Aint key_lb, key_extent;
    MPI_Type_get_extent(dt_key, &key_lb, &key_extent);

    /* get extent of keysat type */
    MPI_Aint keysat_lb, keysat_extent;
    MPI_Type_get_extent(dt_keysat, &keysat_lb, &keysat_extent);

    /* get extent of sat type */
    MPI_Aint sat_lb, sat_extent;
    MPI_Type_get_extent(dt_sat, &sat_lb, &sat_extent);

    /* compute size of sort element and allocate buffer */
    size_t sortbufsize = (size_t)keysat_extent * incount;
    void* sortbuf = MFU_MALLOC(sortbufsize);

    /* copy data into sort elements */
    uint64_t idx = 0;
    char* sortptr = (char*) sortbuf;
    while (idx < incount) {
        /* copy in access time */
        int i;
        for (i = 0; i < nfields; i++) {
            if (fields[i] == FILENAME) {
                const char* name = mfu_flist_file_get_name(flist, idx);
                strcpy(sortptr, name);
            }
            sortptr += lengths[i];
        }

        /* pack file element */
        sortptr += mfu_flist_file_pack(sortptr, flist, idx);

        idx++;
    }

    /* sort data */
    void* outsortbuf;
    int outsortcount;
    DTCMP_Handle handle;
    int sort_rc = DTCMP_Sortz(
                      sortbuf, (int)incount, &outsortbuf, &outsortcount,
                      dt_key, dt_keysat, op_key, DTCMP_FLAG_NONE,
                      MPI_COMM_WORLD, &handle
                  );
    if (sort_rc != DTCMP_SUCCESS) {
        MFU_ABORT(1, "Failed to sort data");
    }

    /* step through sorted data filenames */
    idx = 0;
    sortptr = (char*) outsortbuf;
    while (idx < (uint64_t)outsortcount) {
        sortptr += key_extent;
        sortptr += mfu_flist_file_unpack(sortptr, flist2);
        idx++;
    }

    /* build summary of new list */
    mfu_flist_summarize(flist2);

    /* free memory */
    DTCMP_Free(&handle);

    /* free ops */
    DTCMP_Op_free(&op_key);
    DTCMP_Op_free(&op_filepath_rev);
    DTCMP_Op_free(&op_filepath);

    /* free types */
    MPI_Type_free(&dt_keysat);
    MPI_Type_free(&dt_key);
    MPI_Type_free(&dt_filepath);

    /* free input buffer holding sort elements */
    mfu_free(&sortbuf);

    /* free the satellite type */
    MPI_Type_free(&dt_sat);

    /* return new list and free old one */
    *pflist = flist2;
    mfu_flist_free(&flist);

    return MFU_SUCCESS;
}
Exemple #4
0
/* given an input flist, return a newly allocated flist consisting of 
 * a filtered set by finding all items that match/don't match a given
 * regular expression */
mfu_flist mfu_flist_filter_regex(mfu_flist flist, const char* regex_exp, int exclude, int name)
{
    /* create our list to return */
    mfu_flist dest = mfu_flist_subset(flist);

    /* check if user passed in an expression, if so then filter the list */
    if (regex_exp != NULL) {
        /* compile regular expression, if it fails print error */
        regex_t regex;
        int regex_return = regcomp(&regex, regex_exp, 0);
        if (regex_return) {
            MFU_ABORT(-1, "Could not compile regex: `%s' rc=%d\n", regex_exp, regex_return);
        }

        /* copy the things that don't or do (based on input) match the regex into a
         * filtered list */
        uint64_t idx = 0;
        uint64_t size = mfu_flist_size(flist);
        while (idx < size) {
            /* get full path of item */
            const char* file_name = mfu_flist_file_get_name(flist, idx);

            /* get basename of item (exclude the path) */
            mfu_path* pathname = mfu_path_from_str(file_name);
            mfu_path_basename(pathname);
            char* base = mfu_path_strdup(pathname);

            /* execute regex on item, either against the basename or
             * the full path depending on name flag */
            if (name) {
                /* run regex on basename */
                regex_return = regexec(&regex, base, 0, NULL, 0);
            }
            else {
                /* run regex on full path */
                regex_return = regexec(&regex, file_name, 0, NULL, 0);
            }

            /* copy item to the filtered list */
            if (exclude) {
                /* user wants to exclude items that match, so copy everything that
                 * does not match */
                if (regex_return == REG_NOMATCH) {
                    mfu_flist_file_copy(flist, idx, dest);
                }
            }
            else {
                /* user wants to copy over any matching items */
                if (regex_return == 0) {
                    mfu_flist_file_copy(flist, idx, dest);
                }
            }

            /* free the basename */
            mfu_free(&base);
            mfu_path_delete(&pathname);

            /* get next item in our list */
            idx++;
        }

        /* summarize the filtered list */
        mfu_flist_summarize(dest);
    }

    /* return the filtered list */
    return dest;
}
Exemple #5
0
/* given a list of files print from the start to end of the list */
void mfu_flist_print(mfu_flist flist)
{
    /* number of items to print from start and end of list */
    uint64_t range = 10;

    /* allocate send and receive buffers */
    size_t pack_size = mfu_flist_file_pack_size(flist);
    size_t bufsize = 2 * range * pack_size;
    void* sendbuf = MFU_MALLOC(bufsize);
    void* recvbuf = MFU_MALLOC(bufsize);

    /* get our rank and the size of comm_world */
    int rank, ranks;
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &ranks);

    /* identify the number of items we have, the total number,
     * and our offset in the global list */
    uint64_t count  = mfu_flist_size(flist);
    uint64_t total  = mfu_flist_global_size(flist);
    uint64_t offset = mfu_flist_global_offset(flist);

    /* count the number of items we'll send */
    int num = 0;
    uint64_t idx = 0;
    while (idx < count) {
        uint64_t global = offset + idx;
        if (global < range || (total - global) <= range) {
            num++;
        }
        idx++;
    }

    /* allocate arrays to store counts and displacements */
    int* counts = (int*) MFU_MALLOC((size_t)ranks * sizeof(int));
    int* disps  = (int*) MFU_MALLOC((size_t)ranks * sizeof(int));

    /* tell rank 0 where the data is coming from */
    int bytes = num * (int)pack_size;
    MPI_Gather(&bytes, 1, MPI_INT, counts, 1, MPI_INT, 0, MPI_COMM_WORLD);

    /* pack items into sendbuf */
    idx = 0;
    char* ptr = (char*) sendbuf;
    while (idx < count) {
        uint64_t global = offset + idx;
        if (global < range || (total - global) <= range) {
            ptr += mfu_flist_file_pack(ptr, flist, idx);
        }
        idx++;
    }

    /* compute displacements and total bytes */
    int recvbytes = 0;
    if (rank == 0) {
        int i;
        disps[0] = 0;
        recvbytes += counts[0];
        for (i = 1; i < ranks; i++) {
            disps[i] = disps[i - 1] + counts[i - 1];
            recvbytes += counts[i];
        }
    }

    /* gather data to rank 0 */
    MPI_Gatherv(sendbuf, bytes, MPI_BYTE, recvbuf, counts, disps, MPI_BYTE, 0, MPI_COMM_WORLD);

    /* create temporary list to unpack items into */
    mfu_flist tmplist = mfu_flist_subset(flist);

    /* unpack items into new list */
    if (rank == 0) {
        ptr = (char*) recvbuf;
        char* end = ptr + recvbytes;
        while (ptr < end) {
            mfu_flist_file_unpack(ptr, tmplist);
            ptr += pack_size;
        }
    }

    /* summarize list */
    mfu_flist_summarize(tmplist);

    /* print files */
    if (rank == 0) {
        printf("\n");
        uint64_t tmpidx = 0;
        uint64_t tmpsize = mfu_flist_size(tmplist);
        while (tmpidx < tmpsize) {
            print_file(tmplist, tmpidx);
            tmpidx++;
            if (tmpidx == range && total > 2 * range) {
                /* going to have to leave some out */
                printf("\n<snip>\n\n");
            }
        }
        printf("\n");
    }

    /* free our temporary list */
    mfu_flist_free(&tmplist);

    /* free memory */
    mfu_free(&disps);
    mfu_free(&counts);
    mfu_free(&sendbuf);
    mfu_free(&recvbuf);

    return;
}
Exemple #6
0
/* given an input list and a map function pointer, call map function
 * for each item in list, identify new rank to send item to and then
 * exchange items among ranks and return new output list */
mfu_flist mfu_flist_remap(mfu_flist list, mfu_flist_map_fn map, const void* args)
{
    uint64_t idx;

    /* create new list as subset (actually will be a remapping of
     * input list */
    mfu_flist newlist = mfu_flist_subset(list);

    /* get our rank and number of ranks in job */
    int ranks;
    MPI_Comm_size(MPI_COMM_WORLD, &ranks);

    /* allocate arrays for alltoall */
    size_t bufsize = (size_t)ranks * sizeof(int);
    int* sendsizes  = (int*) MFU_MALLOC(bufsize);
    int* senddisps  = (int*) MFU_MALLOC(bufsize);
    int* sendoffset = (int*) MFU_MALLOC(bufsize);
    int* recvsizes  = (int*) MFU_MALLOC(bufsize);
    int* recvdisps  = (int*) MFU_MALLOC(bufsize);

    /* initialize sendsizes and offsets */
    int i;
    for (i = 0; i < ranks; i++) {
        sendsizes[i]  = 0;
        sendoffset[i] = 0;
    }

    /* get number of elements in our local list */
    uint64_t size = mfu_flist_size(list);

    /* allocate space to record file-to-rank mapping */
    int* file2rank = (int*) MFU_MALLOC(size * sizeof(int));

    /* call map function for each item to identify its new rank,
     * and compute number of bytes we'll send to each rank */
    size_t sendbytes = 0;
    for (idx = 0; idx < size; idx++) {
        /* determine which rank we'll map this file to */
        int dest = map(list, idx, ranks, args);

        /* cache mapping so we don't have to compute it again
         * below while packing items for send */
        file2rank[idx] = dest;

        /* TODO: check that pack size doesn't overflow int */
        /* total number of bytes we'll send to each rank and the total overall */
        size_t count = mfu_flist_file_pack_size(list);
        sendsizes[dest] += (int) count;
        sendbytes += count;
    }

    /* compute send buffer displacements */
    senddisps[0] = 0;
    for (i = 1; i < ranks; i++) {
        senddisps[i] = senddisps[i - 1] + sendsizes[i - 1];
    }

    /* allocate space for send buffer */
    char* sendbuf = (char*) MFU_MALLOC(sendbytes);

    /* copy data into send buffer */
    for (idx = 0; idx < size; idx++) {
        /* determine which rank we mapped this file to */
        int dest = file2rank[idx];

        /* get pointer into send buffer and pack item */
        char* ptr = sendbuf + senddisps[dest] + sendoffset[dest];
        size_t count = mfu_flist_file_pack(ptr, list, idx);

        /* TODO: check that pack size doesn't overflow int */
        /* bump up the offset for this rank */
        sendoffset[dest] += (int) count;
    }

    /* alltoall to get our incoming counts */
    MPI_Alltoall(sendsizes, 1, MPI_INT, recvsizes, 1, MPI_INT, MPI_COMM_WORLD);

    /* compute size of recvbuf and displacements */
    size_t recvbytes = 0;
    recvdisps[0] = 0;
    for (i = 0; i < ranks; i++) {
        recvbytes += (size_t) recvsizes[i];
        if (i > 0) {
            recvdisps[i] = recvdisps[i - 1] + recvsizes[i - 1];
        }
    }

    /* allocate recvbuf */
    char* recvbuf = (char*) MFU_MALLOC(recvbytes);

    /* alltoallv to send data */
    MPI_Alltoallv(
        sendbuf, sendsizes, senddisps, MPI_CHAR,
        recvbuf, recvsizes, recvdisps, MPI_CHAR, MPI_COMM_WORLD
    );

    /* unpack items into new list */
    char* ptr = recvbuf;
    char* recvend = recvbuf + recvbytes;
    while (ptr < recvend) {
        size_t count = mfu_flist_file_unpack(ptr, newlist);
        ptr += count;
    }
    mfu_flist_summarize(newlist);

    /* free memory */
    mfu_free(&file2rank);
    mfu_free(&recvbuf);
    mfu_free(&recvdisps);
    mfu_free(&recvsizes);
    mfu_free(&sendbuf);
    mfu_free(&sendoffset);
    mfu_free(&senddisps);
    mfu_free(&sendsizes);

    /* return list to caller */
    return newlist;
}