Exemplo n.º 1
0
void DTAR_enqueue_copy(CIRCLE_handle *handle) {
    for (uint64_t idx = 0; idx < DTAR_count; idx++) {
        mfu_filetype type = mfu_flist_file_get_type(DTAR_flist, idx);
        /* add copy work only for files */
        if (type == MFU_TYPE_FILE) {
            uint64_t dataoffset = DTAR_offsets[idx] + DTAR_HDR_LENGTH;
            const char * name = mfu_flist_file_get_name(DTAR_flist, idx);
            uint64_t size = mfu_flist_file_get_size(DTAR_flist, idx);

            /* compute number of chunks */
            uint64_t num_chunks = size / DTAR_user_opts.chunk_size;
            for (uint64_t chunk_idx = 0; chunk_idx < num_chunks; chunk_idx++) {
                char* newop = DTAR_encode_operation(
                        COPY_DATA, name, size, chunk_idx, dataoffset);
                handle->enqueue(newop);
                mfu_free(&newop);

            }

            /* create copy work for possibly last item */
            if (num_chunks * DTAR_user_opts.chunk_size < size || num_chunks == 0) {
                char* newop = DTAR_encode_operation(
                        COPY_DATA, name, size, num_chunks, dataoffset);
                handle->enqueue(newop);
                mfu_free(&newop);
            }
        }
    }
}
Exemplo n.º 2
0
int MFU_PRED_EXEC (mfu_flist flist, uint64_t idx, void* arg)
{
    int argmax = 1024*1024;;
    int written = 0;
    int ret;
    char* command = MFU_STRDUP((char*) arg);
    char* cmdline = (char*) MFU_MALLOC(argmax);
    char* subst = strstr(command, "{}");

    if (subst) {
        subst[0] = '\0';
        subst += 2; /* Point to the first char after '{}' */
    }

    const char* name = mfu_flist_file_get_name(flist, idx);

    written = snprintf(cmdline, argmax/sizeof(char), "%s%s%s", command, name, subst);
    if (written > argmax/sizeof(char)) {
        fprintf(stderr, "argument %s to exec too long.\n", cmdline);
        mfu_free(&cmdline);
        mfu_free(&command);
        return -1;
    }

    ret = system(cmdline);

    mfu_free(&cmdline);
    mfu_free(&command);

    return ret ? 0 : 1;
}
Exemplo n.º 3
0
/* append element to tail of linked list */
void mfu_flist_insert_elem(flist_t* flist, elem_t* elem)
{
    /* set head if this is the first item */
    if (flist->list_head == NULL) {
        flist->list_head = elem;
    }

    /* update last element to point to this new element */
    elem_t* tail = flist->list_tail;
    if (tail != NULL) {
        tail->next = elem;
    }

    /* make this element the new tail */
    flist->list_tail = elem;
    elem->next = NULL;

    /* increase list count by one */
    flist->list_count++;

    /* delete the index if we have one, it's out of date */
    mfu_free(&flist->list_index);

    return;
}
Exemplo n.º 4
0
void DCOPY_retry_failed_operation(DCOPY_operation_code_t target, \
                                  CIRCLE_handle* handle, \
                                  DCOPY_operation_t* op)
{
    char* new_op;

    if(DCOPY_user_opts.reliable_filesystem) {
        MFU_LOG(MFU_LOG_ERR, "Not retrying failed operation. " \
            "Reliable filesystem is specified. (op=%d chunk=%ld src=%s dst=%s)",
            target, op->chunk, op->operand, op->dest_full_path);

        DCOPY_abort(EXIT_FAILURE);
    }
    else {
        MFU_LOG(MFU_LOG_INFO, "Attempting to retry operation.");

        new_op = DCOPY_encode_operation(target, op->chunk, op->operand, \
                                        op->source_base_offset, \
                                        op->dest_base_appendix, op->file_size);

        handle->enqueue(new_op);
        mfu_free(&new_op);
    }

    return;
}
Exemplo n.º 5
0
/* delete linked list of stat items */
static void list_delete(flist_t* flist)
{
    elem_t* current = flist->list_head;
    while (current != NULL) {
        elem_t* next = current->next;
        mfu_free(&current->file);
        mfu_free(&current);
        current = next;
    }
    flist->list_count = 0;
    flist->list_head  = NULL;
    flist->list_tail  = NULL;

    /* delete the cached index */
    mfu_free(&flist->list_index);

    return;
}
Exemplo n.º 6
0
/* given the address of a pointer to an operation_t struct,
 * free associated memory and set pointer to NULL */
void DCOPY_opt_free(DCOPY_operation_t** optptr)
{
    if(optptr != NULL) {
        /* get pointer to operation_t struct */
        DCOPY_operation_t* opt = (*optptr);

        if(opt != NULL) {
            /* free memory and then the object itself */
            mfu_free(&opt->dest_full_path);
            mfu_free(&opt);
        }

        /* set caller's pointer to NULL to catch bugs */
        *optptr = NULL;
    }

    return;
}
Exemplo n.º 7
0
/* frees resources allocated in call to parse_path_args() */
void DCOPY_free_path_args(void)
{
    /* free memory associated with destination path */
    mfu_param_path_free(&dest_param);

    /* free memory associated with source paths */
    int i;
    for(i = 0; i < num_src_params; i++) {
        mfu_param_path_free(&src_params[i]);
    }
    num_src_params = 0;
    mfu_free(&src_params);
}
Exemplo n.º 8
0
static int distribution_parse(struct distribute_option *option, const char *string)
{
    char *ptr;
    char *next;
    unsigned long long separator;
    char *str;
    int status = 0;

    if (strncmp(string, "size", strlen("size")) != 0) {
        return -1;
    }

    option->separator_number = 0;
    if (strlen(string) == strlen("size")) {
        return 0;
    }

    if (string[strlen("size")] != ':') {
        return -1;
    }

    str = MFU_STRDUP(string);
    /* Parse separators */
    ptr = str + strlen("size:");
    next = ptr;
    while (ptr && ptr < str + strlen(string)) {
        next = strchr(ptr, ',');
        if (next != NULL) {
            *next = '\0';
            next++;
        }

        if (mfu_abtoull(ptr, &separator) != MFU_SUCCESS) {
            printf("Invalid separator \"%s\"\n", ptr);
            status = -1;
            goto out;
        }

        if (distribute_separator_add(option, separator)) {
            printf("Duplicated separator \"%llu\"\n", separator);
            status = -1;
            goto out;
        }

        ptr = next;
    }

out:
    mfu_free(&str);
    return status;
}
Exemplo n.º 9
0
/* we hash file names based on its parent directory to map all
 * files in the same directory to the same process */
static int map_name(mfu_flist flist, uint64_t idx, int ranks, void* args)
{
    /* get name of item */
    const char* name = mfu_flist_file_get_name(flist, idx);

    /* identify rank to send this file to */
    char* dir = MFU_STRDUP(name);
    dirname(dir);
    size_t dir_len = strlen(dir);
    uint32_t hash = mfu_hash_jenkins(dir, dir_len);
    int rank = (int)(hash % (uint32_t)ranks);
    mfu_free(&dir);
    return rank;
}
Exemplo n.º 10
0
static int lustre_mds_stat(int fd, char* fname, struct stat* sb)
{
    /* allocate a buffer */
    size_t pathlen = strlen(fname) + 1;
    size_t bufsize = pathlen;
    //size_t datasize = sizeof(lstat_t) + lov_user_md_size(LOV_MAX_STRIPE_COUNT, LOV_USER_MAGIC_V3);
    size_t datasize = sizeof(struct lov_user_mds_data) + LOV_MAX_STRIPE_COUNT * sizeof(struct lov_user_ost_data_v1);
    if (datasize > bufsize) {
        bufsize = datasize;
    }
    char* buf = (char*) MFU_MALLOC(bufsize);

    /* Usage: ioctl(fd, IOC_MDC_GETFILEINFO, buf)
     * IN: fd open file descriptor of file's parent directory
     * IN: buf file name (no path)
     * OUT: buf lstat_t */
    strcpy(buf, fname);
    //  strncpy(buf, fname, bufsize);

    int ret = ioctl(fd, IOC_MDC_GETFILEINFO, buf);

    /* Copy lstat_t to struct stat */
    if (ret != -1) {
        lstat_t* ls = (lstat_t*) & ((struct lov_user_mds_data*) buf)->lmd_st;
        sb->st_dev     = ls->st_dev;
        sb->st_ino     = ls->st_ino;
        sb->st_mode    = ls->st_mode;
        sb->st_nlink   = ls->st_nlink;
        sb->st_uid     = ls->st_uid;
        sb->st_gid     = ls->st_gid;
        sb->st_rdev    = ls->st_rdev;
        sb->st_size    = ls->st_size;
        sb->st_blksize = ls->st_blksize;
        sb->st_blocks  = ls->st_blocks;
        sb->st_atime   = ls->st_atime;
        sb->st_mtime   = ls->st_mtime;
        sb->st_ctime   = ls->st_ctime;

        lustre_stripe_info(buf);
    }
    else {
        MFU_LOG(MFU_LOG_ERR, "ioctl fd=%d (errno=%d %s)", fd, errno, strerror(errno));
    }

    /* free the buffer */
    mfu_free(&buf);

    return ret;
}
Exemplo n.º 11
0
int DCOPY_close_file(DCOPY_file_cache_t* cache)
{
    int rc = 0;

    /* close file if we have one */
    char* name = cache->name;
    if (name != NULL) {
        /* TODO: if open for write, fsync? */
        int fd = cache->fd;
        rc = mfu_close(name, fd);
        mfu_free(&cache->name);
    }

    return rc;
}
Exemplo n.º 12
0
/* free resources allocated in list of params */
static void mfu_param_path_free_list(uint64_t num, mfu_param_path* params)
{
    /* make sure we got a valid pointer */
    if (params != NULL) {
        /* free memory for each param */
        uint64_t i;
        for (i = 0; i < num; i++) {
            /* get pointer to param structure */
            mfu_param_path* param = &params[i];

            /* free memory and reinit */
            if (param != NULL) {
                /* free all mememory */
                mfu_free(&param->orig);
                mfu_free(&param->path);
                mfu_free(&param->target);

                /* initialize all fields */
                mfu_param_path_init(param);
            }
        }
    }
    return;
}
Exemplo n.º 13
0
static void DTAR_perform_copy(CIRCLE_handle* handle)
{
    char opstr[CIRCLE_MAX_STRING_LEN];
    char iobuf[FD_BLOCK_SIZE];

    int out_fd = DTAR_writer.fd_tar;

    handle->dequeue(opstr);
    DTAR_operation_t* op = DTAR_decode_operation(opstr);

    uint64_t in_offset = DTAR_user_opts.chunk_size * op->chunk_index;
    int in_fd = open(op->operand, O_RDONLY);

    ssize_t num_of_bytes_read = 0;
    ssize_t num_of_bytes_written = 0;
    ssize_t total_bytes_written = 0;

    uint64_t out_offset = op->offset + in_offset;

    lseek(in_fd, in_offset, SEEK_SET);
    lseek(out_fd, out_offset, SEEK_SET);

    while (total_bytes_written < DTAR_user_opts.chunk_size) {
        num_of_bytes_read = read(in_fd, &iobuf[0], sizeof(iobuf));
        if (! num_of_bytes_read) {
            break;
        }
        num_of_bytes_written = write(out_fd, &iobuf[0], num_of_bytes_read);
        total_bytes_written += num_of_bytes_written;
    }

    uint64_t num_chunks = op->file_size / DTAR_user_opts.chunk_size;
    uint64_t rem = op->file_size - DTAR_user_opts.chunk_size * num_chunks;
    uint64_t last_chunk = (rem) ? num_chunks : num_chunks - 1;

    /* handle last chunk */
    if (op->chunk_index == last_chunk) {
        int padding = 512 - (int) (op->file_size % 512);
        if (padding > 0 && padding != 512) {
            char* buff = (char*) calloc(padding, sizeof(char));
            write(out_fd, buff, padding);
        }
    }

    close(in_fd);
    mfu_free(&op);
}
Exemplo n.º 14
0
int DCOPY_open_file(const char* file, int read_flag, DCOPY_file_cache_t* cache)
{
    int newfd = -1;

    /* see if we have a cached file descriptor */
    char* name = cache->name;
    if (name != NULL) {
        /* we have a cached file descriptor */
        int fd = cache->fd;
        if (strcmp(name, file) == 0 && cache->read == read_flag) {
            /* the file we're trying to open matches name and read/write mode,
             * so just return the cached descriptor */
            return fd;
        } else {
            /* the file we're trying to open is different,
             * close the old file and delete the name */
            mfu_close(name, fd);
            mfu_free(&cache->name);
        }
    }

    /* open the new file */
    if (read_flag) {
        int flags = O_RDONLY;
        if (DCOPY_user_opts.synchronous) {
            flags |= O_DIRECT;
        }
        newfd = mfu_open(file, flags);
    } else {
        int flags = O_WRONLY | O_CREAT;
        if (DCOPY_user_opts.synchronous) {
            flags |= O_DIRECT;
        }
        newfd = mfu_open(file, flags, DCOPY_DEF_PERMS_FILE);
    }

    /* cache the file descriptor */
    if (newfd != -1) {
        cache->name = MFU_STRDUP(file);
        cache->fd   = newfd;
        cache->read = read_flag;
    }

    return newfd;
}
Exemplo n.º 15
0
/* frees array of lists created in call to
 * mfu_flist_split_by_depth */
void mfu_flist_array_free(int levels, mfu_flist** outlists)
{
    /* check that our pointer is valid */
    if (outlists == NULL) {
        return;
    }

    /* free each list */
    int i;
    mfu_flist* lists = *outlists;
    for (i = 0; i < levels; i++) {
        mfu_flist_free(&lists[i]);
    }

    /* free the array of lists and set caller's pointer to NULL */
    mfu_free(outlists);
    return;
}
Exemplo n.º 16
0
/* free resouces in file list */
void mfu_flist_free(mfu_flist* pbflist)
{
    /* convert handle to flist_t */
    flist_t* flist = *(flist_t**)pbflist;

    /* delete linked list */
    list_delete(flist);

    /* free user and group structures */
    mfu_flist_usrgrp_free(flist);

    mfu_free(&flist);

    /* set caller's pointer to NULL */
    *pbflist = MFU_FLIST_NULL;

    return;
}
Exemplo n.º 17
0
int DCOPY_close_file(DCOPY_file_cache_t* cache)
{
    int rc = 0;

    /* close file if we have one */
    char* name = cache->name;
    if (name != NULL) {
        int fd = cache->fd;

        /* if open for write, fsync */
        int read_flag = cache->read;
        if (! read_flag) {
            rc = mfu_fsync(name, fd);
        }

        /* close the file and delete the name string */
        rc = mfu_close(name, fd);
        mfu_free(&cache->name);
    }

    return rc;
}
Exemplo n.º 18
0
/* given a list of param_paths, walk each one and add to flist */
void mfu_flist_walk_param_paths(uint64_t num,
                                const mfu_param_path* params,
                                mfu_walk_opts_t* walk_opts,
                                mfu_flist flist)
{
    /* allocate memory to hold a list of paths */
    const char** path_list = (const char**) MFU_MALLOC(num * sizeof(char*));

    /* fill list of paths and print each one */
    uint64_t i;
    for (i = 0; i < num; i++) {
        /* get path for this step */
        path_list[i] = params[i].path;
    }

    /* walk file tree and record stat data for each file */
    mfu_flist_walk_paths((uint64_t) num, path_list, walk_opts, flist);

    /* free the list */
    mfu_free(&path_list);

    return;
}
Exemplo n.º 19
0
static int add_type(mfu_pred* p, char t)
{
    mode_t* type = (mode_t*) MFU_MALLOC(sizeof(mode_t));
    switch (t) {
    case 'b':
        *type = S_IFBLK;
        break;
    case 'c':
        *type = S_IFCHR;
        break;
    case 'd':
        *type = S_IFDIR;
        break;
    case 'f':
        *type = S_IFREG;
        break;
    case 'l':
        *type = S_IFLNK;
        break;
    case 'p':
        *type = S_IFIFO;
        break;
    case 's':
        *type = S_IFSOCK;
        break;

    default:
        /* unsupported type character */
        mfu_free(&type);
        return -1;
        break;
    }

    /* add check for this type */
    mfu_pred_add(p, MFU_PRED_TYPE, (void *)type);
    return 1;
}
Exemplo n.º 20
0
/* for each depth, sort files by filename and then remove, to test
 * whether it matters to limit the number of directories each process
 * has to reference (e.g., locking) */
static void remove_sort(mfu_flist list, uint64_t* rmcount)
{
    /* bail out if total count is 0 */
    uint64_t all_count = mfu_flist_global_size(list);
    if (all_count == 0) {
        return;
    }

    /* get maximum file name and number of items */
    int chars = (int) mfu_flist_file_max_name(list);
    uint64_t my_count = mfu_flist_size(list);

    /* create key datatype (filename) and comparison op */
    MPI_Datatype dt_key;
    DTCMP_Op op_str;
    DTCMP_Str_create_ascend(chars, &dt_key, &op_str);

    /* create keysat datatype (filename + type) */
    MPI_Datatype types[2], dt_keysat;
    types[0] = dt_key;
    types[1] = MPI_CHAR;
    DTCMP_Type_create_series(2, types, &dt_keysat);

    /* allocate send buffer */
    int sendcount = (int) my_count;
    size_t sendbufsize = (size_t)(sendcount * (chars + 1));
    char* sendbuf = (char*) MFU_MALLOC(sendbufsize);

    /* copy data into buffer */
    char* ptr = sendbuf;
    uint64_t idx;
    for (idx = 0; idx < my_count; idx++) {
        /* encode the filename first */
        const char* name = mfu_flist_file_get_name(list, idx);
        strcpy(ptr, name);
        ptr += chars;

        /* last character encodes item type */
        mfu_filetype type = mfu_flist_file_get_type(list, idx);
        if (type == MFU_TYPE_DIR) {
            ptr[0] = 'd';
        }
        else if (type == MFU_TYPE_FILE || type == MFU_TYPE_LINK) {
            ptr[0] = 'f';
        }
        else {
            ptr[0] = 'u';
        }
        ptr++;
    }

    /* sort items */
    void* recvbuf;
    int recvcount;
    DTCMP_Handle handle;
    DTCMP_Sortz(
        sendbuf, sendcount, &recvbuf, &recvcount,
        dt_key, dt_keysat, op_str, DTCMP_FLAG_NONE, MPI_COMM_WORLD, &handle
    );

    /* delete data */
    int delcount = 0;
    ptr = (char*)recvbuf;
    while (delcount < recvcount) {
        /* get item name */
        char* name = ptr;
        ptr += chars;

        /* get item type */
        char type = ptr[0];
        ptr++;

        /* delete item */
        remove_type(type, name);
        delcount++;
    }

    /* record number of items we deleted */
    *rmcount = (uint64_t) delcount;

    /* free output data */
    DTCMP_Free(&handle);

    /* free our send buffer */
    mfu_free(&sendbuf);

    /* free key comparison operation */
    DTCMP_Op_free(&op_str);

    /* free datatypes */
    MPI_Type_free(&dt_keysat);
    MPI_Type_free(&dt_key);

    return;
}
Exemplo n.º 21
0
/* for given depth, hash directory name and map to processes to
 * test whether having all files in same directory on one process
 * matters */
size_t mfu_flist_distribute_map(mfu_flist list, char** buffer,
                                  mfu_flist_name_encode_fn encode,
                                  mfu_flist_map_fn map, void* args)
{
    uint64_t idx;

    /* get our rank and number of ranks in job */
    int ranks;
    MPI_Comm_size(MPI_COMM_WORLD, &ranks);

    /* allocate arrays for alltoall */
    size_t bufsize = (size_t)ranks * sizeof(int);
    int* sendsizes  = (int*) MFU_MALLOC(bufsize);
    int* senddisps  = (int*) MFU_MALLOC(bufsize);
    int* sendoffset = (int*) MFU_MALLOC(bufsize);
    int* recvsizes  = (int*) MFU_MALLOC(bufsize);
    int* recvdisps  = (int*) MFU_MALLOC(bufsize);

    /* initialize sendsizes and offsets */
    int i;
    for (i = 0; i < ranks; i++) {
        sendsizes[i]  = 0;
        sendoffset[i] = 0;
    }

    /* compute number of bytes we'll send to each rank */
    size_t sendbytes = 0;
    uint64_t size = mfu_flist_size(list);
    for (idx = 0; idx < size; idx++) {
        int dest = map(list, idx, ranks, args);

        /* TODO: check that pack size doesn't overflow int */
        /* total number of bytes we'll send to each rank and the total overall */
        size_t count = encode(NULL, list, idx, args);
        sendsizes[dest] += (int) count;
        sendbytes += count;
    }

    /* compute displacements */
    senddisps[0] = 0;
    for (i = 1; i < ranks; i++) {
        senddisps[i] = senddisps[i - 1] + sendsizes[i - 1];
    }

    /* allocate space */
    char* sendbuf = (char*) MFU_MALLOC(sendbytes);

    /* copy data into buffer */
    for (idx = 0; idx < size; idx++) {
        int dest = map(list, idx, ranks, args);

        /* identify region to be sent to rank */
        char* path = sendbuf + senddisps[dest] + sendoffset[dest];
        size_t count = encode(path, list, idx, args);

        /* TODO: check that pack size doesn't overflow int */
        /* bump up the offset for this rank */
        sendoffset[dest] += (int) count;
    }

    /* alltoall to specify incoming counts */
    MPI_Alltoall(sendsizes, 1, MPI_INT, recvsizes, 1, MPI_INT, MPI_COMM_WORLD);

    /* compute size of recvbuf and displacements */
    size_t recvbytes = 0;
    recvdisps[0] = 0;
    for (i = 0; i < ranks; i++) {
        recvbytes += (size_t) recvsizes[i];
        if (i > 0) {
            recvdisps[i] = recvdisps[i - 1] + recvsizes[i - 1];
        }
    }

    /* allocate recvbuf */
    char* recvbuf = (char*) MFU_MALLOC(recvbytes);

    /* alltoallv to send data */
    MPI_Alltoallv(
        sendbuf, sendsizes, senddisps, MPI_CHAR,
        recvbuf, recvsizes, recvdisps, MPI_CHAR, MPI_COMM_WORLD
    );

    /* free memory */
    mfu_free(&recvdisps);
    mfu_free(&recvsizes);
    mfu_free(&sendbuf);
    mfu_free(&sendoffset);
    mfu_free(&senddisps);
    mfu_free(&sendsizes);

    *buffer = recvbuf;
    return recvbytes;
}
Exemplo n.º 22
0
/* for given depth, evenly spread the files among processes for
 * improved load balancing */
static void remove_spread(mfu_flist flist, uint64_t* rmcount)
{
    uint64_t idx;

    /* initialize our remove count */
    *rmcount = 0;

    /* get our rank and number of ranks in job */
    int rank, ranks;
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &ranks);

    /* allocate memory for alltoall exchanges */
    size_t bufsize = (size_t)ranks * sizeof(int);
    int* sendcounts = (int*) MFU_MALLOC(bufsize);
    int* sendsizes  = (int*) MFU_MALLOC(bufsize);
    int* senddisps  = (int*) MFU_MALLOC(bufsize);
    int* recvsizes  = (int*) MFU_MALLOC(bufsize);
    int* recvdisps  = (int*) MFU_MALLOC(bufsize);

    /* get number of items */
    uint64_t my_count  = mfu_flist_size(flist);
    uint64_t all_count = mfu_flist_global_size(flist);
    uint64_t offset    = mfu_flist_global_offset(flist);

    /* compute number of bytes we'll send */
    size_t sendbytes = 0;
    for (idx = 0; idx < my_count; idx++) {
        const char* name = mfu_flist_file_get_name(flist, idx);
        size_t len = strlen(name) + 2;
        sendbytes += len;
    }

    /* compute the number of items that each rank should have */
    uint64_t low = all_count / (uint64_t)ranks;
    uint64_t extra = all_count - low * (uint64_t)ranks;

    /* compute number that we'll send to each rank and initialize sendsizes and offsets */
    uint64_t i;
    for (i = 0; i < (uint64_t)ranks; i++) {
        /* compute starting element id and count for given rank */
        uint64_t start, num;
        if (i < extra) {
            num = low + 1;
            start = i * num;
        }
        else {
            num = low;
            start = (i - extra) * num + extra * (low + 1);
        }

        /* compute the number of items we'll send to this task */
        uint64_t sendcnt = 0;
        if (my_count > 0) {
            if (start <= offset && offset < start + num) {
                /* this rank overlaps our range,
                 * and its first element comes at or before our first element */
                sendcnt = num - (offset - start);
                if (my_count < sendcnt) {
                    /* the number the rank could receive from us
                     * is more than we have left */
                    sendcnt = my_count;
                }
            }
            else if (offset < start && start < offset + my_count) {
                /* this rank overlaps our range,
                 * and our first element comes strictly before its first element */
                sendcnt = my_count - (start - offset);
                if (num < sendcnt) {
                    /* the number the rank can receive from us
                     * is less than we have left */
                    sendcnt = num;
                }
            }
        }

        /* record the number of items we'll send to this task */
        sendcounts[i]  = (int) sendcnt;

        /* set sizes and displacements to 0, we'll fix this later */
        sendsizes[i] = 0;
        senddisps[i] = 0;
    }

    /* allocate space */
    char* sendbuf = (char*) MFU_MALLOC(sendbytes);

    /* copy data into buffer */
    int dest = -1;
    int disp = 0;
    for (idx = 0; idx < my_count; idx++) {
        /* get name and type of item */
        const char* name = mfu_flist_file_get_name(flist, idx);
        mfu_filetype type = mfu_flist_file_get_type(flist, idx);

        /* get rank that we're packing data for */
        if (dest == -1) {
            dest = get_first_nonzero(sendcounts, ranks);
            if (dest == -1) {
                /* error */
            }
            /* about to copy first item for this rank,
             * record its displacement */
            senddisps[dest] = disp;
        }

        /* identify region to be sent to rank */
        char* path = sendbuf + disp;

        /* first character encodes item type */
        if (type == MFU_TYPE_DIR) {
            path[0] = 'd';
        }
        else if (type == MFU_TYPE_FILE || type == MFU_TYPE_LINK) {
            path[0] = 'f';
        }
        else {
            path[0] = 'u';
        }

        /* now copy in the path */
        strcpy(&path[1], name);

        /* TODO: check that we don't overflow the int */
        /* add bytes to sendsizes and increase displacement */
        size_t count = strlen(name) + 2;
        sendsizes[dest] += (int) count;
        disp += (int) count;

        /* decrement the count for this rank */
        sendcounts[dest]--;
        if (sendcounts[dest] == 0) {
            dest = -1;
        }
    }

    /* compute displacements */
    senddisps[0] = 0;
    for (i = 1; i < (uint64_t)ranks; i++) {
        senddisps[i] = senddisps[i - 1] + sendsizes[i - 1];
    }

    /* alltoall to specify incoming counts */
    MPI_Alltoall(sendsizes, 1, MPI_INT, recvsizes, 1, MPI_INT, MPI_COMM_WORLD);

    /* compute size of recvbuf and displacements */
    size_t recvbytes = 0;
    recvdisps[0] = 0;
    for (i = 0; i < (uint64_t)ranks; i++) {
        recvbytes += (size_t) recvsizes[i];
        if (i > 0) {
            recvdisps[i] = recvdisps[i - 1] + recvsizes[i - 1];
        }
    }

    /* allocate recvbuf */
    char* recvbuf = (char*) MFU_MALLOC(recvbytes);

    /* alltoallv to send data */
    MPI_Alltoallv(
        sendbuf, sendsizes, senddisps, MPI_CHAR,
        recvbuf, recvsizes, recvdisps, MPI_CHAR, MPI_COMM_WORLD
    );

    /* delete data */
    char* item = recvbuf;
    while (item < recvbuf + recvbytes) {
        /* get item name and type */
        char type = item[0];
        char* name = &item[1];

        /* delete item */
        remove_type(type, name);

        /* keep tally of number of items we deleted */
        *rmcount++;

        /* go to next item */
        size_t item_size = strlen(item) + 1;
        item += item_size;
    }

    /* free memory */
    mfu_free(&recvbuf);
    mfu_free(&recvdisps);
    mfu_free(&recvsizes);
    mfu_free(&sendbuf);
    mfu_free(&senddisps);
    mfu_free(&sendsizes);
    mfu_free(&sendcounts);

    return;
}
Exemplo n.º 23
0
/* check that source and destination paths are valid */
void mfu_param_path_check_copy(uint64_t num, const mfu_param_path* paths, 
        const mfu_param_path* destpath, int* flag_valid, int* flag_copy_into_dir)
{
    /* initialize output params */
    *flag_valid = 0;
    *flag_copy_into_dir = 0;

    /* need at least two paths to have a shot at being valid */
    if (num < 1 || paths == NULL || destpath == NULL) {
        return;
    }

    /* get current rank */
    int rank;
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);

    /* assume path parameters are valid */
    int valid = 1;

    /* just have rank 0 check */
    if(rank == 0) {
        /* count number of readable source paths */
        uint64_t i;
        int num_readable = 0;
        for(i = 0; i < num; i++) {
            const char* path = paths[i].path;
            if(mfu_access(path, R_OK) == 0) {
                num_readable++;
            }
            else {
                /* found a source path that we can't read, not fatal,
                 * but print an error to notify user */
                const char* orig = paths[i].orig;
                MFU_LOG(MFU_LOG_ERR, "Could not read `%s' (errno=%d %s)",
                    orig, errno, strerror(errno));
            }
        }

        /* verify that we have at least one source path */
        if(num_readable < 1) {
            MFU_LOG(MFU_LOG_ERR, "At least one valid source must be specified");
            valid = 0;
            goto bcast;
        }

        /*
         * First we need to determine if the last argument is a file or a directory.
         * We first attempt to see if the last argument already exists on disk. If it
         * doesn't, we then look at the sources to see if we can determine what the
         * last argument should be.
         */

        bool dest_exists = false;
        bool dest_is_dir = false;
        bool dest_is_file = false;
        bool dest_is_link_to_dir = false;
        bool dest_is_link_to_file = false;
        bool dest_required_to_be_dir = false;

        /* check whether dest exists, its type, and whether it's writable */
        if(destpath->path_stat_valid) {
            /* we could stat dest path, so something is there */
            dest_exists = true;

            /* now determine its type */
            if(S_ISDIR(destpath->path_stat.st_mode)) {
                /* dest is a directory */
                dest_is_dir  = true;
            }
            else if(S_ISREG(destpath->path_stat.st_mode)) {
                /* dest is a file */
                dest_is_file = true;
            }
            else if(S_ISLNK(destpath->path_stat.st_mode)) {
                /* dest is a symlink, but to what? */
                if (destpath->target_stat_valid) {
                    /* target of the symlink exists, determine what it is */
                    if(S_ISDIR(destpath->target_stat.st_mode)) {
                        /* dest is link to a directory */
                        dest_is_link_to_dir = true;
                    }
                    else if(S_ISREG(destpath->target_stat.st_mode)) {
                        /* dest is link to a file */
                        dest_is_link_to_file = true;
                    }
                    else {
                        /* unsupported type */
                        MFU_LOG(MFU_LOG_ERR, "Unsupported filetype `%s' --> `%s'",
                            destpath->orig, destpath->target);
                        valid = 0;
                        goto bcast;
                    }
                }
                else {
                    /* dest is a link, but its target does not exist,
                     * consider this an error */
                    MFU_LOG(MFU_LOG_ERR, "Destination is broken symlink `%s'",
                        destpath->orig);
                    valid = 0;
                    goto bcast;
                }
            }
            else {
                /* unsupported type */
                MFU_LOG(MFU_LOG_ERR, "Unsupported filetype `%s'",
                    destpath->orig);
                valid = 0;
                goto bcast;
            }

            /* check that dest is writable */
            if(mfu_access(destpath->path, W_OK) < 0) {
                MFU_LOG(MFU_LOG_ERR, "Destination is not writable `%s'",
                    destpath->path);
                valid = 0;
                goto bcast;
            }
        }
        else {
            /* destination does not exist, so we'll be creating it,
             * check that its parent is writable */

            /* compute parent path */
            mfu_path* parent = mfu_path_from_str(destpath->path);
            mfu_path_dirname(parent);
            char* parent_str = mfu_path_strdup(parent);
            mfu_path_delete(&parent);

            /* check that parent is writable */
            if(mfu_access(parent_str, W_OK) < 0) {
                MFU_LOG(MFU_LOG_ERR, "Destination parent directory is not writable `%s'",
                    parent_str);
                valid = 0;
                mfu_free(&parent_str);
                goto bcast;
            }
            mfu_free(&parent_str);
        }

        /* determine whether caller *requires* copy into dir */

        /* TODO: if caller specifies dest/ or dest/. */

        /* if caller specifies more than one source,
         * then dest has to be a directory */
        if(num > 1) {
            dest_required_to_be_dir = true;
        }

        /* if caller requires dest to be a directory, and if dest does not
         * exist or it does it exist but it's not a directory, then abort */
        if(dest_required_to_be_dir &&
           (!dest_exists || (!dest_is_dir && !dest_is_link_to_dir)))
        {
            MFU_LOG(MFU_LOG_ERR, "Destination is not a directory `%s'",
                destpath->orig);
            valid = 0;
            goto bcast;
        }

        /* we copy into a directory if any of the following:
         *   1) user specified more than one source
         *   2) destination already exists and is a directory
         *   3) destination already exists and is a link to a directory */
        bool copy_into_dir = (dest_required_to_be_dir || dest_is_dir || dest_is_link_to_dir);
        *flag_copy_into_dir = copy_into_dir ? 1 : 0;
    }

bcast:
    /* get status from rank 0 */
    MPI_Bcast(&valid, 1, MPI_INT, 0, MPI_COMM_WORLD);

    /* set valid flag */
    *flag_valid = valid;

    /* rank 0 broadcasts whether we're copying into a directory */
    MPI_Bcast(flag_copy_into_dir, 1, MPI_INT, 0, MPI_COMM_WORLD);

    return;
}
Exemplo n.º 24
0
int main(int argc, char** argv)
{
    /* initialize MPI */
    MPI_Init(&argc, &argv);
    mfu_init();

    /* get our rank and the size of comm_world */
    int rank, ranks;
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &ranks);

    /* pointer to mfu_walk_opts */
    mfu_walk_opts_t* walk_opts = mfu_walk_opts_new();

    /* parse command line options */
    char* inputname = NULL;
    char* ownername = NULL;
    char* groupname = NULL;
    char* modestr   = NULL;
    char* regex_exp = NULL;
    mfu_perms* head = NULL;
    int walk        = 0;
    int exclude     = 0;
    int name        = 0;

    /* verbose by default */
    mfu_debug_level = MFU_LOG_VERBOSE;

    int option_index = 0;
    static struct option long_options[] = {
        {"input",    1, 0, 'i'},
        {"owner",    1, 0, 'u'},
        {"group",    1, 0, 'g'},
        {"mode",     1, 0, 'm'},
        {"exclude",  1, 0, 'e'},
        {"match",    1, 0, 'a'},
        {"name",     0, 0, 'n'},
        {"progress", 1, 0, 'P'},
        {"verbose",  0, 0, 'v'},
        {"quiet",    0, 0, 'q'},
        {"help",     0, 0, 'h'},
        {0, 0, 0, 0}
    };

    int usage = 0;
    while (1) {
        int c = getopt_long(
                    argc, argv, "i:u:g:m:nvqh",
                    long_options, &option_index
                );

        if (c == -1) {
            break;
        }

        switch (c) {
            case 'i':
                inputname = MFU_STRDUP(optarg);
                break;
            case 'u':
                ownername = MFU_STRDUP(optarg);
                break;
            case 'g':
                groupname = MFU_STRDUP(optarg);
                break;
            case 'm':
                modestr = MFU_STRDUP(optarg);
                break;
            case 'e':
                regex_exp = MFU_STRDUP(optarg);
                exclude = 1;
                break;
            case 'a':
                regex_exp = MFU_STRDUP(optarg);
                exclude = 0;
                break;
            case 'n':
                name = 1;
                break;
            case 'P':
                mfu_progress_timeout = atoi(optarg);
                break;
            case 'v':
                mfu_debug_level = MFU_LOG_VERBOSE;
                break;
            case 'q':
                mfu_debug_level = MFU_LOG_NONE;
                break;
            case 'h':
                usage = 1;
                break;
            case '?':
                usage = 1;
                break;
            default:
                if (rank == 0) {
                    printf("?? getopt returned character code 0%o ??\n", c);
                }
        }
    }

    /* check that we got a valid progress value */
    if (mfu_progress_timeout < 0) {
        if (rank == 0) {
            MFU_LOG(MFU_LOG_ERR, "Seconds in --progress must be non-negative: %d invalid", mfu_progress_timeout);
        }
        usage = 1;
    }

    /* paths to walk come after the options */
    int numpaths = 0;
    mfu_param_path* paths = NULL;
    if (optind < argc) {
        /* got a path to walk */
        walk = 1;

        /* determine number of paths specified by user */
        numpaths = argc - optind;

        /* allocate space for each path */
        paths = (mfu_param_path*) MFU_MALLOC((size_t)numpaths * sizeof(mfu_param_path));

        /* process each path */
        const char** argpaths = (const char**)(&argv[optind]);
        mfu_param_path_set_all(numpaths, argpaths, paths);

        /* advance to next set of options */
        optind += numpaths;

        /* don't allow input file and walk */
        if (inputname != NULL) {
            usage = 1;
        }
    }
    else {
        /* if we're not walking, we must be reading,
         * and for that we need a file */
        if (inputname == NULL) {
            usage = 1;
        }
    }

    /* check that our mode string parses correctly */
    if (modestr != NULL) {
        int valid = mfu_perms_parse(modestr, &head);
        if (! valid) {
            usage = 1;
            if (rank == 0) {
                printf("invalid mode string: %s\n", modestr);
            }

            /* free the head of the list */
            mfu_perms_free(&head);
        }
    }

    /* print usage if we need to */
    if (usage) {
        if (rank == 0) {
            print_usage();
        }
        mfu_finalize();
        MPI_Finalize();
        return 1;
    }

    /* create an empty file list */
    mfu_flist flist = mfu_flist_new();

    /* flag used to check if permissions need to be
     * set on the walk */
    if (head != NULL) {
        mfu_perms_need_dir_rx(head, walk_opts);
    }

    /* get our list of files, either by walking or reading an
     * input file */
    if (walk) {
        /* if in octal mode set use_stat=0 to stat each file on walk */
        if (head != NULL && head->octal && ownername == NULL && groupname == NULL) {
            walk_opts->use_stat = 0;
        }
        /* walk list of input paths */
        mfu_flist_walk_param_paths(numpaths, paths, walk_opts, flist);
    }
    else {
        /* read list from file */
        mfu_flist_read_cache(inputname, flist);
    }

    /* assume we'll use the full list */
    mfu_flist srclist = flist;

    /* filter the list if needed */
    mfu_flist filtered_flist = MFU_FLIST_NULL;
    if (regex_exp != NULL) {
        /* filter the list based on regex */
        filtered_flist = mfu_flist_filter_regex(flist, regex_exp, exclude, name);

        /* update our source list to use the filtered list instead of the original */
        srclist = filtered_flist;
    }

    /* change group and permissions */
    mfu_flist_chmod(srclist, ownername, groupname, head);

    /* free list if it was used */
    if (filtered_flist != MFU_FLIST_NULL){
        /* free the filtered flist (if any) */
        mfu_flist_free(&filtered_flist);
    }

    /* free the file list */
    mfu_flist_free(&flist);

    /* free the path parameters */
    mfu_param_path_free_all(numpaths, paths);

    /* free memory allocated to hold params */
    mfu_free(&paths);

    /* free the owner and group names */
    mfu_free(&ownername);
    mfu_free(&groupname);

    /* free the modestr */
    mfu_free(&modestr);

    /* free the match_pattern if it isn't null */
    if (regex_exp != NULL) {
        mfu_free(&regex_exp);
    }

    /* free the head of the list */
    mfu_perms_free(&head);

    /* free the input file name */
    mfu_free(&inputname);

    /* free the walk options */
    mfu_walk_opts_delete(&walk_opts);

    /* shut down MPI */
    mfu_finalize();
    MPI_Finalize();

    return 0;
}
Exemplo n.º 25
0
int main(int argc, char** argv)
{
    int i;

    /* initialize MPI */
    MPI_Init(&argc, &argv);
    mfu_init();

    /* get our rank and the size of comm_world */
    int rank, ranks;
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &ranks);

    /* parse command line options */
    char* inputname = NULL;
    char* regex_exp = NULL;
    int walk        = 0;
    int exclude     = 0;
    int name        = 0;
    int dryrun      = 0;

    int option_index = 0;
    static struct option long_options[] = {
        {"input",    1, 0, 'i'},
        {"lite",     0, 0, 'l'},
        {"exclude",  1, 0, 'e'},
        {"match",    1, 0, 'a'},
        {"name",     0, 0, 'n'},        
        {"help",     0, 0, 'h'},
        {"dryrun",   0, 0, 'd'},
        {"verbose",  0, 0, 'v'},
        {0, 0, 0, 0}
    };

    int usage = 0;
    while (1) {
        int c = getopt_long(
                    argc, argv, "i:nlhv",
                    long_options, &option_index
                );

        if (c == -1) {
            break;
        }

        switch (c) {
            case 'i':
                inputname = MFU_STRDUP(optarg);
                break;
            case 'l':
                walk_stat = 0;
                break;
            case 'e':
                regex_exp = MFU_STRDUP(optarg);
                exclude = 1;
                break;
            case 'a':
                regex_exp = MFU_STRDUP(optarg);
                exclude = 0;
                break;
            case 'n':
                name = 1;
                break;
            case 'h':
                usage = 1;
                break;
            case 'd':
                dryrun = 1;
                break;            
            case 'v':
                mfu_debug_level = MFU_LOG_VERBOSE;
                break;
            case '?':
                usage = 1;
                break;
            default:
                if (rank == 0) {
                    printf("?? getopt returned character code 0%o ??\n", c);
                }
        }
    }

    /* paths to walk come after the options */
    int numpaths = 0;
    mfu_param_path* paths = NULL;
    if (optind < argc) {
        /* got a path to walk */
        walk = 1;

        /* determine number of paths specified by user */
        numpaths = argc - optind;

        /* allocate space for each path */
        paths = (mfu_param_path*) MFU_MALLOC((size_t)numpaths * sizeof(mfu_param_path));

        /* process each path */
        char** argpaths = &argv[optind];
        mfu_param_path_set_all(numpaths, argpaths, paths);

        /* advance to next set of options */
        optind += numpaths;

        /* don't allow input file and walk */
        if (inputname != NULL) {
            usage = 1;
        }
    }
    else {
        /* if we're not walking, we must be reading,
         * and for that we need a file */
        if (inputname == NULL) {
            usage = 1;
        }
    }

    /* print usage if we need to */
    if (usage) {
        if (rank == 0) {
            print_usage();
        }
        mfu_finalize();
        MPI_Finalize();
        return 1;
    }

    /* create an empty file list */
    mfu_flist flist = mfu_flist_new();

    /* get our list of files, either by walking or reading an
     * input file */
    if (walk) {
        /* walk list of input paths */
        mfu_param_path_walk(numpaths, paths, walk_stat, flist, dir_perm);
    }
    else {
        /* read list from file */
        mfu_flist_read_cache(inputname, flist);
    }

    /* assume we'll use the full list */
    mfu_flist srclist = flist;

    /* filter the list if needed */
    mfu_flist filtered_flist = MFU_FLIST_NULL;
    if (regex_exp != NULL) {
        /* filter the list based on regex */
        filtered_flist = mfu_flist_filter_regex(flist, regex_exp, exclude, name);

        /* update our source list to use the filtered list instead of the original */
        srclist = filtered_flist;
    }

    /* only actually delete files if the user wasn't doing a dry run */
    if (dryrun) {
        /* just print what we would delete without actually doing anything,
         * this is useful if the user is trying to get a regex right */
        mfu_flist_print(srclist);
    } else {
        /* remove files */
        mfu_flist_unlink(srclist);
    }

    /* free list if it was used */
    if (filtered_flist != MFU_FLIST_NULL){
        /* free the filtered flist (if any) */
        mfu_flist_free(&filtered_flist);
    }

    /* free the file list */
    mfu_flist_free(&flist);

    /* free the path parameters */
    mfu_param_path_free_all(numpaths, paths);

    /* free memory allocated to hold params */
    mfu_free(&paths);

    /* free the regex string if we have one */
    mfu_free(&regex_exp);

    /* free the input file name */
    mfu_free(&inputname);

    /* shut down MPI */
    mfu_finalize();
    MPI_Finalize();

    return 0;
}
Exemplo n.º 26
0
int main (int argc, char** argv)
{
    /* initialize MPI */
    MPI_Init(&argc, &argv);
    mfu_init();

    /* get our rank and the size of comm_world */
    int rank, ranks;
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &ranks);

    /* pointer to mfu_walk_opts */
    mfu_walk_opts_t* walk_opts = mfu_walk_opts_new();

    /* capture current time for any time based queries,
     * to get a consistent value, capture and bcast from rank 0 */
    mfu_pred_times* now_t = mfu_pred_now();

    int ch;

    mfu_pred* pred_head = mfu_pred_new();
    char* inputname  = NULL;
    char* outputname = NULL;
    int walk = 0;
    int text = 0;

    static struct option long_options[] = {
        {"input",     1, 0, 'i'},
        {"output",    1, 0, 'o'},
        {"verbose",   0, 0, 'v'},
        {"quiet",     0, 0, 'q'},
        {"help",      0, 0, 'h'},

        { "maxdepth", required_argument, NULL, 'd' },

        { "amin",     required_argument, NULL, 'a' },
        { "anewer",   required_argument, NULL, 'B' },
        { "atime",    required_argument, NULL, 'A' },
        { "cmin",     required_argument, NULL, 'c' },
        { "cnewer",   required_argument, NULL, 'D' },
        { "ctime",    required_argument, NULL, 'C' },
        { "mmin",     required_argument, NULL, 'm' },
        { "newer",    required_argument, NULL, 'N' },
        { "mtime",    required_argument, NULL, 'M' },

        { "gid",      required_argument, NULL, 'g' },
        { "group",    required_argument, NULL, 'G' },
        { "uid",      required_argument, NULL, 'u' },
        { "user",     required_argument, NULL, 'U' },

        { "name",     required_argument, NULL, 'n' },
        { "path",     required_argument, NULL, 'P' },
        { "regex",    required_argument, NULL, 'r' },
        { "size",     required_argument, NULL, 's' },
        { "type",     required_argument, NULL, 't' },

        { "print",    no_argument,       NULL, 'p' },
        { "exec",     required_argument, NULL, 'e' },
        { NULL, 0, NULL, 0 },
    };

    options.maxdepth = INT_MAX;

    int usage = 0;
    while (1) {
        int c = getopt_long(
                    argc, argv, "i:o:vqh",
                    long_options, NULL
                );

        if (c == -1) {
            break;
        }

        int i;
        int space;
        char* buf;
        mfu_pred_times* t;
        mfu_pred_times_rel* tr;
        regex_t* r;
        int ret;

        /* verbose by default */
        mfu_debug_level = MFU_LOG_VERBOSE;

    	switch (c) {
    	case 'e':
            space = 1024 * 1024;
    	    buf = (char *)MFU_MALLOC(space);
    	    for (i = optind-1; strcmp(";", argv[i]); i++) {
    	        if (i > argc) {
                    if (rank == 0) {
    	                printf("%s: exec missing terminating ';'\n", argv[0]);
                    }
    	            exit(1);
    	        }
    	        strncat(buf, argv[i], space);
    	        space -= strlen(argv[i]) + 1; /* save room for space or null */
    	        if (space <= 0) {
                    if (rank == 0) {
    	                printf("%s: exec argument list too long.\n", argv[0]);
                    }
    	            mfu_free(&buf);
    	            continue;
    	        }
    	        strcat(buf, " ");
    	        optind++;
    	    }
    	    buf[strlen(buf)] = '\0'; /* clobbers trailing space */
    	    mfu_pred_add(pred_head, MFU_PRED_EXEC, buf);
    	    break;

    	case 'd':
    	    options.maxdepth = atoi(optarg);
    	    break;

    	case 'g':
            /* TODO: error check argument */
    	    buf = MFU_STRDUP(optarg);
    	    mfu_pred_add(pred_head, MFU_PRED_GID, (void *)buf);
    	    break;

    	case 'G':
    	    buf = MFU_STRDUP(optarg);
    	    mfu_pred_add(pred_head, MFU_PRED_GROUP, (void *)buf);
    	    break;

    	case 'u':
            /* TODO: error check argument */
    	    buf = MFU_STRDUP(optarg);
    	    mfu_pred_add(pred_head, MFU_PRED_UID, (void *)buf);
    	    break;

    	case 'U':
    	    buf = MFU_STRDUP(optarg);
    	    mfu_pred_add(pred_head, MFU_PRED_USER, (void *)buf);
    	    break;

    	case 's':
    	    buf = MFU_STRDUP(optarg);
    	    mfu_pred_add(pred_head, MFU_PRED_SIZE, (void *)buf);
    	    break;

    	case 'n':
    	    mfu_pred_add(pred_head, MFU_PRED_NAME, MFU_STRDUP(optarg));
    	    break;
    	case 'P':
    	    mfu_pred_add(pred_head, MFU_PRED_PATH, MFU_STRDUP(optarg));
    	    break;
    	case 'r':
            r = (regex_t*) MFU_MALLOC(sizeof(regex_t));
            ret = regcomp(r, optarg, 0);
            if (ret) {
                MFU_ABORT(-1, "Could not compile regex: `%s' rc=%d\n", optarg, ret);
            }
    	    mfu_pred_add(pred_head, MFU_PRED_REGEX, (void*)r);
    	    break;

    	case 'a':
            tr = mfu_pred_relative(optarg, now_t);
    	    mfu_pred_add(pred_head, MFU_PRED_AMIN, (void *)tr);
    	    break;
    	case 'm':
            tr = mfu_pred_relative(optarg, now_t);
    	    mfu_pred_add(pred_head, MFU_PRED_MMIN, (void *)tr);
    	    break;
    	case 'c':
            tr = mfu_pred_relative(optarg, now_t);
    	    mfu_pred_add(pred_head, MFU_PRED_CMIN, (void *)tr);
    	    break;

    	case 'A':
            tr = mfu_pred_relative(optarg, now_t);
    	    mfu_pred_add(pred_head, MFU_PRED_ATIME, (void *)tr);
    	    break;
    	case 'M':
            tr = mfu_pred_relative(optarg, now_t);
    	    mfu_pred_add(pred_head, MFU_PRED_MTIME, (void *)tr);
    	    break;
    	case 'C':
            tr = mfu_pred_relative(optarg, now_t);
    	    mfu_pred_add(pred_head, MFU_PRED_CTIME, (void *)tr);
    	    break;

    	case 'B':
            t = get_mtimes(optarg);
            if (t == NULL) {
                if (rank == 0) {
    	            printf("%s: can't find file %s\n", argv[0], optarg);
                }
    	        exit(1);
    	    }
    	    mfu_pred_add(pred_head, MFU_PRED_ANEWER, (void *)t);
    	    break;
    	case 'N':
            t = get_mtimes(optarg);
            if (t == NULL) {
                if (rank == 0) {
    	            printf("%s: can't find file %s\n", argv[0], optarg);
                }
    	        exit(1);
    	    }
    	    mfu_pred_add(pred_head, MFU_PRED_MNEWER, (void *)t);
    	    break;
    	case 'D':
            t = get_mtimes(optarg);
            if (t == NULL) {
                if (rank == 0) {
    	            printf("%s: can't find file %s\n", argv[0], optarg);
                }
    	        exit(1);
    	    }
    	    mfu_pred_add(pred_head, MFU_PRED_CNEWER, (void *)t);
    	    break;

    	case 'p':
    	    mfu_pred_add(pred_head, MFU_PRED_PRINT, NULL);
    	    break;

    	case 't':
            ret = add_type(pred_head, *optarg);
            if (ret != 1) {
                if (rank == 0) {
    	            printf("%s: unsupported file type %s\n", argv[0], optarg);
                }
    	        exit(1);
            }
    	    break;

        case 'i':
            inputname = MFU_STRDUP(optarg);
            break;
        case 'o':
            outputname = MFU_STRDUP(optarg);
            break;
        case 'v':
            mfu_debug_level = MFU_LOG_VERBOSE;
            break;
        case 'q':
            mfu_debug_level = MFU_LOG_NONE;
            break;
        case 'h':
            usage = 1;
            break;
        case '?':
            usage = 1;
            break;
        default:
            if (rank == 0) {
                printf("?? getopt returned character code 0%o ??\n", c);
            }
    	}
    }

    pred_commit(pred_head);

    /* paths to walk come after the options */
    int numpaths = 0;
    mfu_param_path* paths = NULL;
    if (optind < argc) {
        /* got a path to walk */
        walk = 1;

        /* determine number of paths specified by user */
        numpaths = argc - optind;

        /* allocate space for each path */
        paths = (mfu_param_path*) MFU_MALLOC((size_t)numpaths * sizeof(mfu_param_path));

        /* process each path */
        char** p = &argv[optind];
        mfu_param_path_set_all((uint64_t)numpaths, (const char**)p, paths);
        optind += numpaths;

        /* don't allow user to specify input file with walk */
        if (inputname != NULL) {
            usage = 1;
        }
    }
    else {
        /* if we're not walking, we must be reading,
         * and for that we need a file */
        if (inputname == NULL) {
            usage = 1;
        }
    }

    if (usage) {
        if (rank == 0) {
            print_usage();
        }
        mfu_finalize();
        MPI_Finalize();
        return 0;
    }


    /* create an empty file list */
    mfu_flist flist = mfu_flist_new();

    if (walk) {
        /* walk list of input paths */
        mfu_flist_walk_param_paths(numpaths, paths, walk_opts, flist);
    }
    else {
        /* read data from cache file */
        mfu_flist_read_cache(inputname, flist);
    }

    /* apply predicates to each item in list */
    mfu_flist flist2 = mfu_flist_filter_pred(flist, pred_head);

    /* write data to cache file */
    if (outputname != NULL) {
        if (!text) {
            mfu_flist_write_cache(outputname, flist2);
        } else {
            mfu_flist_write_text(outputname, flist2);
        }
    }

    /* free off the filtered list */
    mfu_flist_free(&flist2);

    /* free users, groups, and files objects */
    mfu_flist_free(&flist);

    /* free predicate list */
    mfu_pred_free(&pred_head);

    /* free memory allocated for options */
    mfu_free(&outputname);
    mfu_free(&inputname);

    /* free the path parameters */
    mfu_param_path_free_all(numpaths, paths);

    /* free memory allocated to hold params */
    mfu_free(&paths);

    /* free structure holding current time */
    mfu_free(&now_t);

    /* free the walk options */
    mfu_walk_opts_delete(&walk_opts);

    /* shut down MPI */
    mfu_finalize();
    MPI_Finalize();

    return 0;
}
Exemplo n.º 27
0
void mfu_param_path_set_all(uint64_t num, const char** paths, mfu_param_path* params)
{
    /* get our rank and number of ranks */
    int rank, ranks;
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &ranks);

    /* determine number we should look up */
    uint64_t count = num / (uint64_t) ranks;
    uint64_t extra = num - count * (uint64_t) ranks;
    if (rank < (int) extra) {
        /* procs whose rank is less than extra each
         * handle one extra param than those whose
         * rank is equal or greater than extra */
        count++;
    }

    /* determine our starting point */
    uint64_t start = 0;
    if (rank < (int) extra) {
        /* for these procs, count is one more than procs with ranks >= extra */
        start = (uint64_t)rank * count;
    } else {
        /* for these procs, count is one less than procs with ranks < extra */
        start = extra * (count + 1) + ((uint64_t)rank - extra) * count;
    }

    /* TODO: allocate temporary params */
    mfu_param_path* p = MFU_MALLOC(count * sizeof(mfu_param_path));

    /* track maximum path length */
    uint64_t bytes = 0;

    /* process each path we're responsible for */
    uint64_t i;
    for (i = 0; i < count; i++) {
        /* get pointer to param structure */
        mfu_param_path* param = &p[i];

        /* initialize all fields */
        mfu_param_path_init(param);

        /* lookup the path */
        uint64_t path_idx = start + i;
        const char* path = paths[path_idx];
        
        /* set param fields for path */
        if (path != NULL) {
            /* make a copy of original path */
            param->orig = MFU_STRDUP(path);

            /* get absolute path and remove ".", "..", consecutive "/",
             * and trailing "/" characters */
            param->path = mfu_path_strdup_abs_reduce_str(path);

            /* get stat info for simplified path */
            if (mfu_lstat(param->path, &param->path_stat) == 0) {
                param->path_stat_valid = 1;
            }

            /* TODO: we use realpath below, which is nice since it takes out
             * ".", "..", symlinks, and adds the absolute path, however, it
             * fails if the file/directory does not already exist, which is
             * often the case for dest path. */

            /* resolve any symlinks */
            char target[PATH_MAX];
            if (realpath(path, target) != NULL) {
                /* make a copy of resolved name */
                param->target = MFU_STRDUP(target);

                /* get stat info for resolved path */
                if (mfu_lstat(param->target, &param->target_stat) == 0) {
                    param->target_stat_valid = 1;
                }
            }

            /* add in bytes needed to pack this param */
            bytes += (uint64_t) mfu_pack_param_size(param);
        }
    }

    /* TODO: eventually it would be nice to leave this data distributed,
     * however for now some tools expect all params to be defined */

    /* allgather to get bytes on each process */
    int* recvcounts = (int*) MFU_MALLOC(ranks * sizeof(int));
    int* recvdispls = (int*) MFU_MALLOC(ranks * sizeof(int));
    int sendcount = (int) bytes;
    MPI_Allgather(&sendcount, 1, MPI_INT, recvcounts, 1, MPI_INT, MPI_COMM_WORLD);

    /* compute displacements and total number of bytes that we'll receive */
    uint64_t allbytes = 0;
    int disp = 0;
    for (i = 0; i < (uint64_t) ranks; i++) {
        recvdispls[i] = disp;
        disp += recvcounts[i];
        allbytes += (uint64_t) recvcounts[i];
    }

    /* allocate memory for send and recv buffers */
    char* sendbuf = MFU_MALLOC(bytes);
    char* recvbuf = MFU_MALLOC(allbytes);

    /* pack send buffer */
    char* ptr = sendbuf;
    for (i = 0; i < count; i++) {
        mfu_pack_param(&ptr, &p[i]);
    }
    
    /* allgatherv to collect data */
    MPI_Allgatherv(sendbuf, sendcount, MPI_BYTE, recvbuf, recvcounts, recvdispls, MPI_BYTE, MPI_COMM_WORLD);

    /* unpack recv buffer into caller's params */
    ptr = recvbuf;
    for (i = 0; i < num; i++) {
        mfu_unpack_param((const char**)(&ptr), &params[i]);
    }

    /* Loop through the list of files &/or directories, and check the params 
     * struct to see if all of them are valid file names. If one is not, let 
     * the user know by printing a warning */
    if (rank == 0) {
        for (i = 0; i < num; i++) {
            /* get pointer to param structure */
            mfu_param_path* param = &params[i];
            if (param->path_stat_valid == 0) {
                /* failed to find a file at this location, let user know (may be a typo) */
                MFU_LOG(MFU_LOG_WARN, "Warning: `%s' does not exist", param->orig); 
            }
        }
    }

    /* free message buffers */
    mfu_free(&recvbuf);
    mfu_free(&sendbuf);

    /* free arrays for recv counts and displacements */
    mfu_free(&recvdispls);
    mfu_free(&recvcounts);

    /* free temporary params */
    mfu_param_path_free_list(count, p);
    mfu_free(&p);

    return;
}
Exemplo n.º 28
0
static int sort_files_readdir(const char* sortfields, mfu_flist* pflist)
{
    /* get list from caller */
    mfu_flist flist = *pflist;

    /* create a new list as subset of original list */
    mfu_flist flist2 = mfu_flist_subset(flist);

    uint64_t incount = mfu_flist_size(flist);
    uint64_t chars   = mfu_flist_file_max_name(flist);

    /* create datatype for packed file list element */
    MPI_Datatype dt_sat;
    size_t bytes = mfu_flist_file_pack_size(flist);
    MPI_Type_contiguous((int)bytes, MPI_BYTE, &dt_sat);

    /* get our rank and the size of comm_world */
    int rank, ranks;
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &ranks);

    /* build type for file path */
    MPI_Datatype dt_filepath;
    MPI_Type_contiguous((int)chars, MPI_CHAR, &dt_filepath);
    MPI_Type_commit(&dt_filepath);

    /* build comparison op for filenames */
    DTCMP_Op op_filepath;
    if (DTCMP_Op_create(dt_filepath, my_strcmp, &op_filepath) != DTCMP_SUCCESS) {
        MFU_ABORT(1, "Failed to create sorting operation for filepath");
    }

    /* build comparison op for filenames */
    DTCMP_Op op_filepath_rev;
    if (DTCMP_Op_create(dt_filepath, my_strcmp_rev, &op_filepath_rev) != DTCMP_SUCCESS) {
        MFU_ABORT(1, "Failed to create reverse sorting operation for filepath");
    }

    /* TODO: process sort fields */
    const int MAXFIELDS = 1;
    MPI_Datatype types[MAXFIELDS];
    DTCMP_Op ops[MAXFIELDS];
    sort_field fields[MAXFIELDS];
    size_t lengths[MAXFIELDS];
    int nfields = 0;
    for (nfields = 0; nfields < MAXFIELDS; nfields++) {
        types[nfields]   = MPI_DATATYPE_NULL;
        ops[nfields]     = DTCMP_OP_NULL;
    }
    nfields = 0;
    char* sortfields_copy = MFU_STRDUP(sortfields);
    char* token = strtok(sortfields_copy, ",");
    while (token != NULL) {
        int valid = 1;
        if (strcmp(token, "name") == 0) {
            types[nfields]   = dt_filepath;
            ops[nfields]     = op_filepath;
            fields[nfields]  = FILENAME;
            lengths[nfields] = chars;
        }
        else if (strcmp(token, "-name") == 0) {
            types[nfields]   = dt_filepath;
            ops[nfields]     = op_filepath_rev;
            fields[nfields]  = FILENAME;
            lengths[nfields] = chars;
        }
        else {
            /* invalid token */
            valid = 0;
            if (rank == 0) {
                MFU_LOG(MFU_LOG_ERR, "Invalid sort field: %s\n", token);
            }
        }
        if (valid) {
            nfields++;
        }
        if (nfields > MAXFIELDS) {
            /* TODO: print warning if we have too many fields */
            break;
        }
        token = strtok(NULL, ",");
    }
    mfu_free(&sortfields_copy);

    /* build key type */
    MPI_Datatype dt_key;
    if (DTCMP_Type_create_series(nfields, types, &dt_key) != DTCMP_SUCCESS) {
        MFU_ABORT(1, "Failed to create type for key");
    }

    /* create sort op */
    DTCMP_Op op_key;
    if (DTCMP_Op_create_series(nfields, ops, &op_key) != DTCMP_SUCCESS) {
        MFU_ABORT(1, "Failed to create sorting operation for key");
    }

    /* build keysat type */
    MPI_Datatype dt_keysat, keysat_types[2];
    keysat_types[0] = dt_key;
    keysat_types[1] = dt_sat;
    if (DTCMP_Type_create_series(2, keysat_types, &dt_keysat) != DTCMP_SUCCESS) {
        MFU_ABORT(1, "Failed to create type for keysat");
    }

    /* get extent of key type */
    MPI_Aint key_lb, key_extent;
    MPI_Type_get_extent(dt_key, &key_lb, &key_extent);

    /* get extent of keysat type */
    MPI_Aint keysat_lb, keysat_extent;
    MPI_Type_get_extent(dt_keysat, &keysat_lb, &keysat_extent);

    /* get extent of sat type */
    MPI_Aint sat_lb, sat_extent;
    MPI_Type_get_extent(dt_sat, &sat_lb, &sat_extent);

    /* compute size of sort element and allocate buffer */
    size_t sortbufsize = (size_t)keysat_extent * incount;
    void* sortbuf = MFU_MALLOC(sortbufsize);

    /* copy data into sort elements */
    uint64_t idx = 0;
    char* sortptr = (char*) sortbuf;
    while (idx < incount) {
        /* copy in access time */
        int i;
        for (i = 0; i < nfields; i++) {
            if (fields[i] == FILENAME) {
                const char* name = mfu_flist_file_get_name(flist, idx);
                strcpy(sortptr, name);
            }
            sortptr += lengths[i];
        }

        /* pack file element */
        sortptr += mfu_flist_file_pack(sortptr, flist, idx);

        idx++;
    }

    /* sort data */
    void* outsortbuf;
    int outsortcount;
    DTCMP_Handle handle;
    int sort_rc = DTCMP_Sortz(
                      sortbuf, (int)incount, &outsortbuf, &outsortcount,
                      dt_key, dt_keysat, op_key, DTCMP_FLAG_NONE,
                      MPI_COMM_WORLD, &handle
                  );
    if (sort_rc != DTCMP_SUCCESS) {
        MFU_ABORT(1, "Failed to sort data");
    }

    /* step through sorted data filenames */
    idx = 0;
    sortptr = (char*) outsortbuf;
    while (idx < (uint64_t)outsortcount) {
        sortptr += key_extent;
        sortptr += mfu_flist_file_unpack(sortptr, flist2);
        idx++;
    }

    /* build summary of new list */
    mfu_flist_summarize(flist2);

    /* free memory */
    DTCMP_Free(&handle);

    /* free ops */
    DTCMP_Op_free(&op_key);
    DTCMP_Op_free(&op_filepath_rev);
    DTCMP_Op_free(&op_filepath);

    /* free types */
    MPI_Type_free(&dt_keysat);
    MPI_Type_free(&dt_key);
    MPI_Type_free(&dt_filepath);

    /* free input buffer holding sort elements */
    mfu_free(&sortbuf);

    /* free the satellite type */
    MPI_Type_free(&dt_sat);

    /* return new list and free old one */
    *pflist = flist2;
    mfu_flist_free(&flist);

    return MFU_SUCCESS;
}
Exemplo n.º 29
0
int main(int argc, char* argv[])
{
    MPI_Init(&argc, &argv);
    mfu_init();

    /* get our rank and number of ranks in the job */
    int rank, ranks;
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &ranks);

    /* pointer to mfu_walk_opts */
    mfu_walk_opts_t* walk_opts = mfu_walk_opts_new();

    uint64_t idx;
    int option_index = 0;
    int usage = 0;
    int report = 0;
    unsigned int numpaths = 0;
    mfu_param_path* paths = NULL;
    unsigned long long bytes;

    /* verbose by default */
    mfu_debug_level = MFU_LOG_VERBOSE;

    /* default to 1MB stripe size, stripe across all OSTs, and all files are candidates */
    int stripes = -1;
    uint64_t stripe_size = 1048576;
    uint64_t min_size = 0;

    static struct option long_options[] = {
        {"count",    1, 0, 'c'},
        {"size",     1, 0, 's'},
        {"minsize",  1, 0, 'm'},
        {"report",   0, 0, 'r'},
        {"progress", 1, 0, 'P'},
        {"verbose",  0, 0, 'v'},
        {"quiet",    0, 0, 'q'},
        {"help",     0, 0, 'h'},
        {0, 0, 0, 0}
    };

    while (1) {
        int c = getopt_long(argc, argv, "c:s:m:rvqh",
                    long_options, &option_index);

        if (c == -1) {
            break;
        }

        switch (c) {
            case 'c':
                /* stripe count */
                stripes = atoi(optarg);
                break;
            case 's':
                /* stripe size in bytes */
                if (mfu_abtoull(optarg, &bytes) != MFU_SUCCESS) {
                    if (rank == 0) {
                        MFU_LOG(MFU_LOG_ERR, "Failed to parse stripe size: %s", optarg);
                    }
                    MPI_Abort(MPI_COMM_WORLD, 1);
                }
                stripe_size = (uint64_t)bytes;
                break;
            case 'm':
                /* min file size in bytes */
                if (mfu_abtoull(optarg, &bytes) != MFU_SUCCESS) {
                    if (rank == 0) {
                        MFU_LOG(MFU_LOG_ERR, "Failed to parse minimum file size: %s", optarg);
                    }
                    MPI_Abort(MPI_COMM_WORLD, 1);
                }
                min_size = (uint64_t)bytes;
                break;
            case 'r':
                /* report striping info */
		report = 1;
                break;
            case 'P':
                mfu_progress_timeout = atoi(optarg);
                break;
            case 'v':
                mfu_debug_level = MFU_LOG_VERBOSE;
                break;
            case 'q':
                mfu_debug_level = MFU_LOG_NONE;
                break;
            case 'h':
                /* display usage */
                usage = 1;
                break;
            case '?':
                /* display usage */
                usage = 1;
                break;
            default:
                if (rank == 0) {
                    printf("?? getopt returned character code 0%o ??\n", c);
                }
        }
    }

    /* check that we got a valid progress value */
    if (mfu_progress_timeout < 0) {
        if (rank == 0) {
            MFU_LOG(MFU_LOG_ERR, "Seconds in --progress must be non-negative: %d invalid", mfu_progress_timeout);
        }
        usage = 1;
    }

    /* paths to walk come after the options */
    if (optind < argc) {
        /* determine number of paths specified by user */
        numpaths = argc - optind;

        /* allocate space for each path */
        paths = (mfu_param_path*) MFU_MALLOC((size_t)numpaths * sizeof(mfu_param_path));

        /* process each path */
        char** p = &argv[optind];
        mfu_param_path_set_all((uint64_t)numpaths, (const char**)p, paths);
        optind += numpaths;
    } else {
        usage = 1;
    }

    /* if we need to print usage, print it and exit */
    if (usage) {
        if (rank == 0) {
            print_usage();
        }

        mfu_finalize();
        MPI_Finalize();
        return 1;
    }

    /* nothing to do if lustre support is disabled */
#ifndef LUSTRE_SUPPORT
    if (rank == 0) {
        MFU_LOG(MFU_LOG_ERR, "Lustre support is disabled.");
    }
    MPI_Abort(MPI_COMM_WORLD, 1);
#endif

    /* stripe count must be -1 for all available or greater than 0 */
    if (stripes < -1) {
        if (rank == 0) {
            MFU_LOG(MFU_LOG_ERR, "Stripe count must be -1 for all servers, 0 for lustre file system default, or a positive value");
        }
        MPI_Abort(MPI_COMM_WORLD, 1);
    }

    /* lustre requires stripe sizes to be aligned */
    if (stripe_size > 0 && stripe_size % 65536 != 0) {
        if (rank == 0) {
            MFU_LOG(MFU_LOG_ERR, "Stripe size must be a multiple of 65536");
        }
        MPI_Abort(MPI_COMM_WORLD, 1);
    }

    /* TODO: verify that source / target are on Lustre */

    /* walk list of input paths and stat as we walk */
    mfu_flist flist = mfu_flist_new();
    mfu_flist_walk_param_paths(numpaths, paths, walk_opts, flist);

    /* filter down our list to files which don't meet our striping requirements */
    mfu_flist filtered = filter_list(flist, stripes, stripe_size, min_size, &create_prog_count_total, &stripe_prog_bytes_total);
    mfu_flist_free(&flist);

    MPI_Barrier(MPI_COMM_WORLD);

    /* report the file size and stripe count of all files we found */
    if (report) {
        /* report the files in our filtered list */
        stripe_info_report(filtered);

        /* free the paths and our list */
        mfu_flist_free(&filtered);
        mfu_param_path_free_all(numpaths, paths);
        mfu_free(&paths);

        /* finalize */
        mfu_finalize();
        MPI_Finalize();
        return 0;
    }

    /* generate a global suffix for our temp files and have each node check it's list */
    char suffix[8];
    uint64_t retry;

    /* seed our random number generator */
    srand(time(NULL));

    /* keep trying to make a valid random suffix...*/
    do {
        uint64_t attempt = 0;

        /* make rank 0 responsible for generating a random suffix */
        if (rank == 0) {
            generate_suffix(suffix, sizeof(suffix));
        }

        /* broadcast the random suffix to all ranks */
        MPI_Bcast(suffix, sizeof(suffix), MPI_CHAR, 0, MPI_COMM_WORLD);

        /* check that the file doesn't already exist */
        uint64_t size = mfu_flist_size(filtered);
        for (idx = 0; idx < size; idx++) {
            char temp_path[PATH_MAX];
            strcpy(temp_path, mfu_flist_file_get_name(filtered, idx));
            strcat(temp_path, suffix);
            if(!mfu_access(temp_path, F_OK)) {
                /* the file already exists */
                attempt = 1;
                break;
            }
        }

        /* do a reduce to figure out if a rank has a file collision */
        MPI_Allreduce(&attempt, &retry, 1, MPI_UINT64_T, MPI_MAX, MPI_COMM_WORLD);
    } while(retry != 0);

    /* initialize progress messages while creating files */
    create_prog_count = 0;
    create_prog = mfu_progress_start(mfu_progress_timeout, 1, MPI_COMM_WORLD, create_progress_fn);

    /* create new files so we can restripe */
    uint64_t size = mfu_flist_size(filtered);
    for (idx = 0; idx < size; idx++) {
        char temp_path[PATH_MAX];
        strcpy(temp_path, mfu_flist_file_get_name(filtered, idx));
        strcat(temp_path, suffix);

        /* create a striped file at the temp file path */
        mfu_stripe_set(temp_path, stripe_size, stripes);

        /* update our status for file create progress */
        create_prog_count++;
        mfu_progress_update(&create_prog_count, create_prog);
    }

    /* finalize file create progress messages */
    mfu_progress_complete(&create_prog_count, &create_prog);

    MPI_Barrier(MPI_COMM_WORLD);

    /* initialize progress messages while copying data */
    stripe_prog_bytes = 0;
    stripe_prog = mfu_progress_start(mfu_progress_timeout, 1, MPI_COMM_WORLD, stripe_progress_fn);

    /* found a suffix, now we need to break our files into chunks based on stripe size */
    mfu_file_chunk* file_chunks = mfu_file_chunk_list_alloc(filtered, stripe_size);
    mfu_file_chunk* p = file_chunks;
    while (p != NULL) {
        /* build path to temp file */
        char temp_path[PATH_MAX];
        strcpy(temp_path, p->name);
        strcat(temp_path, suffix);

        /* write each chunk in our list */
        write_file_chunk(p, temp_path);

        /* move on to next file chunk */
        p = p->next;
    }
    mfu_file_chunk_list_free(&file_chunks);

    /* finalize progress messages */
    mfu_progress_complete(&stripe_prog_bytes, &stripe_prog);

    MPI_Barrier(MPI_COMM_WORLD);

    /* remove input file and rename temp file */
    for (idx = 0; idx < size; idx++) {
        /* build path to temp file */
        const char *in_path = mfu_flist_file_get_name(filtered, idx);
        char out_path[PATH_MAX];
        strcpy(out_path, in_path);
        strcat(out_path, suffix);

        /* change the mode of the newly restriped file to be the same as the old one */
        mode_t mode = (mode_t) mfu_flist_file_get_mode(filtered, idx);
        if (mfu_chmod(out_path, mode) != 0) {
            MFU_LOG(MFU_LOG_ERR, "Failed to chmod file %s (%s)", out_path, strerror(errno));
            MPI_Abort(MPI_COMM_WORLD, 1);
        }

        /* rename the new, restriped file to the old name */
        if (rename(out_path, in_path) != 0) {
            MFU_LOG(MFU_LOG_ERR, "Failed to rename file %s to %s", out_path, in_path);
            MPI_Abort(MPI_COMM_WORLD, 1);
        }
    }

    /* wait for everyone to finish */
    MPI_Barrier(MPI_COMM_WORLD);

    /* free the walk options */
    mfu_walk_opts_delete(&walk_opts);

    /* free filtered list, path parameters */
    mfu_flist_free(&filtered);
    mfu_param_path_free_all(numpaths, paths);
    mfu_free(&paths);

    mfu_finalize();
    MPI_Finalize();

    return 0;
}
Exemplo n.º 30
0
/* write a chunk of the file */
static void write_file_chunk(mfu_file_chunk* p, const char* out_path)
{
    size_t chunk_size = 1024*1024;
    uint64_t base = (off_t)p->offset;
    uint64_t file_size = (off_t)p->file_size;
    const char *in_path = p->name;
    uint64_t stripe_size = (off_t)p->length;

    /* if the file size is 0, there's no data to restripe */
    /* if the stripe size is 0, then there's no work to be done */
    if (file_size == 0 || stripe_size == 0) {
        return;
    }

    /* allocate buffer */
    void* buf = MFU_MALLOC(chunk_size);

    /* open input file for reading */
    int in_fd = mfu_open(in_path, O_RDONLY);
    if (in_fd < 0) {
        MFU_LOG(MFU_LOG_ERR, "Failed to open input file %s (%s)", in_path, strerror(errno));
        MPI_Abort(MPI_COMM_WORLD, 1);
    }

    /* open output file for writing */
    int out_fd = mfu_open(out_path, O_WRONLY);
    if (out_fd < 0) {
        MFU_LOG(MFU_LOG_ERR, "Failed to open output file %s (%s)", out_path, strerror(errno));
        MPI_Abort(MPI_COMM_WORLD, 1);
    }

    /* write data */
    uint64_t chunk_id = 0;
    uint64_t stripe_read = 0;
    while (stripe_read < stripe_size) {
        /* determine number of bytes to read */
        /* try to read a full chunk's worth of bytes */
        size_t read_size = chunk_size;

        /* if the stripe doesn't have that much left */
        uint64_t remainder = stripe_size - stripe_read;
        if (remainder < (uint64_t) read_size) {
            read_size = (size_t) remainder;
        }

        /* get byte offset to read from */
        uint64_t offset = base + (chunk_id * chunk_size);
        if (offset < file_size) {
            /* the first byte falls within the file size,
             * now check the last byte */
            uint64_t last = offset + (uint64_t) read_size;
            if (last > file_size) {
                /* the last byte is beyond the end, set read size
                 * to the most we can read */
                read_size = (size_t) (file_size - offset);
            }
        } else {
            /* the first byte we need to read is past the end of
             * the file, so don't read anything */
            read_size = 0;
        }

        /* bail if we don't have anything to read */
        if (read_size == 0) {
            break;
        }

        /* seek to correct spot in input file */
        off_t pos = (off_t) offset;
        off_t seek_rc = mfu_lseek(in_path, in_fd, pos, SEEK_SET);
        if (seek_rc == (off_t)-1) {
            MFU_LOG(MFU_LOG_ERR, "Failed to seek in input file %s (%s)", in_path, strerror(errno));
            MPI_Abort(MPI_COMM_WORLD, 1);
        }

        /* read chunk from input */
        ssize_t nread = mfu_read(in_path, in_fd, buf, read_size);

        /* check for errors */
        if (nread < 0) {
            MFU_LOG(MFU_LOG_ERR, "Failed to read data from input file %s (%s)", in_path, strerror(errno));
            MPI_Abort(MPI_COMM_WORLD, 1);
        }

        /* check for short reads */
        if (nread != read_size) {
            MFU_LOG(MFU_LOG_ERR, "Got a short read from input file %s", in_path);
            MPI_Abort(MPI_COMM_WORLD, 1);
        }

        /* seek to correct spot in output file */
        seek_rc = mfu_lseek(out_path, out_fd, pos, SEEK_SET);
        if (seek_rc == (off_t)-1) {
            MFU_LOG(MFU_LOG_ERR, "Failed to seek in output file %s (%s)", out_path, strerror(errno));
            MPI_Abort(MPI_COMM_WORLD, 1);
        }

        /* write chunk to output */
        ssize_t nwrite = mfu_write(out_path, out_fd, buf, read_size);

        /* check for errors */
        if (nwrite < 0) {
            MFU_LOG(MFU_LOG_ERR, "Failed to write data to output file %s (%s)", out_path, strerror(errno));
            MPI_Abort(MPI_COMM_WORLD, 1);
        }

        /* check for short reads */
        if (nwrite != read_size) {
            MFU_LOG(MFU_LOG_ERR, "Got a short write to output file %s", out_path);
            MPI_Abort(MPI_COMM_WORLD, 1);
        }

        /* update our byte count for progress messages */
        stripe_prog_bytes += read_size;
        mfu_progress_update(&stripe_prog_bytes, stripe_prog);

        /* go on to the next chunk in this stripe, we assume we
         * read the whole chunk size, if we didn't it's because
         * the stripe is smaller or we're at the end of the file,
         * but in either case we're done so it doesn't hurt to
         * over estimate in this calculation */
        stripe_read += (uint64_t) chunk_size;
        chunk_id++;
    }

    /* close files */
    mfu_fsync(out_path, out_fd);
    mfu_close(out_path, out_fd);
    mfu_close(in_path, in_fd);

    /* free buffer */
    mfu_free(&buf);
}