void DTAR_enqueue_copy(CIRCLE_handle *handle) { for (uint64_t idx = 0; idx < DTAR_count; idx++) { mfu_filetype type = mfu_flist_file_get_type(DTAR_flist, idx); /* add copy work only for files */ if (type == MFU_TYPE_FILE) { uint64_t dataoffset = DTAR_offsets[idx] + DTAR_HDR_LENGTH; const char * name = mfu_flist_file_get_name(DTAR_flist, idx); uint64_t size = mfu_flist_file_get_size(DTAR_flist, idx); /* compute number of chunks */ uint64_t num_chunks = size / DTAR_user_opts.chunk_size; for (uint64_t chunk_idx = 0; chunk_idx < num_chunks; chunk_idx++) { char* newop = DTAR_encode_operation( COPY_DATA, name, size, chunk_idx, dataoffset); handle->enqueue(newop); mfu_free(&newop); } /* create copy work for possibly last item */ if (num_chunks * DTAR_user_opts.chunk_size < size || num_chunks == 0) { char* newop = DTAR_encode_operation( COPY_DATA, name, size, num_chunks, dataoffset); handle->enqueue(newop); mfu_free(&newop); } } } }
int MFU_PRED_EXEC (mfu_flist flist, uint64_t idx, void* arg) { int argmax = 1024*1024;; int written = 0; int ret; char* command = MFU_STRDUP((char*) arg); char* cmdline = (char*) MFU_MALLOC(argmax); char* subst = strstr(command, "{}"); if (subst) { subst[0] = '\0'; subst += 2; /* Point to the first char after '{}' */ } const char* name = mfu_flist_file_get_name(flist, idx); written = snprintf(cmdline, argmax/sizeof(char), "%s%s%s", command, name, subst); if (written > argmax/sizeof(char)) { fprintf(stderr, "argument %s to exec too long.\n", cmdline); mfu_free(&cmdline); mfu_free(&command); return -1; } ret = system(cmdline); mfu_free(&cmdline); mfu_free(&command); return ret ? 0 : 1; }
/* append element to tail of linked list */ void mfu_flist_insert_elem(flist_t* flist, elem_t* elem) { /* set head if this is the first item */ if (flist->list_head == NULL) { flist->list_head = elem; } /* update last element to point to this new element */ elem_t* tail = flist->list_tail; if (tail != NULL) { tail->next = elem; } /* make this element the new tail */ flist->list_tail = elem; elem->next = NULL; /* increase list count by one */ flist->list_count++; /* delete the index if we have one, it's out of date */ mfu_free(&flist->list_index); return; }
void DCOPY_retry_failed_operation(DCOPY_operation_code_t target, \ CIRCLE_handle* handle, \ DCOPY_operation_t* op) { char* new_op; if(DCOPY_user_opts.reliable_filesystem) { MFU_LOG(MFU_LOG_ERR, "Not retrying failed operation. " \ "Reliable filesystem is specified. (op=%d chunk=%ld src=%s dst=%s)", target, op->chunk, op->operand, op->dest_full_path); DCOPY_abort(EXIT_FAILURE); } else { MFU_LOG(MFU_LOG_INFO, "Attempting to retry operation."); new_op = DCOPY_encode_operation(target, op->chunk, op->operand, \ op->source_base_offset, \ op->dest_base_appendix, op->file_size); handle->enqueue(new_op); mfu_free(&new_op); } return; }
/* delete linked list of stat items */ static void list_delete(flist_t* flist) { elem_t* current = flist->list_head; while (current != NULL) { elem_t* next = current->next; mfu_free(¤t->file); mfu_free(¤t); current = next; } flist->list_count = 0; flist->list_head = NULL; flist->list_tail = NULL; /* delete the cached index */ mfu_free(&flist->list_index); return; }
/* given the address of a pointer to an operation_t struct, * free associated memory and set pointer to NULL */ void DCOPY_opt_free(DCOPY_operation_t** optptr) { if(optptr != NULL) { /* get pointer to operation_t struct */ DCOPY_operation_t* opt = (*optptr); if(opt != NULL) { /* free memory and then the object itself */ mfu_free(&opt->dest_full_path); mfu_free(&opt); } /* set caller's pointer to NULL to catch bugs */ *optptr = NULL; } return; }
/* frees resources allocated in call to parse_path_args() */ void DCOPY_free_path_args(void) { /* free memory associated with destination path */ mfu_param_path_free(&dest_param); /* free memory associated with source paths */ int i; for(i = 0; i < num_src_params; i++) { mfu_param_path_free(&src_params[i]); } num_src_params = 0; mfu_free(&src_params); }
static int distribution_parse(struct distribute_option *option, const char *string) { char *ptr; char *next; unsigned long long separator; char *str; int status = 0; if (strncmp(string, "size", strlen("size")) != 0) { return -1; } option->separator_number = 0; if (strlen(string) == strlen("size")) { return 0; } if (string[strlen("size")] != ':') { return -1; } str = MFU_STRDUP(string); /* Parse separators */ ptr = str + strlen("size:"); next = ptr; while (ptr && ptr < str + strlen(string)) { next = strchr(ptr, ','); if (next != NULL) { *next = '\0'; next++; } if (mfu_abtoull(ptr, &separator) != MFU_SUCCESS) { printf("Invalid separator \"%s\"\n", ptr); status = -1; goto out; } if (distribute_separator_add(option, separator)) { printf("Duplicated separator \"%llu\"\n", separator); status = -1; goto out; } ptr = next; } out: mfu_free(&str); return status; }
/* we hash file names based on its parent directory to map all * files in the same directory to the same process */ static int map_name(mfu_flist flist, uint64_t idx, int ranks, void* args) { /* get name of item */ const char* name = mfu_flist_file_get_name(flist, idx); /* identify rank to send this file to */ char* dir = MFU_STRDUP(name); dirname(dir); size_t dir_len = strlen(dir); uint32_t hash = mfu_hash_jenkins(dir, dir_len); int rank = (int)(hash % (uint32_t)ranks); mfu_free(&dir); return rank; }
static int lustre_mds_stat(int fd, char* fname, struct stat* sb) { /* allocate a buffer */ size_t pathlen = strlen(fname) + 1; size_t bufsize = pathlen; //size_t datasize = sizeof(lstat_t) + lov_user_md_size(LOV_MAX_STRIPE_COUNT, LOV_USER_MAGIC_V3); size_t datasize = sizeof(struct lov_user_mds_data) + LOV_MAX_STRIPE_COUNT * sizeof(struct lov_user_ost_data_v1); if (datasize > bufsize) { bufsize = datasize; } char* buf = (char*) MFU_MALLOC(bufsize); /* Usage: ioctl(fd, IOC_MDC_GETFILEINFO, buf) * IN: fd open file descriptor of file's parent directory * IN: buf file name (no path) * OUT: buf lstat_t */ strcpy(buf, fname); // strncpy(buf, fname, bufsize); int ret = ioctl(fd, IOC_MDC_GETFILEINFO, buf); /* Copy lstat_t to struct stat */ if (ret != -1) { lstat_t* ls = (lstat_t*) & ((struct lov_user_mds_data*) buf)->lmd_st; sb->st_dev = ls->st_dev; sb->st_ino = ls->st_ino; sb->st_mode = ls->st_mode; sb->st_nlink = ls->st_nlink; sb->st_uid = ls->st_uid; sb->st_gid = ls->st_gid; sb->st_rdev = ls->st_rdev; sb->st_size = ls->st_size; sb->st_blksize = ls->st_blksize; sb->st_blocks = ls->st_blocks; sb->st_atime = ls->st_atime; sb->st_mtime = ls->st_mtime; sb->st_ctime = ls->st_ctime; lustre_stripe_info(buf); } else { MFU_LOG(MFU_LOG_ERR, "ioctl fd=%d (errno=%d %s)", fd, errno, strerror(errno)); } /* free the buffer */ mfu_free(&buf); return ret; }
int DCOPY_close_file(DCOPY_file_cache_t* cache) { int rc = 0; /* close file if we have one */ char* name = cache->name; if (name != NULL) { /* TODO: if open for write, fsync? */ int fd = cache->fd; rc = mfu_close(name, fd); mfu_free(&cache->name); } return rc; }
/* free resources allocated in list of params */ static void mfu_param_path_free_list(uint64_t num, mfu_param_path* params) { /* make sure we got a valid pointer */ if (params != NULL) { /* free memory for each param */ uint64_t i; for (i = 0; i < num; i++) { /* get pointer to param structure */ mfu_param_path* param = ¶ms[i]; /* free memory and reinit */ if (param != NULL) { /* free all mememory */ mfu_free(¶m->orig); mfu_free(¶m->path); mfu_free(¶m->target); /* initialize all fields */ mfu_param_path_init(param); } } } return; }
static void DTAR_perform_copy(CIRCLE_handle* handle) { char opstr[CIRCLE_MAX_STRING_LEN]; char iobuf[FD_BLOCK_SIZE]; int out_fd = DTAR_writer.fd_tar; handle->dequeue(opstr); DTAR_operation_t* op = DTAR_decode_operation(opstr); uint64_t in_offset = DTAR_user_opts.chunk_size * op->chunk_index; int in_fd = open(op->operand, O_RDONLY); ssize_t num_of_bytes_read = 0; ssize_t num_of_bytes_written = 0; ssize_t total_bytes_written = 0; uint64_t out_offset = op->offset + in_offset; lseek(in_fd, in_offset, SEEK_SET); lseek(out_fd, out_offset, SEEK_SET); while (total_bytes_written < DTAR_user_opts.chunk_size) { num_of_bytes_read = read(in_fd, &iobuf[0], sizeof(iobuf)); if (! num_of_bytes_read) { break; } num_of_bytes_written = write(out_fd, &iobuf[0], num_of_bytes_read); total_bytes_written += num_of_bytes_written; } uint64_t num_chunks = op->file_size / DTAR_user_opts.chunk_size; uint64_t rem = op->file_size - DTAR_user_opts.chunk_size * num_chunks; uint64_t last_chunk = (rem) ? num_chunks : num_chunks - 1; /* handle last chunk */ if (op->chunk_index == last_chunk) { int padding = 512 - (int) (op->file_size % 512); if (padding > 0 && padding != 512) { char* buff = (char*) calloc(padding, sizeof(char)); write(out_fd, buff, padding); } } close(in_fd); mfu_free(&op); }
int DCOPY_open_file(const char* file, int read_flag, DCOPY_file_cache_t* cache) { int newfd = -1; /* see if we have a cached file descriptor */ char* name = cache->name; if (name != NULL) { /* we have a cached file descriptor */ int fd = cache->fd; if (strcmp(name, file) == 0 && cache->read == read_flag) { /* the file we're trying to open matches name and read/write mode, * so just return the cached descriptor */ return fd; } else { /* the file we're trying to open is different, * close the old file and delete the name */ mfu_close(name, fd); mfu_free(&cache->name); } } /* open the new file */ if (read_flag) { int flags = O_RDONLY; if (DCOPY_user_opts.synchronous) { flags |= O_DIRECT; } newfd = mfu_open(file, flags); } else { int flags = O_WRONLY | O_CREAT; if (DCOPY_user_opts.synchronous) { flags |= O_DIRECT; } newfd = mfu_open(file, flags, DCOPY_DEF_PERMS_FILE); } /* cache the file descriptor */ if (newfd != -1) { cache->name = MFU_STRDUP(file); cache->fd = newfd; cache->read = read_flag; } return newfd; }
/* frees array of lists created in call to * mfu_flist_split_by_depth */ void mfu_flist_array_free(int levels, mfu_flist** outlists) { /* check that our pointer is valid */ if (outlists == NULL) { return; } /* free each list */ int i; mfu_flist* lists = *outlists; for (i = 0; i < levels; i++) { mfu_flist_free(&lists[i]); } /* free the array of lists and set caller's pointer to NULL */ mfu_free(outlists); return; }
/* free resouces in file list */ void mfu_flist_free(mfu_flist* pbflist) { /* convert handle to flist_t */ flist_t* flist = *(flist_t**)pbflist; /* delete linked list */ list_delete(flist); /* free user and group structures */ mfu_flist_usrgrp_free(flist); mfu_free(&flist); /* set caller's pointer to NULL */ *pbflist = MFU_FLIST_NULL; return; }
int DCOPY_close_file(DCOPY_file_cache_t* cache) { int rc = 0; /* close file if we have one */ char* name = cache->name; if (name != NULL) { int fd = cache->fd; /* if open for write, fsync */ int read_flag = cache->read; if (! read_flag) { rc = mfu_fsync(name, fd); } /* close the file and delete the name string */ rc = mfu_close(name, fd); mfu_free(&cache->name); } return rc; }
/* given a list of param_paths, walk each one and add to flist */ void mfu_flist_walk_param_paths(uint64_t num, const mfu_param_path* params, mfu_walk_opts_t* walk_opts, mfu_flist flist) { /* allocate memory to hold a list of paths */ const char** path_list = (const char**) MFU_MALLOC(num * sizeof(char*)); /* fill list of paths and print each one */ uint64_t i; for (i = 0; i < num; i++) { /* get path for this step */ path_list[i] = params[i].path; } /* walk file tree and record stat data for each file */ mfu_flist_walk_paths((uint64_t) num, path_list, walk_opts, flist); /* free the list */ mfu_free(&path_list); return; }
static int add_type(mfu_pred* p, char t) { mode_t* type = (mode_t*) MFU_MALLOC(sizeof(mode_t)); switch (t) { case 'b': *type = S_IFBLK; break; case 'c': *type = S_IFCHR; break; case 'd': *type = S_IFDIR; break; case 'f': *type = S_IFREG; break; case 'l': *type = S_IFLNK; break; case 'p': *type = S_IFIFO; break; case 's': *type = S_IFSOCK; break; default: /* unsupported type character */ mfu_free(&type); return -1; break; } /* add check for this type */ mfu_pred_add(p, MFU_PRED_TYPE, (void *)type); return 1; }
/* for each depth, sort files by filename and then remove, to test * whether it matters to limit the number of directories each process * has to reference (e.g., locking) */ static void remove_sort(mfu_flist list, uint64_t* rmcount) { /* bail out if total count is 0 */ uint64_t all_count = mfu_flist_global_size(list); if (all_count == 0) { return; } /* get maximum file name and number of items */ int chars = (int) mfu_flist_file_max_name(list); uint64_t my_count = mfu_flist_size(list); /* create key datatype (filename) and comparison op */ MPI_Datatype dt_key; DTCMP_Op op_str; DTCMP_Str_create_ascend(chars, &dt_key, &op_str); /* create keysat datatype (filename + type) */ MPI_Datatype types[2], dt_keysat; types[0] = dt_key; types[1] = MPI_CHAR; DTCMP_Type_create_series(2, types, &dt_keysat); /* allocate send buffer */ int sendcount = (int) my_count; size_t sendbufsize = (size_t)(sendcount * (chars + 1)); char* sendbuf = (char*) MFU_MALLOC(sendbufsize); /* copy data into buffer */ char* ptr = sendbuf; uint64_t idx; for (idx = 0; idx < my_count; idx++) { /* encode the filename first */ const char* name = mfu_flist_file_get_name(list, idx); strcpy(ptr, name); ptr += chars; /* last character encodes item type */ mfu_filetype type = mfu_flist_file_get_type(list, idx); if (type == MFU_TYPE_DIR) { ptr[0] = 'd'; } else if (type == MFU_TYPE_FILE || type == MFU_TYPE_LINK) { ptr[0] = 'f'; } else { ptr[0] = 'u'; } ptr++; } /* sort items */ void* recvbuf; int recvcount; DTCMP_Handle handle; DTCMP_Sortz( sendbuf, sendcount, &recvbuf, &recvcount, dt_key, dt_keysat, op_str, DTCMP_FLAG_NONE, MPI_COMM_WORLD, &handle ); /* delete data */ int delcount = 0; ptr = (char*)recvbuf; while (delcount < recvcount) { /* get item name */ char* name = ptr; ptr += chars; /* get item type */ char type = ptr[0]; ptr++; /* delete item */ remove_type(type, name); delcount++; } /* record number of items we deleted */ *rmcount = (uint64_t) delcount; /* free output data */ DTCMP_Free(&handle); /* free our send buffer */ mfu_free(&sendbuf); /* free key comparison operation */ DTCMP_Op_free(&op_str); /* free datatypes */ MPI_Type_free(&dt_keysat); MPI_Type_free(&dt_key); return; }
/* for given depth, hash directory name and map to processes to * test whether having all files in same directory on one process * matters */ size_t mfu_flist_distribute_map(mfu_flist list, char** buffer, mfu_flist_name_encode_fn encode, mfu_flist_map_fn map, void* args) { uint64_t idx; /* get our rank and number of ranks in job */ int ranks; MPI_Comm_size(MPI_COMM_WORLD, &ranks); /* allocate arrays for alltoall */ size_t bufsize = (size_t)ranks * sizeof(int); int* sendsizes = (int*) MFU_MALLOC(bufsize); int* senddisps = (int*) MFU_MALLOC(bufsize); int* sendoffset = (int*) MFU_MALLOC(bufsize); int* recvsizes = (int*) MFU_MALLOC(bufsize); int* recvdisps = (int*) MFU_MALLOC(bufsize); /* initialize sendsizes and offsets */ int i; for (i = 0; i < ranks; i++) { sendsizes[i] = 0; sendoffset[i] = 0; } /* compute number of bytes we'll send to each rank */ size_t sendbytes = 0; uint64_t size = mfu_flist_size(list); for (idx = 0; idx < size; idx++) { int dest = map(list, idx, ranks, args); /* TODO: check that pack size doesn't overflow int */ /* total number of bytes we'll send to each rank and the total overall */ size_t count = encode(NULL, list, idx, args); sendsizes[dest] += (int) count; sendbytes += count; } /* compute displacements */ senddisps[0] = 0; for (i = 1; i < ranks; i++) { senddisps[i] = senddisps[i - 1] + sendsizes[i - 1]; } /* allocate space */ char* sendbuf = (char*) MFU_MALLOC(sendbytes); /* copy data into buffer */ for (idx = 0; idx < size; idx++) { int dest = map(list, idx, ranks, args); /* identify region to be sent to rank */ char* path = sendbuf + senddisps[dest] + sendoffset[dest]; size_t count = encode(path, list, idx, args); /* TODO: check that pack size doesn't overflow int */ /* bump up the offset for this rank */ sendoffset[dest] += (int) count; } /* alltoall to specify incoming counts */ MPI_Alltoall(sendsizes, 1, MPI_INT, recvsizes, 1, MPI_INT, MPI_COMM_WORLD); /* compute size of recvbuf and displacements */ size_t recvbytes = 0; recvdisps[0] = 0; for (i = 0; i < ranks; i++) { recvbytes += (size_t) recvsizes[i]; if (i > 0) { recvdisps[i] = recvdisps[i - 1] + recvsizes[i - 1]; } } /* allocate recvbuf */ char* recvbuf = (char*) MFU_MALLOC(recvbytes); /* alltoallv to send data */ MPI_Alltoallv( sendbuf, sendsizes, senddisps, MPI_CHAR, recvbuf, recvsizes, recvdisps, MPI_CHAR, MPI_COMM_WORLD ); /* free memory */ mfu_free(&recvdisps); mfu_free(&recvsizes); mfu_free(&sendbuf); mfu_free(&sendoffset); mfu_free(&senddisps); mfu_free(&sendsizes); *buffer = recvbuf; return recvbytes; }
/* for given depth, evenly spread the files among processes for * improved load balancing */ static void remove_spread(mfu_flist flist, uint64_t* rmcount) { uint64_t idx; /* initialize our remove count */ *rmcount = 0; /* get our rank and number of ranks in job */ int rank, ranks; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &ranks); /* allocate memory for alltoall exchanges */ size_t bufsize = (size_t)ranks * sizeof(int); int* sendcounts = (int*) MFU_MALLOC(bufsize); int* sendsizes = (int*) MFU_MALLOC(bufsize); int* senddisps = (int*) MFU_MALLOC(bufsize); int* recvsizes = (int*) MFU_MALLOC(bufsize); int* recvdisps = (int*) MFU_MALLOC(bufsize); /* get number of items */ uint64_t my_count = mfu_flist_size(flist); uint64_t all_count = mfu_flist_global_size(flist); uint64_t offset = mfu_flist_global_offset(flist); /* compute number of bytes we'll send */ size_t sendbytes = 0; for (idx = 0; idx < my_count; idx++) { const char* name = mfu_flist_file_get_name(flist, idx); size_t len = strlen(name) + 2; sendbytes += len; } /* compute the number of items that each rank should have */ uint64_t low = all_count / (uint64_t)ranks; uint64_t extra = all_count - low * (uint64_t)ranks; /* compute number that we'll send to each rank and initialize sendsizes and offsets */ uint64_t i; for (i = 0; i < (uint64_t)ranks; i++) { /* compute starting element id and count for given rank */ uint64_t start, num; if (i < extra) { num = low + 1; start = i * num; } else { num = low; start = (i - extra) * num + extra * (low + 1); } /* compute the number of items we'll send to this task */ uint64_t sendcnt = 0; if (my_count > 0) { if (start <= offset && offset < start + num) { /* this rank overlaps our range, * and its first element comes at or before our first element */ sendcnt = num - (offset - start); if (my_count < sendcnt) { /* the number the rank could receive from us * is more than we have left */ sendcnt = my_count; } } else if (offset < start && start < offset + my_count) { /* this rank overlaps our range, * and our first element comes strictly before its first element */ sendcnt = my_count - (start - offset); if (num < sendcnt) { /* the number the rank can receive from us * is less than we have left */ sendcnt = num; } } } /* record the number of items we'll send to this task */ sendcounts[i] = (int) sendcnt; /* set sizes and displacements to 0, we'll fix this later */ sendsizes[i] = 0; senddisps[i] = 0; } /* allocate space */ char* sendbuf = (char*) MFU_MALLOC(sendbytes); /* copy data into buffer */ int dest = -1; int disp = 0; for (idx = 0; idx < my_count; idx++) { /* get name and type of item */ const char* name = mfu_flist_file_get_name(flist, idx); mfu_filetype type = mfu_flist_file_get_type(flist, idx); /* get rank that we're packing data for */ if (dest == -1) { dest = get_first_nonzero(sendcounts, ranks); if (dest == -1) { /* error */ } /* about to copy first item for this rank, * record its displacement */ senddisps[dest] = disp; } /* identify region to be sent to rank */ char* path = sendbuf + disp; /* first character encodes item type */ if (type == MFU_TYPE_DIR) { path[0] = 'd'; } else if (type == MFU_TYPE_FILE || type == MFU_TYPE_LINK) { path[0] = 'f'; } else { path[0] = 'u'; } /* now copy in the path */ strcpy(&path[1], name); /* TODO: check that we don't overflow the int */ /* add bytes to sendsizes and increase displacement */ size_t count = strlen(name) + 2; sendsizes[dest] += (int) count; disp += (int) count; /* decrement the count for this rank */ sendcounts[dest]--; if (sendcounts[dest] == 0) { dest = -1; } } /* compute displacements */ senddisps[0] = 0; for (i = 1; i < (uint64_t)ranks; i++) { senddisps[i] = senddisps[i - 1] + sendsizes[i - 1]; } /* alltoall to specify incoming counts */ MPI_Alltoall(sendsizes, 1, MPI_INT, recvsizes, 1, MPI_INT, MPI_COMM_WORLD); /* compute size of recvbuf and displacements */ size_t recvbytes = 0; recvdisps[0] = 0; for (i = 0; i < (uint64_t)ranks; i++) { recvbytes += (size_t) recvsizes[i]; if (i > 0) { recvdisps[i] = recvdisps[i - 1] + recvsizes[i - 1]; } } /* allocate recvbuf */ char* recvbuf = (char*) MFU_MALLOC(recvbytes); /* alltoallv to send data */ MPI_Alltoallv( sendbuf, sendsizes, senddisps, MPI_CHAR, recvbuf, recvsizes, recvdisps, MPI_CHAR, MPI_COMM_WORLD ); /* delete data */ char* item = recvbuf; while (item < recvbuf + recvbytes) { /* get item name and type */ char type = item[0]; char* name = &item[1]; /* delete item */ remove_type(type, name); /* keep tally of number of items we deleted */ *rmcount++; /* go to next item */ size_t item_size = strlen(item) + 1; item += item_size; } /* free memory */ mfu_free(&recvbuf); mfu_free(&recvdisps); mfu_free(&recvsizes); mfu_free(&sendbuf); mfu_free(&senddisps); mfu_free(&sendsizes); mfu_free(&sendcounts); return; }
/* check that source and destination paths are valid */ void mfu_param_path_check_copy(uint64_t num, const mfu_param_path* paths, const mfu_param_path* destpath, int* flag_valid, int* flag_copy_into_dir) { /* initialize output params */ *flag_valid = 0; *flag_copy_into_dir = 0; /* need at least two paths to have a shot at being valid */ if (num < 1 || paths == NULL || destpath == NULL) { return; } /* get current rank */ int rank; MPI_Comm_rank(MPI_COMM_WORLD, &rank); /* assume path parameters are valid */ int valid = 1; /* just have rank 0 check */ if(rank == 0) { /* count number of readable source paths */ uint64_t i; int num_readable = 0; for(i = 0; i < num; i++) { const char* path = paths[i].path; if(mfu_access(path, R_OK) == 0) { num_readable++; } else { /* found a source path that we can't read, not fatal, * but print an error to notify user */ const char* orig = paths[i].orig; MFU_LOG(MFU_LOG_ERR, "Could not read `%s' (errno=%d %s)", orig, errno, strerror(errno)); } } /* verify that we have at least one source path */ if(num_readable < 1) { MFU_LOG(MFU_LOG_ERR, "At least one valid source must be specified"); valid = 0; goto bcast; } /* * First we need to determine if the last argument is a file or a directory. * We first attempt to see if the last argument already exists on disk. If it * doesn't, we then look at the sources to see if we can determine what the * last argument should be. */ bool dest_exists = false; bool dest_is_dir = false; bool dest_is_file = false; bool dest_is_link_to_dir = false; bool dest_is_link_to_file = false; bool dest_required_to_be_dir = false; /* check whether dest exists, its type, and whether it's writable */ if(destpath->path_stat_valid) { /* we could stat dest path, so something is there */ dest_exists = true; /* now determine its type */ if(S_ISDIR(destpath->path_stat.st_mode)) { /* dest is a directory */ dest_is_dir = true; } else if(S_ISREG(destpath->path_stat.st_mode)) { /* dest is a file */ dest_is_file = true; } else if(S_ISLNK(destpath->path_stat.st_mode)) { /* dest is a symlink, but to what? */ if (destpath->target_stat_valid) { /* target of the symlink exists, determine what it is */ if(S_ISDIR(destpath->target_stat.st_mode)) { /* dest is link to a directory */ dest_is_link_to_dir = true; } else if(S_ISREG(destpath->target_stat.st_mode)) { /* dest is link to a file */ dest_is_link_to_file = true; } else { /* unsupported type */ MFU_LOG(MFU_LOG_ERR, "Unsupported filetype `%s' --> `%s'", destpath->orig, destpath->target); valid = 0; goto bcast; } } else { /* dest is a link, but its target does not exist, * consider this an error */ MFU_LOG(MFU_LOG_ERR, "Destination is broken symlink `%s'", destpath->orig); valid = 0; goto bcast; } } else { /* unsupported type */ MFU_LOG(MFU_LOG_ERR, "Unsupported filetype `%s'", destpath->orig); valid = 0; goto bcast; } /* check that dest is writable */ if(mfu_access(destpath->path, W_OK) < 0) { MFU_LOG(MFU_LOG_ERR, "Destination is not writable `%s'", destpath->path); valid = 0; goto bcast; } } else { /* destination does not exist, so we'll be creating it, * check that its parent is writable */ /* compute parent path */ mfu_path* parent = mfu_path_from_str(destpath->path); mfu_path_dirname(parent); char* parent_str = mfu_path_strdup(parent); mfu_path_delete(&parent); /* check that parent is writable */ if(mfu_access(parent_str, W_OK) < 0) { MFU_LOG(MFU_LOG_ERR, "Destination parent directory is not writable `%s'", parent_str); valid = 0; mfu_free(&parent_str); goto bcast; } mfu_free(&parent_str); } /* determine whether caller *requires* copy into dir */ /* TODO: if caller specifies dest/ or dest/. */ /* if caller specifies more than one source, * then dest has to be a directory */ if(num > 1) { dest_required_to_be_dir = true; } /* if caller requires dest to be a directory, and if dest does not * exist or it does it exist but it's not a directory, then abort */ if(dest_required_to_be_dir && (!dest_exists || (!dest_is_dir && !dest_is_link_to_dir))) { MFU_LOG(MFU_LOG_ERR, "Destination is not a directory `%s'", destpath->orig); valid = 0; goto bcast; } /* we copy into a directory if any of the following: * 1) user specified more than one source * 2) destination already exists and is a directory * 3) destination already exists and is a link to a directory */ bool copy_into_dir = (dest_required_to_be_dir || dest_is_dir || dest_is_link_to_dir); *flag_copy_into_dir = copy_into_dir ? 1 : 0; } bcast: /* get status from rank 0 */ MPI_Bcast(&valid, 1, MPI_INT, 0, MPI_COMM_WORLD); /* set valid flag */ *flag_valid = valid; /* rank 0 broadcasts whether we're copying into a directory */ MPI_Bcast(flag_copy_into_dir, 1, MPI_INT, 0, MPI_COMM_WORLD); return; }
int main(int argc, char** argv) { /* initialize MPI */ MPI_Init(&argc, &argv); mfu_init(); /* get our rank and the size of comm_world */ int rank, ranks; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &ranks); /* pointer to mfu_walk_opts */ mfu_walk_opts_t* walk_opts = mfu_walk_opts_new(); /* parse command line options */ char* inputname = NULL; char* ownername = NULL; char* groupname = NULL; char* modestr = NULL; char* regex_exp = NULL; mfu_perms* head = NULL; int walk = 0; int exclude = 0; int name = 0; /* verbose by default */ mfu_debug_level = MFU_LOG_VERBOSE; int option_index = 0; static struct option long_options[] = { {"input", 1, 0, 'i'}, {"owner", 1, 0, 'u'}, {"group", 1, 0, 'g'}, {"mode", 1, 0, 'm'}, {"exclude", 1, 0, 'e'}, {"match", 1, 0, 'a'}, {"name", 0, 0, 'n'}, {"progress", 1, 0, 'P'}, {"verbose", 0, 0, 'v'}, {"quiet", 0, 0, 'q'}, {"help", 0, 0, 'h'}, {0, 0, 0, 0} }; int usage = 0; while (1) { int c = getopt_long( argc, argv, "i:u:g:m:nvqh", long_options, &option_index ); if (c == -1) { break; } switch (c) { case 'i': inputname = MFU_STRDUP(optarg); break; case 'u': ownername = MFU_STRDUP(optarg); break; case 'g': groupname = MFU_STRDUP(optarg); break; case 'm': modestr = MFU_STRDUP(optarg); break; case 'e': regex_exp = MFU_STRDUP(optarg); exclude = 1; break; case 'a': regex_exp = MFU_STRDUP(optarg); exclude = 0; break; case 'n': name = 1; break; case 'P': mfu_progress_timeout = atoi(optarg); break; case 'v': mfu_debug_level = MFU_LOG_VERBOSE; break; case 'q': mfu_debug_level = MFU_LOG_NONE; break; case 'h': usage = 1; break; case '?': usage = 1; break; default: if (rank == 0) { printf("?? getopt returned character code 0%o ??\n", c); } } } /* check that we got a valid progress value */ if (mfu_progress_timeout < 0) { if (rank == 0) { MFU_LOG(MFU_LOG_ERR, "Seconds in --progress must be non-negative: %d invalid", mfu_progress_timeout); } usage = 1; } /* paths to walk come after the options */ int numpaths = 0; mfu_param_path* paths = NULL; if (optind < argc) { /* got a path to walk */ walk = 1; /* determine number of paths specified by user */ numpaths = argc - optind; /* allocate space for each path */ paths = (mfu_param_path*) MFU_MALLOC((size_t)numpaths * sizeof(mfu_param_path)); /* process each path */ const char** argpaths = (const char**)(&argv[optind]); mfu_param_path_set_all(numpaths, argpaths, paths); /* advance to next set of options */ optind += numpaths; /* don't allow input file and walk */ if (inputname != NULL) { usage = 1; } } else { /* if we're not walking, we must be reading, * and for that we need a file */ if (inputname == NULL) { usage = 1; } } /* check that our mode string parses correctly */ if (modestr != NULL) { int valid = mfu_perms_parse(modestr, &head); if (! valid) { usage = 1; if (rank == 0) { printf("invalid mode string: %s\n", modestr); } /* free the head of the list */ mfu_perms_free(&head); } } /* print usage if we need to */ if (usage) { if (rank == 0) { print_usage(); } mfu_finalize(); MPI_Finalize(); return 1; } /* create an empty file list */ mfu_flist flist = mfu_flist_new(); /* flag used to check if permissions need to be * set on the walk */ if (head != NULL) { mfu_perms_need_dir_rx(head, walk_opts); } /* get our list of files, either by walking or reading an * input file */ if (walk) { /* if in octal mode set use_stat=0 to stat each file on walk */ if (head != NULL && head->octal && ownername == NULL && groupname == NULL) { walk_opts->use_stat = 0; } /* walk list of input paths */ mfu_flist_walk_param_paths(numpaths, paths, walk_opts, flist); } else { /* read list from file */ mfu_flist_read_cache(inputname, flist); } /* assume we'll use the full list */ mfu_flist srclist = flist; /* filter the list if needed */ mfu_flist filtered_flist = MFU_FLIST_NULL; if (regex_exp != NULL) { /* filter the list based on regex */ filtered_flist = mfu_flist_filter_regex(flist, regex_exp, exclude, name); /* update our source list to use the filtered list instead of the original */ srclist = filtered_flist; } /* change group and permissions */ mfu_flist_chmod(srclist, ownername, groupname, head); /* free list if it was used */ if (filtered_flist != MFU_FLIST_NULL){ /* free the filtered flist (if any) */ mfu_flist_free(&filtered_flist); } /* free the file list */ mfu_flist_free(&flist); /* free the path parameters */ mfu_param_path_free_all(numpaths, paths); /* free memory allocated to hold params */ mfu_free(&paths); /* free the owner and group names */ mfu_free(&ownername); mfu_free(&groupname); /* free the modestr */ mfu_free(&modestr); /* free the match_pattern if it isn't null */ if (regex_exp != NULL) { mfu_free(®ex_exp); } /* free the head of the list */ mfu_perms_free(&head); /* free the input file name */ mfu_free(&inputname); /* free the walk options */ mfu_walk_opts_delete(&walk_opts); /* shut down MPI */ mfu_finalize(); MPI_Finalize(); return 0; }
int main(int argc, char** argv) { int i; /* initialize MPI */ MPI_Init(&argc, &argv); mfu_init(); /* get our rank and the size of comm_world */ int rank, ranks; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &ranks); /* parse command line options */ char* inputname = NULL; char* regex_exp = NULL; int walk = 0; int exclude = 0; int name = 0; int dryrun = 0; int option_index = 0; static struct option long_options[] = { {"input", 1, 0, 'i'}, {"lite", 0, 0, 'l'}, {"exclude", 1, 0, 'e'}, {"match", 1, 0, 'a'}, {"name", 0, 0, 'n'}, {"help", 0, 0, 'h'}, {"dryrun", 0, 0, 'd'}, {"verbose", 0, 0, 'v'}, {0, 0, 0, 0} }; int usage = 0; while (1) { int c = getopt_long( argc, argv, "i:nlhv", long_options, &option_index ); if (c == -1) { break; } switch (c) { case 'i': inputname = MFU_STRDUP(optarg); break; case 'l': walk_stat = 0; break; case 'e': regex_exp = MFU_STRDUP(optarg); exclude = 1; break; case 'a': regex_exp = MFU_STRDUP(optarg); exclude = 0; break; case 'n': name = 1; break; case 'h': usage = 1; break; case 'd': dryrun = 1; break; case 'v': mfu_debug_level = MFU_LOG_VERBOSE; break; case '?': usage = 1; break; default: if (rank == 0) { printf("?? getopt returned character code 0%o ??\n", c); } } } /* paths to walk come after the options */ int numpaths = 0; mfu_param_path* paths = NULL; if (optind < argc) { /* got a path to walk */ walk = 1; /* determine number of paths specified by user */ numpaths = argc - optind; /* allocate space for each path */ paths = (mfu_param_path*) MFU_MALLOC((size_t)numpaths * sizeof(mfu_param_path)); /* process each path */ char** argpaths = &argv[optind]; mfu_param_path_set_all(numpaths, argpaths, paths); /* advance to next set of options */ optind += numpaths; /* don't allow input file and walk */ if (inputname != NULL) { usage = 1; } } else { /* if we're not walking, we must be reading, * and for that we need a file */ if (inputname == NULL) { usage = 1; } } /* print usage if we need to */ if (usage) { if (rank == 0) { print_usage(); } mfu_finalize(); MPI_Finalize(); return 1; } /* create an empty file list */ mfu_flist flist = mfu_flist_new(); /* get our list of files, either by walking or reading an * input file */ if (walk) { /* walk list of input paths */ mfu_param_path_walk(numpaths, paths, walk_stat, flist, dir_perm); } else { /* read list from file */ mfu_flist_read_cache(inputname, flist); } /* assume we'll use the full list */ mfu_flist srclist = flist; /* filter the list if needed */ mfu_flist filtered_flist = MFU_FLIST_NULL; if (regex_exp != NULL) { /* filter the list based on regex */ filtered_flist = mfu_flist_filter_regex(flist, regex_exp, exclude, name); /* update our source list to use the filtered list instead of the original */ srclist = filtered_flist; } /* only actually delete files if the user wasn't doing a dry run */ if (dryrun) { /* just print what we would delete without actually doing anything, * this is useful if the user is trying to get a regex right */ mfu_flist_print(srclist); } else { /* remove files */ mfu_flist_unlink(srclist); } /* free list if it was used */ if (filtered_flist != MFU_FLIST_NULL){ /* free the filtered flist (if any) */ mfu_flist_free(&filtered_flist); } /* free the file list */ mfu_flist_free(&flist); /* free the path parameters */ mfu_param_path_free_all(numpaths, paths); /* free memory allocated to hold params */ mfu_free(&paths); /* free the regex string if we have one */ mfu_free(®ex_exp); /* free the input file name */ mfu_free(&inputname); /* shut down MPI */ mfu_finalize(); MPI_Finalize(); return 0; }
int main (int argc, char** argv) { /* initialize MPI */ MPI_Init(&argc, &argv); mfu_init(); /* get our rank and the size of comm_world */ int rank, ranks; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &ranks); /* pointer to mfu_walk_opts */ mfu_walk_opts_t* walk_opts = mfu_walk_opts_new(); /* capture current time for any time based queries, * to get a consistent value, capture and bcast from rank 0 */ mfu_pred_times* now_t = mfu_pred_now(); int ch; mfu_pred* pred_head = mfu_pred_new(); char* inputname = NULL; char* outputname = NULL; int walk = 0; int text = 0; static struct option long_options[] = { {"input", 1, 0, 'i'}, {"output", 1, 0, 'o'}, {"verbose", 0, 0, 'v'}, {"quiet", 0, 0, 'q'}, {"help", 0, 0, 'h'}, { "maxdepth", required_argument, NULL, 'd' }, { "amin", required_argument, NULL, 'a' }, { "anewer", required_argument, NULL, 'B' }, { "atime", required_argument, NULL, 'A' }, { "cmin", required_argument, NULL, 'c' }, { "cnewer", required_argument, NULL, 'D' }, { "ctime", required_argument, NULL, 'C' }, { "mmin", required_argument, NULL, 'm' }, { "newer", required_argument, NULL, 'N' }, { "mtime", required_argument, NULL, 'M' }, { "gid", required_argument, NULL, 'g' }, { "group", required_argument, NULL, 'G' }, { "uid", required_argument, NULL, 'u' }, { "user", required_argument, NULL, 'U' }, { "name", required_argument, NULL, 'n' }, { "path", required_argument, NULL, 'P' }, { "regex", required_argument, NULL, 'r' }, { "size", required_argument, NULL, 's' }, { "type", required_argument, NULL, 't' }, { "print", no_argument, NULL, 'p' }, { "exec", required_argument, NULL, 'e' }, { NULL, 0, NULL, 0 }, }; options.maxdepth = INT_MAX; int usage = 0; while (1) { int c = getopt_long( argc, argv, "i:o:vqh", long_options, NULL ); if (c == -1) { break; } int i; int space; char* buf; mfu_pred_times* t; mfu_pred_times_rel* tr; regex_t* r; int ret; /* verbose by default */ mfu_debug_level = MFU_LOG_VERBOSE; switch (c) { case 'e': space = 1024 * 1024; buf = (char *)MFU_MALLOC(space); for (i = optind-1; strcmp(";", argv[i]); i++) { if (i > argc) { if (rank == 0) { printf("%s: exec missing terminating ';'\n", argv[0]); } exit(1); } strncat(buf, argv[i], space); space -= strlen(argv[i]) + 1; /* save room for space or null */ if (space <= 0) { if (rank == 0) { printf("%s: exec argument list too long.\n", argv[0]); } mfu_free(&buf); continue; } strcat(buf, " "); optind++; } buf[strlen(buf)] = '\0'; /* clobbers trailing space */ mfu_pred_add(pred_head, MFU_PRED_EXEC, buf); break; case 'd': options.maxdepth = atoi(optarg); break; case 'g': /* TODO: error check argument */ buf = MFU_STRDUP(optarg); mfu_pred_add(pred_head, MFU_PRED_GID, (void *)buf); break; case 'G': buf = MFU_STRDUP(optarg); mfu_pred_add(pred_head, MFU_PRED_GROUP, (void *)buf); break; case 'u': /* TODO: error check argument */ buf = MFU_STRDUP(optarg); mfu_pred_add(pred_head, MFU_PRED_UID, (void *)buf); break; case 'U': buf = MFU_STRDUP(optarg); mfu_pred_add(pred_head, MFU_PRED_USER, (void *)buf); break; case 's': buf = MFU_STRDUP(optarg); mfu_pred_add(pred_head, MFU_PRED_SIZE, (void *)buf); break; case 'n': mfu_pred_add(pred_head, MFU_PRED_NAME, MFU_STRDUP(optarg)); break; case 'P': mfu_pred_add(pred_head, MFU_PRED_PATH, MFU_STRDUP(optarg)); break; case 'r': r = (regex_t*) MFU_MALLOC(sizeof(regex_t)); ret = regcomp(r, optarg, 0); if (ret) { MFU_ABORT(-1, "Could not compile regex: `%s' rc=%d\n", optarg, ret); } mfu_pred_add(pred_head, MFU_PRED_REGEX, (void*)r); break; case 'a': tr = mfu_pred_relative(optarg, now_t); mfu_pred_add(pred_head, MFU_PRED_AMIN, (void *)tr); break; case 'm': tr = mfu_pred_relative(optarg, now_t); mfu_pred_add(pred_head, MFU_PRED_MMIN, (void *)tr); break; case 'c': tr = mfu_pred_relative(optarg, now_t); mfu_pred_add(pred_head, MFU_PRED_CMIN, (void *)tr); break; case 'A': tr = mfu_pred_relative(optarg, now_t); mfu_pred_add(pred_head, MFU_PRED_ATIME, (void *)tr); break; case 'M': tr = mfu_pred_relative(optarg, now_t); mfu_pred_add(pred_head, MFU_PRED_MTIME, (void *)tr); break; case 'C': tr = mfu_pred_relative(optarg, now_t); mfu_pred_add(pred_head, MFU_PRED_CTIME, (void *)tr); break; case 'B': t = get_mtimes(optarg); if (t == NULL) { if (rank == 0) { printf("%s: can't find file %s\n", argv[0], optarg); } exit(1); } mfu_pred_add(pred_head, MFU_PRED_ANEWER, (void *)t); break; case 'N': t = get_mtimes(optarg); if (t == NULL) { if (rank == 0) { printf("%s: can't find file %s\n", argv[0], optarg); } exit(1); } mfu_pred_add(pred_head, MFU_PRED_MNEWER, (void *)t); break; case 'D': t = get_mtimes(optarg); if (t == NULL) { if (rank == 0) { printf("%s: can't find file %s\n", argv[0], optarg); } exit(1); } mfu_pred_add(pred_head, MFU_PRED_CNEWER, (void *)t); break; case 'p': mfu_pred_add(pred_head, MFU_PRED_PRINT, NULL); break; case 't': ret = add_type(pred_head, *optarg); if (ret != 1) { if (rank == 0) { printf("%s: unsupported file type %s\n", argv[0], optarg); } exit(1); } break; case 'i': inputname = MFU_STRDUP(optarg); break; case 'o': outputname = MFU_STRDUP(optarg); break; case 'v': mfu_debug_level = MFU_LOG_VERBOSE; break; case 'q': mfu_debug_level = MFU_LOG_NONE; break; case 'h': usage = 1; break; case '?': usage = 1; break; default: if (rank == 0) { printf("?? getopt returned character code 0%o ??\n", c); } } } pred_commit(pred_head); /* paths to walk come after the options */ int numpaths = 0; mfu_param_path* paths = NULL; if (optind < argc) { /* got a path to walk */ walk = 1; /* determine number of paths specified by user */ numpaths = argc - optind; /* allocate space for each path */ paths = (mfu_param_path*) MFU_MALLOC((size_t)numpaths * sizeof(mfu_param_path)); /* process each path */ char** p = &argv[optind]; mfu_param_path_set_all((uint64_t)numpaths, (const char**)p, paths); optind += numpaths; /* don't allow user to specify input file with walk */ if (inputname != NULL) { usage = 1; } } else { /* if we're not walking, we must be reading, * and for that we need a file */ if (inputname == NULL) { usage = 1; } } if (usage) { if (rank == 0) { print_usage(); } mfu_finalize(); MPI_Finalize(); return 0; } /* create an empty file list */ mfu_flist flist = mfu_flist_new(); if (walk) { /* walk list of input paths */ mfu_flist_walk_param_paths(numpaths, paths, walk_opts, flist); } else { /* read data from cache file */ mfu_flist_read_cache(inputname, flist); } /* apply predicates to each item in list */ mfu_flist flist2 = mfu_flist_filter_pred(flist, pred_head); /* write data to cache file */ if (outputname != NULL) { if (!text) { mfu_flist_write_cache(outputname, flist2); } else { mfu_flist_write_text(outputname, flist2); } } /* free off the filtered list */ mfu_flist_free(&flist2); /* free users, groups, and files objects */ mfu_flist_free(&flist); /* free predicate list */ mfu_pred_free(&pred_head); /* free memory allocated for options */ mfu_free(&outputname); mfu_free(&inputname); /* free the path parameters */ mfu_param_path_free_all(numpaths, paths); /* free memory allocated to hold params */ mfu_free(&paths); /* free structure holding current time */ mfu_free(&now_t); /* free the walk options */ mfu_walk_opts_delete(&walk_opts); /* shut down MPI */ mfu_finalize(); MPI_Finalize(); return 0; }
void mfu_param_path_set_all(uint64_t num, const char** paths, mfu_param_path* params) { /* get our rank and number of ranks */ int rank, ranks; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &ranks); /* determine number we should look up */ uint64_t count = num / (uint64_t) ranks; uint64_t extra = num - count * (uint64_t) ranks; if (rank < (int) extra) { /* procs whose rank is less than extra each * handle one extra param than those whose * rank is equal or greater than extra */ count++; } /* determine our starting point */ uint64_t start = 0; if (rank < (int) extra) { /* for these procs, count is one more than procs with ranks >= extra */ start = (uint64_t)rank * count; } else { /* for these procs, count is one less than procs with ranks < extra */ start = extra * (count + 1) + ((uint64_t)rank - extra) * count; } /* TODO: allocate temporary params */ mfu_param_path* p = MFU_MALLOC(count * sizeof(mfu_param_path)); /* track maximum path length */ uint64_t bytes = 0; /* process each path we're responsible for */ uint64_t i; for (i = 0; i < count; i++) { /* get pointer to param structure */ mfu_param_path* param = &p[i]; /* initialize all fields */ mfu_param_path_init(param); /* lookup the path */ uint64_t path_idx = start + i; const char* path = paths[path_idx]; /* set param fields for path */ if (path != NULL) { /* make a copy of original path */ param->orig = MFU_STRDUP(path); /* get absolute path and remove ".", "..", consecutive "/", * and trailing "/" characters */ param->path = mfu_path_strdup_abs_reduce_str(path); /* get stat info for simplified path */ if (mfu_lstat(param->path, ¶m->path_stat) == 0) { param->path_stat_valid = 1; } /* TODO: we use realpath below, which is nice since it takes out * ".", "..", symlinks, and adds the absolute path, however, it * fails if the file/directory does not already exist, which is * often the case for dest path. */ /* resolve any symlinks */ char target[PATH_MAX]; if (realpath(path, target) != NULL) { /* make a copy of resolved name */ param->target = MFU_STRDUP(target); /* get stat info for resolved path */ if (mfu_lstat(param->target, ¶m->target_stat) == 0) { param->target_stat_valid = 1; } } /* add in bytes needed to pack this param */ bytes += (uint64_t) mfu_pack_param_size(param); } } /* TODO: eventually it would be nice to leave this data distributed, * however for now some tools expect all params to be defined */ /* allgather to get bytes on each process */ int* recvcounts = (int*) MFU_MALLOC(ranks * sizeof(int)); int* recvdispls = (int*) MFU_MALLOC(ranks * sizeof(int)); int sendcount = (int) bytes; MPI_Allgather(&sendcount, 1, MPI_INT, recvcounts, 1, MPI_INT, MPI_COMM_WORLD); /* compute displacements and total number of bytes that we'll receive */ uint64_t allbytes = 0; int disp = 0; for (i = 0; i < (uint64_t) ranks; i++) { recvdispls[i] = disp; disp += recvcounts[i]; allbytes += (uint64_t) recvcounts[i]; } /* allocate memory for send and recv buffers */ char* sendbuf = MFU_MALLOC(bytes); char* recvbuf = MFU_MALLOC(allbytes); /* pack send buffer */ char* ptr = sendbuf; for (i = 0; i < count; i++) { mfu_pack_param(&ptr, &p[i]); } /* allgatherv to collect data */ MPI_Allgatherv(sendbuf, sendcount, MPI_BYTE, recvbuf, recvcounts, recvdispls, MPI_BYTE, MPI_COMM_WORLD); /* unpack recv buffer into caller's params */ ptr = recvbuf; for (i = 0; i < num; i++) { mfu_unpack_param((const char**)(&ptr), ¶ms[i]); } /* Loop through the list of files &/or directories, and check the params * struct to see if all of them are valid file names. If one is not, let * the user know by printing a warning */ if (rank == 0) { for (i = 0; i < num; i++) { /* get pointer to param structure */ mfu_param_path* param = ¶ms[i]; if (param->path_stat_valid == 0) { /* failed to find a file at this location, let user know (may be a typo) */ MFU_LOG(MFU_LOG_WARN, "Warning: `%s' does not exist", param->orig); } } } /* free message buffers */ mfu_free(&recvbuf); mfu_free(&sendbuf); /* free arrays for recv counts and displacements */ mfu_free(&recvdispls); mfu_free(&recvcounts); /* free temporary params */ mfu_param_path_free_list(count, p); mfu_free(&p); return; }
static int sort_files_readdir(const char* sortfields, mfu_flist* pflist) { /* get list from caller */ mfu_flist flist = *pflist; /* create a new list as subset of original list */ mfu_flist flist2 = mfu_flist_subset(flist); uint64_t incount = mfu_flist_size(flist); uint64_t chars = mfu_flist_file_max_name(flist); /* create datatype for packed file list element */ MPI_Datatype dt_sat; size_t bytes = mfu_flist_file_pack_size(flist); MPI_Type_contiguous((int)bytes, MPI_BYTE, &dt_sat); /* get our rank and the size of comm_world */ int rank, ranks; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &ranks); /* build type for file path */ MPI_Datatype dt_filepath; MPI_Type_contiguous((int)chars, MPI_CHAR, &dt_filepath); MPI_Type_commit(&dt_filepath); /* build comparison op for filenames */ DTCMP_Op op_filepath; if (DTCMP_Op_create(dt_filepath, my_strcmp, &op_filepath) != DTCMP_SUCCESS) { MFU_ABORT(1, "Failed to create sorting operation for filepath"); } /* build comparison op for filenames */ DTCMP_Op op_filepath_rev; if (DTCMP_Op_create(dt_filepath, my_strcmp_rev, &op_filepath_rev) != DTCMP_SUCCESS) { MFU_ABORT(1, "Failed to create reverse sorting operation for filepath"); } /* TODO: process sort fields */ const int MAXFIELDS = 1; MPI_Datatype types[MAXFIELDS]; DTCMP_Op ops[MAXFIELDS]; sort_field fields[MAXFIELDS]; size_t lengths[MAXFIELDS]; int nfields = 0; for (nfields = 0; nfields < MAXFIELDS; nfields++) { types[nfields] = MPI_DATATYPE_NULL; ops[nfields] = DTCMP_OP_NULL; } nfields = 0; char* sortfields_copy = MFU_STRDUP(sortfields); char* token = strtok(sortfields_copy, ","); while (token != NULL) { int valid = 1; if (strcmp(token, "name") == 0) { types[nfields] = dt_filepath; ops[nfields] = op_filepath; fields[nfields] = FILENAME; lengths[nfields] = chars; } else if (strcmp(token, "-name") == 0) { types[nfields] = dt_filepath; ops[nfields] = op_filepath_rev; fields[nfields] = FILENAME; lengths[nfields] = chars; } else { /* invalid token */ valid = 0; if (rank == 0) { MFU_LOG(MFU_LOG_ERR, "Invalid sort field: %s\n", token); } } if (valid) { nfields++; } if (nfields > MAXFIELDS) { /* TODO: print warning if we have too many fields */ break; } token = strtok(NULL, ","); } mfu_free(&sortfields_copy); /* build key type */ MPI_Datatype dt_key; if (DTCMP_Type_create_series(nfields, types, &dt_key) != DTCMP_SUCCESS) { MFU_ABORT(1, "Failed to create type for key"); } /* create sort op */ DTCMP_Op op_key; if (DTCMP_Op_create_series(nfields, ops, &op_key) != DTCMP_SUCCESS) { MFU_ABORT(1, "Failed to create sorting operation for key"); } /* build keysat type */ MPI_Datatype dt_keysat, keysat_types[2]; keysat_types[0] = dt_key; keysat_types[1] = dt_sat; if (DTCMP_Type_create_series(2, keysat_types, &dt_keysat) != DTCMP_SUCCESS) { MFU_ABORT(1, "Failed to create type for keysat"); } /* get extent of key type */ MPI_Aint key_lb, key_extent; MPI_Type_get_extent(dt_key, &key_lb, &key_extent); /* get extent of keysat type */ MPI_Aint keysat_lb, keysat_extent; MPI_Type_get_extent(dt_keysat, &keysat_lb, &keysat_extent); /* get extent of sat type */ MPI_Aint sat_lb, sat_extent; MPI_Type_get_extent(dt_sat, &sat_lb, &sat_extent); /* compute size of sort element and allocate buffer */ size_t sortbufsize = (size_t)keysat_extent * incount; void* sortbuf = MFU_MALLOC(sortbufsize); /* copy data into sort elements */ uint64_t idx = 0; char* sortptr = (char*) sortbuf; while (idx < incount) { /* copy in access time */ int i; for (i = 0; i < nfields; i++) { if (fields[i] == FILENAME) { const char* name = mfu_flist_file_get_name(flist, idx); strcpy(sortptr, name); } sortptr += lengths[i]; } /* pack file element */ sortptr += mfu_flist_file_pack(sortptr, flist, idx); idx++; } /* sort data */ void* outsortbuf; int outsortcount; DTCMP_Handle handle; int sort_rc = DTCMP_Sortz( sortbuf, (int)incount, &outsortbuf, &outsortcount, dt_key, dt_keysat, op_key, DTCMP_FLAG_NONE, MPI_COMM_WORLD, &handle ); if (sort_rc != DTCMP_SUCCESS) { MFU_ABORT(1, "Failed to sort data"); } /* step through sorted data filenames */ idx = 0; sortptr = (char*) outsortbuf; while (idx < (uint64_t)outsortcount) { sortptr += key_extent; sortptr += mfu_flist_file_unpack(sortptr, flist2); idx++; } /* build summary of new list */ mfu_flist_summarize(flist2); /* free memory */ DTCMP_Free(&handle); /* free ops */ DTCMP_Op_free(&op_key); DTCMP_Op_free(&op_filepath_rev); DTCMP_Op_free(&op_filepath); /* free types */ MPI_Type_free(&dt_keysat); MPI_Type_free(&dt_key); MPI_Type_free(&dt_filepath); /* free input buffer holding sort elements */ mfu_free(&sortbuf); /* free the satellite type */ MPI_Type_free(&dt_sat); /* return new list and free old one */ *pflist = flist2; mfu_flist_free(&flist); return MFU_SUCCESS; }
int main(int argc, char* argv[]) { MPI_Init(&argc, &argv); mfu_init(); /* get our rank and number of ranks in the job */ int rank, ranks; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &ranks); /* pointer to mfu_walk_opts */ mfu_walk_opts_t* walk_opts = mfu_walk_opts_new(); uint64_t idx; int option_index = 0; int usage = 0; int report = 0; unsigned int numpaths = 0; mfu_param_path* paths = NULL; unsigned long long bytes; /* verbose by default */ mfu_debug_level = MFU_LOG_VERBOSE; /* default to 1MB stripe size, stripe across all OSTs, and all files are candidates */ int stripes = -1; uint64_t stripe_size = 1048576; uint64_t min_size = 0; static struct option long_options[] = { {"count", 1, 0, 'c'}, {"size", 1, 0, 's'}, {"minsize", 1, 0, 'm'}, {"report", 0, 0, 'r'}, {"progress", 1, 0, 'P'}, {"verbose", 0, 0, 'v'}, {"quiet", 0, 0, 'q'}, {"help", 0, 0, 'h'}, {0, 0, 0, 0} }; while (1) { int c = getopt_long(argc, argv, "c:s:m:rvqh", long_options, &option_index); if (c == -1) { break; } switch (c) { case 'c': /* stripe count */ stripes = atoi(optarg); break; case 's': /* stripe size in bytes */ if (mfu_abtoull(optarg, &bytes) != MFU_SUCCESS) { if (rank == 0) { MFU_LOG(MFU_LOG_ERR, "Failed to parse stripe size: %s", optarg); } MPI_Abort(MPI_COMM_WORLD, 1); } stripe_size = (uint64_t)bytes; break; case 'm': /* min file size in bytes */ if (mfu_abtoull(optarg, &bytes) != MFU_SUCCESS) { if (rank == 0) { MFU_LOG(MFU_LOG_ERR, "Failed to parse minimum file size: %s", optarg); } MPI_Abort(MPI_COMM_WORLD, 1); } min_size = (uint64_t)bytes; break; case 'r': /* report striping info */ report = 1; break; case 'P': mfu_progress_timeout = atoi(optarg); break; case 'v': mfu_debug_level = MFU_LOG_VERBOSE; break; case 'q': mfu_debug_level = MFU_LOG_NONE; break; case 'h': /* display usage */ usage = 1; break; case '?': /* display usage */ usage = 1; break; default: if (rank == 0) { printf("?? getopt returned character code 0%o ??\n", c); } } } /* check that we got a valid progress value */ if (mfu_progress_timeout < 0) { if (rank == 0) { MFU_LOG(MFU_LOG_ERR, "Seconds in --progress must be non-negative: %d invalid", mfu_progress_timeout); } usage = 1; } /* paths to walk come after the options */ if (optind < argc) { /* determine number of paths specified by user */ numpaths = argc - optind; /* allocate space for each path */ paths = (mfu_param_path*) MFU_MALLOC((size_t)numpaths * sizeof(mfu_param_path)); /* process each path */ char** p = &argv[optind]; mfu_param_path_set_all((uint64_t)numpaths, (const char**)p, paths); optind += numpaths; } else { usage = 1; } /* if we need to print usage, print it and exit */ if (usage) { if (rank == 0) { print_usage(); } mfu_finalize(); MPI_Finalize(); return 1; } /* nothing to do if lustre support is disabled */ #ifndef LUSTRE_SUPPORT if (rank == 0) { MFU_LOG(MFU_LOG_ERR, "Lustre support is disabled."); } MPI_Abort(MPI_COMM_WORLD, 1); #endif /* stripe count must be -1 for all available or greater than 0 */ if (stripes < -1) { if (rank == 0) { MFU_LOG(MFU_LOG_ERR, "Stripe count must be -1 for all servers, 0 for lustre file system default, or a positive value"); } MPI_Abort(MPI_COMM_WORLD, 1); } /* lustre requires stripe sizes to be aligned */ if (stripe_size > 0 && stripe_size % 65536 != 0) { if (rank == 0) { MFU_LOG(MFU_LOG_ERR, "Stripe size must be a multiple of 65536"); } MPI_Abort(MPI_COMM_WORLD, 1); } /* TODO: verify that source / target are on Lustre */ /* walk list of input paths and stat as we walk */ mfu_flist flist = mfu_flist_new(); mfu_flist_walk_param_paths(numpaths, paths, walk_opts, flist); /* filter down our list to files which don't meet our striping requirements */ mfu_flist filtered = filter_list(flist, stripes, stripe_size, min_size, &create_prog_count_total, &stripe_prog_bytes_total); mfu_flist_free(&flist); MPI_Barrier(MPI_COMM_WORLD); /* report the file size and stripe count of all files we found */ if (report) { /* report the files in our filtered list */ stripe_info_report(filtered); /* free the paths and our list */ mfu_flist_free(&filtered); mfu_param_path_free_all(numpaths, paths); mfu_free(&paths); /* finalize */ mfu_finalize(); MPI_Finalize(); return 0; } /* generate a global suffix for our temp files and have each node check it's list */ char suffix[8]; uint64_t retry; /* seed our random number generator */ srand(time(NULL)); /* keep trying to make a valid random suffix...*/ do { uint64_t attempt = 0; /* make rank 0 responsible for generating a random suffix */ if (rank == 0) { generate_suffix(suffix, sizeof(suffix)); } /* broadcast the random suffix to all ranks */ MPI_Bcast(suffix, sizeof(suffix), MPI_CHAR, 0, MPI_COMM_WORLD); /* check that the file doesn't already exist */ uint64_t size = mfu_flist_size(filtered); for (idx = 0; idx < size; idx++) { char temp_path[PATH_MAX]; strcpy(temp_path, mfu_flist_file_get_name(filtered, idx)); strcat(temp_path, suffix); if(!mfu_access(temp_path, F_OK)) { /* the file already exists */ attempt = 1; break; } } /* do a reduce to figure out if a rank has a file collision */ MPI_Allreduce(&attempt, &retry, 1, MPI_UINT64_T, MPI_MAX, MPI_COMM_WORLD); } while(retry != 0); /* initialize progress messages while creating files */ create_prog_count = 0; create_prog = mfu_progress_start(mfu_progress_timeout, 1, MPI_COMM_WORLD, create_progress_fn); /* create new files so we can restripe */ uint64_t size = mfu_flist_size(filtered); for (idx = 0; idx < size; idx++) { char temp_path[PATH_MAX]; strcpy(temp_path, mfu_flist_file_get_name(filtered, idx)); strcat(temp_path, suffix); /* create a striped file at the temp file path */ mfu_stripe_set(temp_path, stripe_size, stripes); /* update our status for file create progress */ create_prog_count++; mfu_progress_update(&create_prog_count, create_prog); } /* finalize file create progress messages */ mfu_progress_complete(&create_prog_count, &create_prog); MPI_Barrier(MPI_COMM_WORLD); /* initialize progress messages while copying data */ stripe_prog_bytes = 0; stripe_prog = mfu_progress_start(mfu_progress_timeout, 1, MPI_COMM_WORLD, stripe_progress_fn); /* found a suffix, now we need to break our files into chunks based on stripe size */ mfu_file_chunk* file_chunks = mfu_file_chunk_list_alloc(filtered, stripe_size); mfu_file_chunk* p = file_chunks; while (p != NULL) { /* build path to temp file */ char temp_path[PATH_MAX]; strcpy(temp_path, p->name); strcat(temp_path, suffix); /* write each chunk in our list */ write_file_chunk(p, temp_path); /* move on to next file chunk */ p = p->next; } mfu_file_chunk_list_free(&file_chunks); /* finalize progress messages */ mfu_progress_complete(&stripe_prog_bytes, &stripe_prog); MPI_Barrier(MPI_COMM_WORLD); /* remove input file and rename temp file */ for (idx = 0; idx < size; idx++) { /* build path to temp file */ const char *in_path = mfu_flist_file_get_name(filtered, idx); char out_path[PATH_MAX]; strcpy(out_path, in_path); strcat(out_path, suffix); /* change the mode of the newly restriped file to be the same as the old one */ mode_t mode = (mode_t) mfu_flist_file_get_mode(filtered, idx); if (mfu_chmod(out_path, mode) != 0) { MFU_LOG(MFU_LOG_ERR, "Failed to chmod file %s (%s)", out_path, strerror(errno)); MPI_Abort(MPI_COMM_WORLD, 1); } /* rename the new, restriped file to the old name */ if (rename(out_path, in_path) != 0) { MFU_LOG(MFU_LOG_ERR, "Failed to rename file %s to %s", out_path, in_path); MPI_Abort(MPI_COMM_WORLD, 1); } } /* wait for everyone to finish */ MPI_Barrier(MPI_COMM_WORLD); /* free the walk options */ mfu_walk_opts_delete(&walk_opts); /* free filtered list, path parameters */ mfu_flist_free(&filtered); mfu_param_path_free_all(numpaths, paths); mfu_free(&paths); mfu_finalize(); MPI_Finalize(); return 0; }
/* write a chunk of the file */ static void write_file_chunk(mfu_file_chunk* p, const char* out_path) { size_t chunk_size = 1024*1024; uint64_t base = (off_t)p->offset; uint64_t file_size = (off_t)p->file_size; const char *in_path = p->name; uint64_t stripe_size = (off_t)p->length; /* if the file size is 0, there's no data to restripe */ /* if the stripe size is 0, then there's no work to be done */ if (file_size == 0 || stripe_size == 0) { return; } /* allocate buffer */ void* buf = MFU_MALLOC(chunk_size); /* open input file for reading */ int in_fd = mfu_open(in_path, O_RDONLY); if (in_fd < 0) { MFU_LOG(MFU_LOG_ERR, "Failed to open input file %s (%s)", in_path, strerror(errno)); MPI_Abort(MPI_COMM_WORLD, 1); } /* open output file for writing */ int out_fd = mfu_open(out_path, O_WRONLY); if (out_fd < 0) { MFU_LOG(MFU_LOG_ERR, "Failed to open output file %s (%s)", out_path, strerror(errno)); MPI_Abort(MPI_COMM_WORLD, 1); } /* write data */ uint64_t chunk_id = 0; uint64_t stripe_read = 0; while (stripe_read < stripe_size) { /* determine number of bytes to read */ /* try to read a full chunk's worth of bytes */ size_t read_size = chunk_size; /* if the stripe doesn't have that much left */ uint64_t remainder = stripe_size - stripe_read; if (remainder < (uint64_t) read_size) { read_size = (size_t) remainder; } /* get byte offset to read from */ uint64_t offset = base + (chunk_id * chunk_size); if (offset < file_size) { /* the first byte falls within the file size, * now check the last byte */ uint64_t last = offset + (uint64_t) read_size; if (last > file_size) { /* the last byte is beyond the end, set read size * to the most we can read */ read_size = (size_t) (file_size - offset); } } else { /* the first byte we need to read is past the end of * the file, so don't read anything */ read_size = 0; } /* bail if we don't have anything to read */ if (read_size == 0) { break; } /* seek to correct spot in input file */ off_t pos = (off_t) offset; off_t seek_rc = mfu_lseek(in_path, in_fd, pos, SEEK_SET); if (seek_rc == (off_t)-1) { MFU_LOG(MFU_LOG_ERR, "Failed to seek in input file %s (%s)", in_path, strerror(errno)); MPI_Abort(MPI_COMM_WORLD, 1); } /* read chunk from input */ ssize_t nread = mfu_read(in_path, in_fd, buf, read_size); /* check for errors */ if (nread < 0) { MFU_LOG(MFU_LOG_ERR, "Failed to read data from input file %s (%s)", in_path, strerror(errno)); MPI_Abort(MPI_COMM_WORLD, 1); } /* check for short reads */ if (nread != read_size) { MFU_LOG(MFU_LOG_ERR, "Got a short read from input file %s", in_path); MPI_Abort(MPI_COMM_WORLD, 1); } /* seek to correct spot in output file */ seek_rc = mfu_lseek(out_path, out_fd, pos, SEEK_SET); if (seek_rc == (off_t)-1) { MFU_LOG(MFU_LOG_ERR, "Failed to seek in output file %s (%s)", out_path, strerror(errno)); MPI_Abort(MPI_COMM_WORLD, 1); } /* write chunk to output */ ssize_t nwrite = mfu_write(out_path, out_fd, buf, read_size); /* check for errors */ if (nwrite < 0) { MFU_LOG(MFU_LOG_ERR, "Failed to write data to output file %s (%s)", out_path, strerror(errno)); MPI_Abort(MPI_COMM_WORLD, 1); } /* check for short reads */ if (nwrite != read_size) { MFU_LOG(MFU_LOG_ERR, "Got a short write to output file %s", out_path); MPI_Abort(MPI_COMM_WORLD, 1); } /* update our byte count for progress messages */ stripe_prog_bytes += read_size; mfu_progress_update(&stripe_prog_bytes, stripe_prog); /* go on to the next chunk in this stripe, we assume we * read the whole chunk size, if we didn't it's because * the stripe is smaller or we're at the end of the file, * but in either case we're done so it doesn't hurt to * over estimate in this calculation */ stripe_read += (uint64_t) chunk_size; chunk_id++; } /* close files */ mfu_fsync(out_path, out_fd); mfu_close(out_path, out_fd); mfu_close(in_path, in_fd); /* free buffer */ mfu_free(&buf); }