static void DTAR_write_header(struct archive* ar, uint64_t idx, uint64_t offset) { /* allocate and entry for this item */ struct archive_entry* entry = archive_entry_new(); /* get file name for this item */ /* fill up entry, FIXME: the uglyness of removing leading slash */ const char* fname = mfu_flist_file_get_name(DTAR_flist, idx); archive_entry_copy_pathname(entry, &fname[1]); if (DTAR_user_opts.preserve) { struct archive* source = archive_read_disk_new(); archive_read_disk_set_standard_lookup(source); int fd = open(fname, O_RDONLY); if (archive_read_disk_entry_from_file(source, entry, fd, NULL) != ARCHIVE_OK) { MFU_LOG(MFU_LOG_ERR, "archive_read_disk_entry_from_file(): %s", archive_error_string(ar)); } archive_read_free(source); close(fd); } else { /* TODO: read stat info from mfu_flist */ struct stat stbuf; mfu_lstat(fname, &stbuf); archive_entry_copy_stat(entry, &stbuf); /* set user name of owner */ const char* uname = mfu_flist_file_get_username(DTAR_flist, idx); archive_entry_set_uname(entry, uname); /* set group name */ const char* gname = mfu_flist_file_get_groupname(DTAR_flist, idx); archive_entry_set_gname(entry, gname); } /* TODO: Seems to be a bug here potentially leading to corrupted * archive files. archive_write_free also writes two blocks of * NULL bytes at the end of an archive file, however, each rank * will have a different view of the length of the file, so one * rank may write its NULL blocks over top of the actual data * written by another rank */ /* write entry info to archive */ struct archive* dest = archive_write_new(); archive_write_set_format_pax(dest); if (archive_write_open_fd(dest, DTAR_writer.fd_tar) != ARCHIVE_OK) { MFU_LOG(MFU_LOG_ERR, "archive_write_open_fd(): %s", archive_error_string(ar)); } /* seek to offset in tar archive for this file */ lseek(DTAR_writer.fd_tar, offset, SEEK_SET); /* write header for this item */ if (archive_write_header(dest, entry) != ARCHIVE_OK) { MFU_LOG(MFU_LOG_ERR, "archive_write_header(): %s", archive_error_string(ar)); } archive_entry_free(entry); archive_write_free(dest); }
void DTAR_writer_init() { char compress=DTAR_user_opts->compress; char filename=DTAR_user_opts->dest_path; DTAR_writer->flags= O_WRONLY | O_CREAT | O_TRUNC | O_BINARY | O_CLOEXEC; DTAR_writer->a=archive_write_new(); switch (compress) { case 'j': case 'y': archive_write_add_filter_bzip2(a); break; case 'Z': archive_write_add_filter_compress(a); break; case 'z': archive_write_add_filter_gzip(a); break; default: archive_write_add_filter_none(a); break; } archive_write_set_format_ustar(a); archive_write_set_bytes_per_block(a,0); if (strcmp(filename, "-") == 0) filename = NULL; DTAR_writer->fd_tar=open(filename,DTAR_writer->flags,0666); archive_write_open_fd(a, DTAR_writer->fd_tar); }
int archive_write_open_filename_w(struct archive *a, const wchar_t *filename) { if (filename == NULL || filename[0] == L'\0') return (archive_write_open_fd(a, 1)); return (open_filename(a, 0, filename)); }
int archive_write_open_filename(struct archive *a, const char *filename) { if (filename == NULL || filename[0] == '\0') return (archive_write_open_fd(a, 1)); return (open_filename(a, 1, filename)); }
int repodata_flush(struct xbps_handle *xhp, const char *repodir, xbps_dictionary_t idx, xbps_dictionary_t idxfiles) { struct archive *ar; mode_t myumask; char *repofile, *tname, *xml; int repofd; /* Create a tempfile for our repository archive */ repofile = xbps_repo_path(xhp, repodir); tname = xbps_xasprintf("%s.XXXXXXXXXX", repofile); if ((repofd = mkstemp(tname)) == -1) return errno; /* Create and write our repository archive */ ar = archive_write_new(); assert(ar); archive_write_set_compression_gzip(ar); archive_write_set_format_pax_restricted(ar); archive_write_set_options(ar, "compression-level=9"); archive_write_open_fd(ar, repofd); xml = xbps_dictionary_externalize(idx); assert(xml); if (xbps_archive_append_buf(ar, xml, strlen(xml), XBPS_PKGINDEX, 0644, "root", "root") != 0) { free(xml); return -1; } free(xml); xml = xbps_dictionary_externalize(idxfiles); assert(xml); if (xbps_archive_append_buf(ar, xml, strlen(xml), XBPS_PKGINDEX_FILES, 0644, "root", "root") != 0) { free(xml); return -1; } free(xml); archive_write_finish(ar); /* Write data to tempfile and rename */ fdatasync(repofd); myumask = umask(0); (void)umask(myumask); assert(fchmod(repofd, 0666 & ~myumask) != -1); close(repofd); rename(tname, repofile); free(repofile); free(tname); return 0; }
struct archive * DTAR_new_archive() { struct archive *a = archive_write_new(); archive_write_set_format_pax(a); int r = archive_write_open_fd(a, DTAR_writer.fd_tar); if ( r != ARCHIVE_OK) { MFU_LOG(MFU_LOG_ERR, "archive_write_open_fd(): %s", archive_error_string(a)); return NULL; } return a; }
void DTAR_write_header(struct archive *ar, uint64_t idx, uint64_t offset) { const char * fname = mfu_flist_file_get_name(DTAR_flist, idx); /* fill up entry, FIXME: the uglyness of removing leading slash */ struct archive_entry *entry = archive_entry_new(); archive_entry_copy_pathname(entry, &fname[1]); if (DTAR_user_opts.preserve) { struct archive * source = archive_read_disk_new(); archive_read_disk_set_standard_lookup(source); int fd = open(fname, O_RDONLY); if (archive_read_disk_entry_from_file(source, entry, fd, NULL) != ARCHIVE_OK) { MFU_LOG(MFU_LOG_ERR, "archive_read_disk_entry_from_file(): %s", archive_error_string(ar)); } archive_read_free(source); } else { /* read stat info from mfu_flist */ struct stat stbuf; mfu_lstat(fname, &stbuf); archive_entry_copy_stat(entry, &stbuf); const char* uname = mfu_flist_file_get_username(DTAR_flist, idx); archive_entry_set_uname(entry, uname); const char* gname = mfu_flist_file_get_groupname(DTAR_flist, idx); archive_entry_set_gname(entry, gname); } /* write entry info to archive */ struct archive* dest = archive_write_new(); archive_write_set_format_pax(dest); if (archive_write_open_fd(dest, DTAR_writer.fd_tar) != ARCHIVE_OK) { MFU_LOG(MFU_LOG_ERR, "archive_write_open_fd(): %s", archive_error_string(ar)); } lseek64(DTAR_writer.fd_tar, offset, SEEK_SET); if (archive_write_header(dest, entry) != ARCHIVE_OK) { MFU_LOG(MFU_LOG_ERR, "archive_write_header(): %s", archive_error_string(ar)); } archive_entry_free(entry); archive_write_free(dest); }
static db_writer_t *db_writer_new(repo_t *repo, file_t *db) { db_writer_t *writer = malloc(sizeof(db_writer_t)); writer->archive = archive_write_new(); writer->entry = archive_entry_new(); switch (repo->compression) { case COMPRESS_NONE: archive_write_add_filter_none(writer->archive); break; case COMPRESS_GZIP: archive_write_add_filter_gzip(writer->archive); break; case COMPRESS_BZIP2: archive_write_add_filter_bzip2(writer->archive); break; case COMPRESS_XZ: archive_write_add_filter_xz(writer->archive); break; case COMPRESS_COMPRESS: archive_write_add_filter_compress(writer->archive); break; } archive_write_set_format_pax_restricted(writer->archive); writer->fd = openat(repo->dirfd, db->file, O_CREAT | O_WRONLY | O_TRUNC, 0644); if (writer->fd < 0) err(EXIT_FAILURE, "failed to open %s for writing", db->file); if (flock(writer->fd, LOCK_EX) < 0) err(EXIT_FAILURE, "failed to lock %s", db->file); archive_write_open_fd(writer->archive, writer->fd); buffer_init(&writer->buf, 1024); return writer; }
void tar_mode_u(struct bsdtar *bsdtar) { int64_t end_offset; struct archive *a; struct archive_entry *entry; int format; struct archive_dir_entry *p; struct archive_dir archive_dir; bsdtar->archive_dir = &archive_dir; memset(&archive_dir, 0, sizeof(archive_dir)); format = ARCHIVE_FORMAT_TAR_PAX_RESTRICTED; /* Sanity-test some arguments and the file. */ test_for_append(bsdtar); bsdtar->fd = open(bsdtar->filename, O_RDWR | O_BINARY); if (bsdtar->fd < 0) lafe_errc(1, errno, "Cannot open %s", bsdtar->filename); a = archive_read_new(); archive_read_support_filter_all(a); archive_read_support_format_tar(a); archive_read_support_format_gnutar(a); set_reader_options(bsdtar, a); if (archive_read_open_fd(a, bsdtar->fd, bsdtar->bytes_per_block) != ARCHIVE_OK) { lafe_errc(1, 0, "Can't open %s: %s", bsdtar->filename, archive_error_string(a)); } /* Build a list of all entries and their recorded mod times. */ while (0 == archive_read_next_header(a, &entry)) { if (archive_filter_code(a, 0) != ARCHIVE_FILTER_NONE) { archive_read_free(a); close(bsdtar->fd); lafe_errc(1, 0, "Cannot append to compressed archive."); } if (archive_match_exclude_entry(bsdtar->matching, ARCHIVE_MATCH_MTIME | ARCHIVE_MATCH_OLDER | ARCHIVE_MATCH_EQUAL, entry) != ARCHIVE_OK) lafe_errc(1, 0, "Error : %s", archive_error_string(bsdtar->matching)); /* Record the last format determination we see */ format = archive_format(a); /* Keep going until we hit end-of-archive */ } end_offset = archive_read_header_position(a); archive_read_free(a); /* Re-open archive for writing. */ a = archive_write_new(); /* * Set format to same one auto-detected above. */ archive_write_set_format(a, format); archive_write_set_bytes_per_block(a, bsdtar->bytes_per_block); archive_write_set_bytes_in_last_block(a, bsdtar->bytes_in_last_block); if (lseek(bsdtar->fd, end_offset, SEEK_SET) < 0) lafe_errc(1, errno, "Could not seek to archive end"); set_writer_options(bsdtar, a); if (ARCHIVE_OK != archive_write_open_fd(a, bsdtar->fd)) lafe_errc(1, 0, "%s", archive_error_string(a)); write_archive(a, bsdtar); close(bsdtar->fd); bsdtar->fd = -1; while (bsdtar->archive_dir->head != NULL) { p = bsdtar->archive_dir->head->next; free(bsdtar->archive_dir->head->name); free(bsdtar->archive_dir->head); bsdtar->archive_dir->head = p; } bsdtar->archive_dir->tail = NULL; }
/* * Same as 'c', except we only support tar or empty formats in * uncompressed files on disk. */ void tar_mode_r(struct bsdtar *bsdtar) { int64_t end_offset; int format; struct archive *a; struct archive_entry *entry; int r; /* Sanity-test some arguments and the file. */ test_for_append(bsdtar); format = ARCHIVE_FORMAT_TAR_PAX_RESTRICTED; #if defined(__BORLANDC__) bsdtar->fd = open(bsdtar->filename, O_RDWR | O_CREAT | O_BINARY); #else bsdtar->fd = open(bsdtar->filename, O_RDWR | O_CREAT | O_BINARY, 0666); #endif if (bsdtar->fd < 0) lafe_errc(1, errno, "Cannot open %s", bsdtar->filename); a = archive_read_new(); archive_read_support_filter_all(a); archive_read_support_format_empty(a); archive_read_support_format_tar(a); archive_read_support_format_gnutar(a); set_reader_options(bsdtar, a); r = archive_read_open_fd(a, bsdtar->fd, 10240); if (r != ARCHIVE_OK) lafe_errc(1, archive_errno(a), "Can't read archive %s: %s", bsdtar->filename, archive_error_string(a)); while (0 == archive_read_next_header(a, &entry)) { if (archive_filter_code(a, 0) != ARCHIVE_FILTER_NONE) { archive_read_free(a); close(bsdtar->fd); lafe_errc(1, 0, "Cannot append to compressed archive."); } /* Keep going until we hit end-of-archive */ format = archive_format(a); } end_offset = archive_read_header_position(a); archive_read_free(a); /* Re-open archive for writing */ a = archive_write_new(); /* * Set the format to be used for writing. To allow people to * extend empty files, we need to allow them to specify the format, * which opens the possibility that they will specify a format that * doesn't match the existing format. Hence, the following bit * of arcane ugliness. */ if (cset_get_format(bsdtar->cset) != NULL) { /* If the user requested a format, use that, but ... */ archive_write_set_format_by_name(a, cset_get_format(bsdtar->cset)); /* ... complain if it's not compatible. */ format &= ARCHIVE_FORMAT_BASE_MASK; if (format != (int)(archive_format(a) & ARCHIVE_FORMAT_BASE_MASK) && format != ARCHIVE_FORMAT_EMPTY) { lafe_errc(1, 0, "Format %s is incompatible with the archive %s.", cset_get_format(bsdtar->cset), bsdtar->filename); } } else { /* * Just preserve the current format, with a little care * for formats that libarchive can't write. */ if (format == ARCHIVE_FORMAT_EMPTY) format = ARCHIVE_FORMAT_TAR_PAX_RESTRICTED; archive_write_set_format(a, format); } if (lseek(bsdtar->fd, end_offset, SEEK_SET) < 0) lafe_errc(1, errno, "Could not seek to archive end"); set_writer_options(bsdtar, a); if (ARCHIVE_OK != archive_write_open_fd(a, bsdtar->fd)) lafe_errc(1, 0, "%s", archive_error_string(a)); write_archive(a, bsdtar); /* XXX check return val XXX */ close(bsdtar->fd); bsdtar->fd = -1; }
static int process_package(rpmts ts, char * filename) { FD_t fdi; FD_t gzdi; Header h; int rc = 0; char * rpmio_flags = NULL; struct archive *a; struct archive_entry *entry; if (!strcmp(filename, "-")) { fdi = fdDup(STDIN_FILENO); } else { fdi = Fopen(filename, "r.ufdio"); } if (Ferror(fdi)) { fprintf(stderr, "rpm2archive: %s: %s\n", filename, Fstrerror(fdi)); exit(EXIT_FAILURE); } rc = rpmReadPackageFile(ts, fdi, "rpm2cpio", &h); switch (rc) { case RPMRC_OK: case RPMRC_NOKEY: case RPMRC_NOTTRUSTED: break; case RPMRC_NOTFOUND: fprintf(stderr, _("argument is not an RPM package\n")); exit(EXIT_FAILURE); break; case RPMRC_FAIL: default: fprintf(stderr, _("error reading header from package\n")); exit(EXIT_FAILURE); break; } /* Retrieve payload size and compression type. */ { const char *compr = headerGetString(h, RPMTAG_PAYLOADCOMPRESSOR); rpmio_flags = rstrscat(NULL, "r.", compr ? compr : "gzip", NULL); } gzdi = Fdopen(fdi, rpmio_flags); /* XXX gzdi == fdi */ free(rpmio_flags); if (gzdi == NULL) { fprintf(stderr, _("cannot re-open payload: %s\n"), Fstrerror(gzdi)); exit(EXIT_FAILURE); } rpmfiles files = rpmfilesNew(NULL, h, 0, RPMFI_KEEPHEADER); rpmfi fi = rpmfiNewArchiveReader(gzdi, files, RPMFI_ITER_READ_ARCHIVE_CONTENT_FIRST); /* create archive */ a = archive_write_new(); archive_write_add_filter_gzip(a); archive_write_set_format_pax_restricted(a); if (!strcmp(filename, "-")) { if (isatty(STDOUT_FILENO)) { fprintf(stderr, "Error: refusing to output archive data to a terminal.\n"); exit(EXIT_FAILURE); } archive_write_open_fd(a, STDOUT_FILENO); } else { char * outname = rstrscat(NULL, filename, ".tgz", NULL); archive_write_open_filename(a, outname); _free(outname); // XXX error handling } entry = archive_entry_new(); char * buf = xmalloc(BUFSIZE); char * hardlink = NULL; rc = 0; while (rc >= 0) { rc = rpmfiNext(fi); if (rc == RPMERR_ITER_END) { break; } rpm_mode_t mode = rpmfiFMode(fi); int nlink = rpmfiFNlink(fi); fill_archive_entry(a, entry, fi); if (nlink > 1) { if (rpmfiArchiveHasContent(fi)) { _free(hardlink); hardlink = rstrscat(NULL, ".", rpmfiFN(fi), NULL); } else { archive_entry_set_hardlink(entry, hardlink); } } archive_write_header(a, entry); if (S_ISREG(mode) && (nlink == 1 || rpmfiArchiveHasContent(fi))) { write_file_content(a, buf, fi); } } /* End of iteration is not an error */ if (rc == RPMERR_ITER_END) { rc = 0; } _free(hardlink); Fclose(gzdi); /* XXX gzdi == fdi */ archive_entry_free(entry); archive_write_close(a); archive_write_free(a); buf = _free(buf); rpmfilesFree(files); rpmfiFree(fi); headerFree(h); return rc; }
int main(int argc, char **argv) { int i; unsigned int version = 0, help = 0; char cpiotmpfile[] = CPIOTMPFILE; struct archive *archive; struct archive_entry *entry; struct stat st; char buff[64]; int len, fdfile, fdarchive; DIR * dir; struct dirent * ent; char * filename, * path; off_t pathlength = 0; int8_t rc = EXIT_FAILURE; /* get command line options */ while ((i = getopt_long(argc, argv, optstring, options_long, NULL)) != -1) switch (i) { case 'h': help++; break; case 'V': version++; break; } if (version > 0) printf("%s: %s v%s (compiled: " __DATE__ ", " __TIME__ ")\n", argv[0], PROGNAME, VERSION); if (help > 0) fprintf(stderr, "usage: %s [-h|--help] [-V|--version]\n", argv[0]); if (version > 0 || help > 0) return EXIT_SUCCESS; if ((fdarchive = mkstemp(cpiotmpfile)) < 0) { perror("mkstemp() failed"); goto out10; } if ((archive = archive_write_new()) == NULL) { fprintf(stderr, "archive_write_new() failed.\n"); goto out10; } if (archive_write_set_format_cpio_newc(archive) != ARCHIVE_OK) { fprintf(stderr, "archive_write_set_format_cpio_newc() failed.\n"); goto out10; } if (archive_write_open_fd(archive, fdarchive) != ARCHIVE_OK) { fprintf(stderr, "archive_write_open_fd() failed.\n"); goto out10; } while (1) { path = strdup(CHALLENGEDIR + 1); if (strstr(path + pathlength, "/") == NULL) break; *strstr(path + pathlength, "/") = 0; pathlength = strlen(path) + 1; if (add_dir(archive, path) < 0) { fprintf(stderr, "add_dir() failed"); goto out10; } free(path); } if ((dir = opendir(CHALLENGEDIR)) == NULL) { perror("opendir() failed"); goto out10; } while ((ent = readdir(dir)) != NULL) { filename = malloc(sizeof(CHALLENGEDIR) + strlen(ent->d_name) + 1); sprintf(filename, CHALLENGEDIR "%s", ent->d_name); if (stat(filename, &st) < 0) { perror("stat() failed"); goto out10; } if (S_ISREG(st.st_mode)) { if ((entry = archive_entry_new()) == NULL) { fprintf(stderr, "archive_entry_new() failed.\n"); goto out10; } /* these do not return exit code */ archive_entry_set_pathname(entry, filename + 1); archive_entry_set_size(entry, st.st_size); archive_entry_set_filetype(entry, AE_IFREG); archive_entry_set_perm(entry, 0644); if (archive_write_header(archive, entry) != ARCHIVE_OK) { fprintf(stderr, "archive_write_header() failed"); goto out10; } if ((fdfile = open(filename, O_RDONLY)) < 0) { perror("open() failed"); goto out10; } if ((len = read(fdfile, buff, sizeof(buff))) < 0) { perror("read() failed"); goto out10; } while (len > 0) { if (archive_write_data(archive, buff, len) < 0) { fprintf(stderr, "archive_write_data() failed"); goto out10; } if ((len = read(fdfile, buff, sizeof(buff))) < 0) { perror("read() failed"); goto out10; } } if (close(fdfile) < 0) { perror("close() failed"); goto out10; } archive_entry_free(entry); } free(filename); } if (closedir(dir) < 0) { perror("closedir() failed"); goto out10; } if (archive_write_close(archive) != ARCHIVE_OK) { fprintf(stderr, "archive_write_close() failed"); goto out10; } if (archive_write_free(archive) != ARCHIVE_OK) { fprintf(stderr, "archive_write_free() failed"); goto out10; } if (access(CPIOFILE, F_OK) == 0 && unlink(CPIOFILE) < 0) { perror("unkink() failed"); goto out10; } if (rename(cpiotmpfile, CPIOFILE) < 0) { perror("rename() failed"); goto out10; } rc = EXIT_SUCCESS; out10: if (access(cpiotmpfile, F_OK) == 0) unlink(cpiotmpfile); return rc; }
static void mfu_flist_archive_create_libcircle(mfu_flist flist, const char* archivefile, mfu_archive_options_t* opts) { DTAR_flist = flist; DTAR_user_opts = *opts; MPI_Comm_rank(MPI_COMM_WORLD, &DTAR_rank); /* TODO: stripe the archive file if on parallel file system */ /* init statistics */ DTAR_statistics.total_dirs = 0; DTAR_statistics.total_files = 0; DTAR_statistics.total_links = 0; DTAR_statistics.total_size = 0; DTAR_statistics.total_bytes_copied = 0; time(&(DTAR_statistics.time_started)); DTAR_statistics.wtime_started = MPI_Wtime(); /* create the archive file */ DTAR_writer.name = archivefile; DTAR_writer.flags = O_WRONLY | O_CREAT | O_CLOEXEC | O_LARGEFILE; DTAR_writer.fd_tar = open(archivefile, DTAR_writer.flags, 0664); /* get number of items in our portion of the list */ DTAR_count = mfu_flist_size(DTAR_flist); /* allocate memory for file sizes and offsets */ uint64_t* fsizes = (uint64_t*) MFU_MALLOC(DTAR_count * sizeof(uint64_t)); DTAR_offsets = (uint64_t*) MFU_MALLOC(DTAR_count * sizeof(uint64_t)); /* compute local offsets for each item and total * bytes we're contributing to the archive */ uint64_t idx; uint64_t offset = 0; for (idx = 0; idx < DTAR_count; idx++) { /* assume the item takes no space */ fsizes[idx] = 0; /* identify item type to compute its size in the archive */ mfu_filetype type = mfu_flist_file_get_type(DTAR_flist, idx); if (type == MFU_TYPE_DIR || type == MFU_TYPE_LINK) { /* directories and symlinks only need the header */ fsizes[idx] = DTAR_HDR_LENGTH; } else if (type == MFU_TYPE_FILE) { /* regular file requires a header, plus file content, * and things are packed into blocks of 512 bytes */ uint64_t fsize = mfu_flist_file_get_size(DTAR_flist, idx); /* determine whether file size is integer multiple of 512 bytes */ uint64_t rem = fsize % 512; if (rem == 0) { /* file content is multiple of 512 bytes, so perfect fit */ fsizes[idx] = fsize + DTAR_HDR_LENGTH; } else { /* TODO: check and explain this math */ fsizes[idx] = (fsize / 512 + 4) * 512; } } /* increment our local offset for this item */ DTAR_offsets[idx] = offset; offset += fsizes[idx]; } /* execute scan to figure our global base offset in the archive file */ uint64_t global_offset = 0; MPI_Scan(&offset, &global_offset, 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD); global_offset -= offset; /* update offsets for each of our file to their global offset */ for (idx = 0; idx < DTAR_count; idx++) { DTAR_offsets[idx] += global_offset; } /* create an archive */ struct archive* ar = archive_write_new(); archive_write_set_format_pax(ar); int r = archive_write_open_fd(ar, DTAR_writer.fd_tar); if (r != ARCHIVE_OK) { MFU_LOG(MFU_LOG_ERR, "archive_write_open_fd(): %s", archive_error_string(ar)); DTAR_abort(EXIT_FAILURE); } /* write headers for our files */ for (idx = 0; idx < DTAR_count; idx++) { mfu_filetype type = mfu_flist_file_get_type(DTAR_flist, idx); if (type == MFU_TYPE_FILE || type == MFU_TYPE_DIR || type == MFU_TYPE_LINK) { DTAR_write_header(ar, idx, DTAR_offsets[idx]); } } /* prepare libcircle */ CIRCLE_init(0, NULL, CIRCLE_SPLIT_EQUAL | CIRCLE_CREATE_GLOBAL); CIRCLE_loglevel loglevel = CIRCLE_LOG_WARN; CIRCLE_enable_logging(loglevel); /* register callbacks */ CIRCLE_cb_create(&DTAR_enqueue_copy); CIRCLE_cb_process(&DTAR_perform_copy); /* run the libcircle job to copy data into archive file */ CIRCLE_begin(); CIRCLE_finalize(); /* compute total bytes copied */ uint64_t archive_size = 0; MPI_Allreduce(&offset, &archive_size, 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD); DTAR_statistics.total_size = archive_size; DTAR_statistics.wtime_ended = MPI_Wtime(); time(&(DTAR_statistics.time_ended)); /* print stats */ double rel_time = DTAR_statistics.wtime_ended - \ DTAR_statistics.wtime_started; if (DTAR_rank == 0) { char starttime_str[256]; struct tm* localstart = localtime(&(DTAR_statistics.time_started)); strftime(starttime_str, 256, "%b-%d-%Y, %H:%M:%S", localstart); char endtime_str[256]; struct tm* localend = localtime(&(DTAR_statistics.time_ended)); strftime(endtime_str, 256, "%b-%d-%Y, %H:%M:%S", localend); /* add two 512 blocks at the end */ DTAR_statistics.total_size += 512 * 2; /* convert bandwidth to unit */ double agg_rate_tmp; double agg_rate = (double) DTAR_statistics.total_size / rel_time; const char* agg_rate_units; mfu_format_bytes(agg_rate, &agg_rate_tmp, &agg_rate_units); MFU_LOG(MFU_LOG_INFO, "Started: %s", starttime_str); MFU_LOG(MFU_LOG_INFO, "Completed: %s", endtime_str); MFU_LOG(MFU_LOG_INFO, "Total archive size: %" PRIu64, DTAR_statistics.total_size); MFU_LOG(MFU_LOG_INFO, "Rate: %.3lf %s " \ "(%.3" PRIu64 " bytes in %.3lf seconds)", \ agg_rate_tmp, agg_rate_units, DTAR_statistics.total_size, rel_time); } /* clean up */ mfu_free(&fsizes); mfu_free(&DTAR_offsets); /* close archive file */ archive_write_free(ar); mfu_close(DTAR_writer.name, DTAR_writer.fd_tar); }
void tar_mode_u(struct bsdtar *bsdtar) { off_t end_offset; struct archive *a; struct archive_entry *entry; int format; struct archive_dir_entry *p; struct archive_dir archive_dir; bsdtar->archive_dir = &archive_dir; memset(&archive_dir, 0, sizeof(archive_dir)); format = ARCHIVE_FORMAT_TAR_PAX_RESTRICTED; /* Sanity-test some arguments and the file. */ test_for_append(bsdtar); bsdtar->fd = open(bsdtar->filename, O_RDWR); if (bsdtar->fd < 0) bsdtar_errc(bsdtar, 1, errno, "Cannot open %s", bsdtar->filename); a = archive_read_new(); archive_read_support_compression_all(a); archive_read_support_format_tar(a); archive_read_support_format_gnutar(a); if (archive_read_open_fd(a, bsdtar->fd, bsdtar->bytes_per_block != 0 ? bsdtar->bytes_per_block : DEFAULT_BYTES_PER_BLOCK) != ARCHIVE_OK) { bsdtar_errc(bsdtar, 1, 0, "Can't open %s: %s", bsdtar->filename, archive_error_string(a)); } /* Build a list of all entries and their recorded mod times. */ while (0 == archive_read_next_header(a, &entry)) { if (archive_compression(a) != ARCHIVE_COMPRESSION_NONE) { archive_read_finish(a); close(bsdtar->fd); bsdtar_errc(bsdtar, 1, 0, "Cannot append to compressed archive."); } add_dir_list(bsdtar, archive_entry_pathname(entry), archive_entry_mtime(entry), archive_entry_mtime_nsec(entry)); /* Record the last format determination we see */ format = archive_format(a); /* Keep going until we hit end-of-archive */ } end_offset = archive_read_header_position(a); archive_read_finish(a); /* Re-open archive for writing. */ a = archive_write_new(); archive_write_set_compression_none(a); /* * Set format to same one auto-detected above, except that * we don't write GNU tar format, so use ustar instead. */ if (format == ARCHIVE_FORMAT_TAR_GNUTAR) format = ARCHIVE_FORMAT_TAR_USTAR; archive_write_set_format(a, format); if (bsdtar->bytes_per_block != 0) { archive_write_set_bytes_per_block(a, bsdtar->bytes_per_block); archive_write_set_bytes_in_last_block(a, bsdtar->bytes_per_block); } else archive_write_set_bytes_per_block(a, DEFAULT_BYTES_PER_BLOCK); lseek(bsdtar->fd, end_offset, SEEK_SET); ftruncate(bsdtar->fd, end_offset); if (ARCHIVE_OK != archive_write_set_options(a, bsdtar->option_options)) bsdtar_errc(bsdtar, 1, 0, archive_error_string(a)); if (ARCHIVE_OK != archive_write_open_fd(a, bsdtar->fd)) bsdtar_errc(bsdtar, 1, 0, archive_error_string(a)); write_archive(a, bsdtar); close(bsdtar->fd); bsdtar->fd = -1; while (bsdtar->archive_dir->head != NULL) { p = bsdtar->archive_dir->head->next; free(bsdtar->archive_dir->head->name); free(bsdtar->archive_dir->head); bsdtar->archive_dir->head = p; } bsdtar->archive_dir->tail = NULL; }
/* * Same as 'c', except we only support tar or empty formats in * uncompressed files on disk. */ void tar_mode_r(struct bsdtar *bsdtar) { off_t end_offset; int format; struct archive *a; struct archive_entry *entry; int r; /* Sanity-test some arguments and the file. */ test_for_append(bsdtar); format = ARCHIVE_FORMAT_TAR_PAX_RESTRICTED; bsdtar->fd = open(bsdtar->filename, O_RDWR | O_CREAT, 0666); if (bsdtar->fd < 0) bsdtar_errc(bsdtar, 1, errno, "Cannot open %s", bsdtar->filename); a = archive_read_new(); archive_read_support_compression_all(a); archive_read_support_format_tar(a); archive_read_support_format_gnutar(a); r = archive_read_open_fd(a, bsdtar->fd, 10240); if (r != ARCHIVE_OK) bsdtar_errc(bsdtar, 1, archive_errno(a), "Can't read archive %s: %s", bsdtar->filename, archive_error_string(a)); while (0 == archive_read_next_header(a, &entry)) { if (archive_compression(a) != ARCHIVE_COMPRESSION_NONE) { archive_read_finish(a); close(bsdtar->fd); bsdtar_errc(bsdtar, 1, 0, "Cannot append to compressed archive."); } /* Keep going until we hit end-of-archive */ format = archive_format(a); } end_offset = archive_read_header_position(a); archive_read_finish(a); /* Re-open archive for writing */ a = archive_write_new(); archive_write_set_compression_none(a); /* * Set the format to be used for writing. To allow people to * extend empty files, we need to allow them to specify the format, * which opens the possibility that they will specify a format that * doesn't match the existing format. Hence, the following bit * of arcane ugliness. */ if (bsdtar->create_format != NULL) { /* If the user requested a format, use that, but ... */ archive_write_set_format_by_name(a, bsdtar->create_format); /* ... complain if it's not compatible. */ format &= ARCHIVE_FORMAT_BASE_MASK; if (format != (int)(archive_format(a) & ARCHIVE_FORMAT_BASE_MASK) && format != ARCHIVE_FORMAT_EMPTY) { bsdtar_errc(bsdtar, 1, 0, "Format %s is incompatible with the archive %s.", bsdtar->create_format, bsdtar->filename); } } else { /* * Just preserve the current format, with a little care * for formats that libarchive can't write. */ if (format == ARCHIVE_FORMAT_TAR_GNUTAR) /* TODO: When gtar supports pax, use pax restricted. */ format = ARCHIVE_FORMAT_TAR_USTAR; if (format == ARCHIVE_FORMAT_EMPTY) format = ARCHIVE_FORMAT_TAR_PAX_RESTRICTED; archive_write_set_format(a, format); } lseek(bsdtar->fd, end_offset, SEEK_SET); /* XXX check return val XXX */ if (ARCHIVE_OK != archive_write_set_options(a, bsdtar->option_options)) bsdtar_errc(bsdtar, 1, 0, archive_error_string(a)); if (ARCHIVE_OK != archive_write_open_fd(a, bsdtar->fd)) bsdtar_errc(bsdtar, 1, 0, archive_error_string(a)); write_archive(a, bsdtar); /* XXX check return val XXX */ close(bsdtar->fd); bsdtar->fd = -1; }