static void DTAR_write_header(struct archive* ar, uint64_t idx, uint64_t offset)
{
/* allocate and entry for this item */
struct archive_entry* entry = archive_entry_new();
/* get file name for this item */
/* fill up entry, FIXME: the uglyness of removing leading slash */
const char* fname = mfu_flist_file_get_name(DTAR_flist, idx);
archive_entry_copy_pathname(entry, &fname[1]);
if (DTAR_user_opts.preserve) {
struct archive* source = archive_read_disk_new();
archive_read_disk_set_standard_lookup(source);
int fd = open(fname, O_RDONLY);
if (archive_read_disk_entry_from_file(source, entry, fd, NULL) != ARCHIVE_OK) {
MFU_LOG(MFU_LOG_ERR, "archive_read_disk_entry_from_file(): %s", archive_error_string(ar));
}
archive_read_free(source);
close(fd);
} else {
/* TODO: read stat info from mfu_flist */
struct stat stbuf;
mfu_lstat(fname, &stbuf);
archive_entry_copy_stat(entry, &stbuf);
/* set user name of owner */
const char* uname = mfu_flist_file_get_username(DTAR_flist, idx);
archive_entry_set_uname(entry, uname);
/* set group name */
const char* gname = mfu_flist_file_get_groupname(DTAR_flist, idx);
archive_entry_set_gname(entry, gname);
}
/* TODO: Seems to be a bug here potentially leading to corrupted
* archive files. archive_write_free also writes two blocks of
* NULL bytes at the end of an archive file, however, each rank
* will have a different view of the length of the file, so one
* rank may write its NULL blocks over top of the actual data
* written by another rank */
/* write entry info to archive */
struct archive* dest = archive_write_new();
archive_write_set_format_pax(dest);
if (archive_write_open_fd(dest, DTAR_writer.fd_tar) != ARCHIVE_OK) {
MFU_LOG(MFU_LOG_ERR, "archive_write_open_fd(): %s", archive_error_string(ar));
}
/* seek to offset in tar archive for this file */
lseek(DTAR_writer.fd_tar, offset, SEEK_SET);
/* write header for this item */
if (archive_write_header(dest, entry) != ARCHIVE_OK) {
MFU_LOG(MFU_LOG_ERR, "archive_write_header(): %s", archive_error_string(ar));
}
archive_entry_free(entry);
archive_write_free(dest);
}
void DTAR_writer_init()
{
char compress=DTAR_user_opts->compress;
char filename=DTAR_user_opts->dest_path;
DTAR_writer->flags= O_WRONLY | O_CREAT | O_TRUNC | O_BINARY | O_CLOEXEC;
DTAR_writer->a=archive_write_new();
switch (compress) {
case 'j': case 'y':
archive_write_add_filter_bzip2(a);
break;
case 'Z':
archive_write_add_filter_compress(a);
break;
case 'z':
archive_write_add_filter_gzip(a);
break;
default:
archive_write_add_filter_none(a);
break;
}
archive_write_set_format_ustar(a);
archive_write_set_bytes_per_block(a,0);
if (strcmp(filename, "-") == 0)
filename = NULL;
DTAR_writer->fd_tar=open(filename,DTAR_writer->flags,0666);
archive_write_open_fd(a, DTAR_writer->fd_tar);
}
int
archive_write_open_filename_w(struct archive *a, const wchar_t *filename)
{
if (filename == NULL || filename[0] == L'\0')
return (archive_write_open_fd(a, 1));
return (open_filename(a, 0, filename));
}
int
archive_write_open_filename(struct archive *a, const char *filename)
{
if (filename == NULL || filename[0] == '\0')
return (archive_write_open_fd(a, 1));
return (open_filename(a, 1, filename));
}
int
repodata_flush(struct xbps_handle *xhp, const char *repodir,
xbps_dictionary_t idx, xbps_dictionary_t idxfiles)
{
struct archive *ar;
mode_t myumask;
char *repofile, *tname, *xml;
int repofd;
/* Create a tempfile for our repository archive */
repofile = xbps_repo_path(xhp, repodir);
tname = xbps_xasprintf("%s.XXXXXXXXXX", repofile);
if ((repofd = mkstemp(tname)) == -1)
return errno;
/* Create and write our repository archive */
ar = archive_write_new();
assert(ar);
archive_write_set_compression_gzip(ar);
archive_write_set_format_pax_restricted(ar);
archive_write_set_options(ar, "compression-level=9");
archive_write_open_fd(ar, repofd);
xml = xbps_dictionary_externalize(idx);
assert(xml);
if (xbps_archive_append_buf(ar, xml, strlen(xml),
XBPS_PKGINDEX, 0644, "root", "root") != 0) {
free(xml);
return -1;
}
free(xml);
xml = xbps_dictionary_externalize(idxfiles);
assert(xml);
if (xbps_archive_append_buf(ar, xml, strlen(xml),
XBPS_PKGINDEX_FILES, 0644, "root", "root") != 0) {
free(xml);
return -1;
}
free(xml);
archive_write_finish(ar);
/* Write data to tempfile and rename */
fdatasync(repofd);
myumask = umask(0);
(void)umask(myumask);
assert(fchmod(repofd, 0666 & ~myumask) != -1);
close(repofd);
rename(tname, repofile);
free(repofile);
free(tname);
return 0;
}
struct archive * DTAR_new_archive() {
struct archive *a = archive_write_new();
archive_write_set_format_pax(a);
int r = archive_write_open_fd(a, DTAR_writer.fd_tar);
if ( r != ARCHIVE_OK) {
MFU_LOG(MFU_LOG_ERR, "archive_write_open_fd(): %s", archive_error_string(a));
return NULL;
}
return a;
}
void DTAR_write_header(struct archive *ar, uint64_t idx, uint64_t offset)
{
const char * fname = mfu_flist_file_get_name(DTAR_flist, idx);
/* fill up entry, FIXME: the uglyness of removing leading slash */
struct archive_entry *entry = archive_entry_new();
archive_entry_copy_pathname(entry, &fname[1]);
if (DTAR_user_opts.preserve) {
struct archive * source = archive_read_disk_new();
archive_read_disk_set_standard_lookup(source);
int fd = open(fname, O_RDONLY);
if (archive_read_disk_entry_from_file(source, entry, fd, NULL) != ARCHIVE_OK) {
MFU_LOG(MFU_LOG_ERR, "archive_read_disk_entry_from_file(): %s", archive_error_string(ar));
}
archive_read_free(source);
} else {
/* read stat info from mfu_flist */
struct stat stbuf;
mfu_lstat(fname, &stbuf);
archive_entry_copy_stat(entry, &stbuf);
const char* uname = mfu_flist_file_get_username(DTAR_flist, idx);
archive_entry_set_uname(entry, uname);
const char* gname = mfu_flist_file_get_groupname(DTAR_flist, idx);
archive_entry_set_gname(entry, gname);
}
/* write entry info to archive */
struct archive* dest = archive_write_new();
archive_write_set_format_pax(dest);
if (archive_write_open_fd(dest, DTAR_writer.fd_tar) != ARCHIVE_OK) {
MFU_LOG(MFU_LOG_ERR, "archive_write_open_fd(): %s", archive_error_string(ar));
}
lseek64(DTAR_writer.fd_tar, offset, SEEK_SET);
if (archive_write_header(dest, entry) != ARCHIVE_OK) {
MFU_LOG(MFU_LOG_ERR, "archive_write_header(): %s", archive_error_string(ar));
}
archive_entry_free(entry);
archive_write_free(dest);
}
static db_writer_t *db_writer_new(repo_t *repo, file_t *db)
{
db_writer_t *writer = malloc(sizeof(db_writer_t));
writer->archive = archive_write_new();
writer->entry = archive_entry_new();
switch (repo->compression) {
case COMPRESS_NONE:
archive_write_add_filter_none(writer->archive);
break;
case COMPRESS_GZIP:
archive_write_add_filter_gzip(writer->archive);
break;
case COMPRESS_BZIP2:
archive_write_add_filter_bzip2(writer->archive);
break;
case COMPRESS_XZ:
archive_write_add_filter_xz(writer->archive);
break;
case COMPRESS_COMPRESS:
archive_write_add_filter_compress(writer->archive);
break;
}
archive_write_set_format_pax_restricted(writer->archive);
writer->fd = openat(repo->dirfd, db->file, O_CREAT | O_WRONLY | O_TRUNC, 0644);
if (writer->fd < 0)
err(EXIT_FAILURE, "failed to open %s for writing", db->file);
if (flock(writer->fd, LOCK_EX) < 0)
err(EXIT_FAILURE, "failed to lock %s", db->file);
archive_write_open_fd(writer->archive, writer->fd);
buffer_init(&writer->buf, 1024);
return writer;
}
void
tar_mode_u(struct bsdtar *bsdtar)
{
int64_t end_offset;
struct archive *a;
struct archive_entry *entry;
int format;
struct archive_dir_entry *p;
struct archive_dir archive_dir;
bsdtar->archive_dir = &archive_dir;
memset(&archive_dir, 0, sizeof(archive_dir));
format = ARCHIVE_FORMAT_TAR_PAX_RESTRICTED;
/* Sanity-test some arguments and the file. */
test_for_append(bsdtar);
bsdtar->fd = open(bsdtar->filename, O_RDWR | O_BINARY);
if (bsdtar->fd < 0)
lafe_errc(1, errno,
"Cannot open %s", bsdtar->filename);
a = archive_read_new();
archive_read_support_filter_all(a);
archive_read_support_format_tar(a);
archive_read_support_format_gnutar(a);
set_reader_options(bsdtar, a);
if (archive_read_open_fd(a, bsdtar->fd, bsdtar->bytes_per_block)
!= ARCHIVE_OK) {
lafe_errc(1, 0,
"Can't open %s: %s", bsdtar->filename,
archive_error_string(a));
}
/* Build a list of all entries and their recorded mod times. */
while (0 == archive_read_next_header(a, &entry)) {
if (archive_filter_code(a, 0) != ARCHIVE_FILTER_NONE) {
archive_read_free(a);
close(bsdtar->fd);
lafe_errc(1, 0,
"Cannot append to compressed archive.");
}
if (archive_match_exclude_entry(bsdtar->matching,
ARCHIVE_MATCH_MTIME | ARCHIVE_MATCH_OLDER |
ARCHIVE_MATCH_EQUAL, entry) != ARCHIVE_OK)
lafe_errc(1, 0, "Error : %s",
archive_error_string(bsdtar->matching));
/* Record the last format determination we see */
format = archive_format(a);
/* Keep going until we hit end-of-archive */
}
end_offset = archive_read_header_position(a);
archive_read_free(a);
/* Re-open archive for writing. */
a = archive_write_new();
/*
* Set format to same one auto-detected above.
*/
archive_write_set_format(a, format);
archive_write_set_bytes_per_block(a, bsdtar->bytes_per_block);
archive_write_set_bytes_in_last_block(a, bsdtar->bytes_in_last_block);
if (lseek(bsdtar->fd, end_offset, SEEK_SET) < 0)
lafe_errc(1, errno, "Could not seek to archive end");
set_writer_options(bsdtar, a);
if (ARCHIVE_OK != archive_write_open_fd(a, bsdtar->fd))
lafe_errc(1, 0, "%s", archive_error_string(a));
write_archive(a, bsdtar);
close(bsdtar->fd);
bsdtar->fd = -1;
while (bsdtar->archive_dir->head != NULL) {
p = bsdtar->archive_dir->head->next;
free(bsdtar->archive_dir->head->name);
free(bsdtar->archive_dir->head);
bsdtar->archive_dir->head = p;
}
bsdtar->archive_dir->tail = NULL;
}
/*
* Same as 'c', except we only support tar or empty formats in
* uncompressed files on disk.
*/
void
tar_mode_r(struct bsdtar *bsdtar)
{
int64_t end_offset;
int format;
struct archive *a;
struct archive_entry *entry;
int r;
/* Sanity-test some arguments and the file. */
test_for_append(bsdtar);
format = ARCHIVE_FORMAT_TAR_PAX_RESTRICTED;
#if defined(__BORLANDC__)
bsdtar->fd = open(bsdtar->filename, O_RDWR | O_CREAT | O_BINARY);
#else
bsdtar->fd = open(bsdtar->filename, O_RDWR | O_CREAT | O_BINARY, 0666);
#endif
if (bsdtar->fd < 0)
lafe_errc(1, errno,
"Cannot open %s", bsdtar->filename);
a = archive_read_new();
archive_read_support_filter_all(a);
archive_read_support_format_empty(a);
archive_read_support_format_tar(a);
archive_read_support_format_gnutar(a);
set_reader_options(bsdtar, a);
r = archive_read_open_fd(a, bsdtar->fd, 10240);
if (r != ARCHIVE_OK)
lafe_errc(1, archive_errno(a),
"Can't read archive %s: %s", bsdtar->filename,
archive_error_string(a));
while (0 == archive_read_next_header(a, &entry)) {
if (archive_filter_code(a, 0) != ARCHIVE_FILTER_NONE) {
archive_read_free(a);
close(bsdtar->fd);
lafe_errc(1, 0,
"Cannot append to compressed archive.");
}
/* Keep going until we hit end-of-archive */
format = archive_format(a);
}
end_offset = archive_read_header_position(a);
archive_read_free(a);
/* Re-open archive for writing */
a = archive_write_new();
/*
* Set the format to be used for writing. To allow people to
* extend empty files, we need to allow them to specify the format,
* which opens the possibility that they will specify a format that
* doesn't match the existing format. Hence, the following bit
* of arcane ugliness.
*/
if (cset_get_format(bsdtar->cset) != NULL) {
/* If the user requested a format, use that, but ... */
archive_write_set_format_by_name(a,
cset_get_format(bsdtar->cset));
/* ... complain if it's not compatible. */
format &= ARCHIVE_FORMAT_BASE_MASK;
if (format != (int)(archive_format(a) & ARCHIVE_FORMAT_BASE_MASK)
&& format != ARCHIVE_FORMAT_EMPTY) {
lafe_errc(1, 0,
"Format %s is incompatible with the archive %s.",
cset_get_format(bsdtar->cset), bsdtar->filename);
}
} else {
/*
* Just preserve the current format, with a little care
* for formats that libarchive can't write.
*/
if (format == ARCHIVE_FORMAT_EMPTY)
format = ARCHIVE_FORMAT_TAR_PAX_RESTRICTED;
archive_write_set_format(a, format);
}
if (lseek(bsdtar->fd, end_offset, SEEK_SET) < 0)
lafe_errc(1, errno, "Could not seek to archive end");
set_writer_options(bsdtar, a);
if (ARCHIVE_OK != archive_write_open_fd(a, bsdtar->fd))
lafe_errc(1, 0, "%s", archive_error_string(a));
write_archive(a, bsdtar); /* XXX check return val XXX */
close(bsdtar->fd);
bsdtar->fd = -1;
}
static int process_package(rpmts ts, char * filename)
{
FD_t fdi;
FD_t gzdi;
Header h;
int rc = 0;
char * rpmio_flags = NULL;
struct archive *a;
struct archive_entry *entry;
if (!strcmp(filename, "-")) {
fdi = fdDup(STDIN_FILENO);
} else {
fdi = Fopen(filename, "r.ufdio");
}
if (Ferror(fdi)) {
fprintf(stderr, "rpm2archive: %s: %s\n",
filename, Fstrerror(fdi));
exit(EXIT_FAILURE);
}
rc = rpmReadPackageFile(ts, fdi, "rpm2cpio", &h);
switch (rc) {
case RPMRC_OK:
case RPMRC_NOKEY:
case RPMRC_NOTTRUSTED:
break;
case RPMRC_NOTFOUND:
fprintf(stderr, _("argument is not an RPM package\n"));
exit(EXIT_FAILURE);
break;
case RPMRC_FAIL:
default:
fprintf(stderr, _("error reading header from package\n"));
exit(EXIT_FAILURE);
break;
}
/* Retrieve payload size and compression type. */
{ const char *compr = headerGetString(h, RPMTAG_PAYLOADCOMPRESSOR);
rpmio_flags = rstrscat(NULL, "r.", compr ? compr : "gzip", NULL);
}
gzdi = Fdopen(fdi, rpmio_flags); /* XXX gzdi == fdi */
free(rpmio_flags);
if (gzdi == NULL) {
fprintf(stderr, _("cannot re-open payload: %s\n"), Fstrerror(gzdi));
exit(EXIT_FAILURE);
}
rpmfiles files = rpmfilesNew(NULL, h, 0, RPMFI_KEEPHEADER);
rpmfi fi = rpmfiNewArchiveReader(gzdi, files, RPMFI_ITER_READ_ARCHIVE_CONTENT_FIRST);
/* create archive */
a = archive_write_new();
archive_write_add_filter_gzip(a);
archive_write_set_format_pax_restricted(a);
if (!strcmp(filename, "-")) {
if (isatty(STDOUT_FILENO)) {
fprintf(stderr, "Error: refusing to output archive data to a terminal.\n");
exit(EXIT_FAILURE);
}
archive_write_open_fd(a, STDOUT_FILENO);
} else {
char * outname = rstrscat(NULL, filename, ".tgz", NULL);
archive_write_open_filename(a, outname);
_free(outname);
// XXX error handling
}
entry = archive_entry_new();
char * buf = xmalloc(BUFSIZE);
char * hardlink = NULL;
rc = 0;
while (rc >= 0) {
rc = rpmfiNext(fi);
if (rc == RPMERR_ITER_END) {
break;
}
rpm_mode_t mode = rpmfiFMode(fi);
int nlink = rpmfiFNlink(fi);
fill_archive_entry(a, entry, fi);
if (nlink > 1) {
if (rpmfiArchiveHasContent(fi)) {
_free(hardlink);
hardlink = rstrscat(NULL, ".", rpmfiFN(fi), NULL);
} else {
archive_entry_set_hardlink(entry, hardlink);
}
}
archive_write_header(a, entry);
if (S_ISREG(mode) && (nlink == 1 || rpmfiArchiveHasContent(fi))) {
write_file_content(a, buf, fi);
}
}
/* End of iteration is not an error */
if (rc == RPMERR_ITER_END) {
rc = 0;
}
_free(hardlink);
Fclose(gzdi); /* XXX gzdi == fdi */
archive_entry_free(entry);
archive_write_close(a);
archive_write_free(a);
buf = _free(buf);
rpmfilesFree(files);
rpmfiFree(fi);
headerFree(h);
return rc;
}
int main(int argc, char **argv) {
int i;
unsigned int version = 0, help = 0;
char cpiotmpfile[] = CPIOTMPFILE;
struct archive *archive;
struct archive_entry *entry;
struct stat st;
char buff[64];
int len, fdfile, fdarchive;
DIR * dir;
struct dirent * ent;
char * filename, * path;
off_t pathlength = 0;
int8_t rc = EXIT_FAILURE;
/* get command line options */
while ((i = getopt_long(argc, argv, optstring, options_long, NULL)) != -1)
switch (i) {
case 'h':
help++;
break;
case 'V':
version++;
break;
}
if (version > 0)
printf("%s: %s v%s (compiled: " __DATE__ ", " __TIME__ ")\n", argv[0], PROGNAME, VERSION);
if (help > 0)
fprintf(stderr, "usage: %s [-h|--help] [-V|--version]\n", argv[0]);
if (version > 0 || help > 0)
return EXIT_SUCCESS;
if ((fdarchive = mkstemp(cpiotmpfile)) < 0) {
perror("mkstemp() failed");
goto out10;
}
if ((archive = archive_write_new()) == NULL) {
fprintf(stderr, "archive_write_new() failed.\n");
goto out10;
}
if (archive_write_set_format_cpio_newc(archive) != ARCHIVE_OK) {
fprintf(stderr, "archive_write_set_format_cpio_newc() failed.\n");
goto out10;
}
if (archive_write_open_fd(archive, fdarchive) != ARCHIVE_OK) {
fprintf(stderr, "archive_write_open_fd() failed.\n");
goto out10;
}
while (1) {
path = strdup(CHALLENGEDIR + 1);
if (strstr(path + pathlength, "/") == NULL)
break;
*strstr(path + pathlength, "/") = 0;
pathlength = strlen(path) + 1;
if (add_dir(archive, path) < 0) {
fprintf(stderr, "add_dir() failed");
goto out10;
}
free(path);
}
if ((dir = opendir(CHALLENGEDIR)) == NULL) {
perror("opendir() failed");
goto out10;
}
while ((ent = readdir(dir)) != NULL) {
filename = malloc(sizeof(CHALLENGEDIR) + strlen(ent->d_name) + 1);
sprintf(filename, CHALLENGEDIR "%s", ent->d_name);
if (stat(filename, &st) < 0) {
perror("stat() failed");
goto out10;
}
if (S_ISREG(st.st_mode)) {
if ((entry = archive_entry_new()) == NULL) {
fprintf(stderr, "archive_entry_new() failed.\n");
goto out10;
}
/* these do not return exit code */
archive_entry_set_pathname(entry, filename + 1);
archive_entry_set_size(entry, st.st_size);
archive_entry_set_filetype(entry, AE_IFREG);
archive_entry_set_perm(entry, 0644);
if (archive_write_header(archive, entry) != ARCHIVE_OK) {
fprintf(stderr, "archive_write_header() failed");
goto out10;
}
if ((fdfile = open(filename, O_RDONLY)) < 0) {
perror("open() failed");
goto out10;
}
if ((len = read(fdfile, buff, sizeof(buff))) < 0) {
perror("read() failed");
goto out10;
}
while (len > 0) {
if (archive_write_data(archive, buff, len) < 0) {
fprintf(stderr, "archive_write_data() failed");
goto out10;
}
if ((len = read(fdfile, buff, sizeof(buff))) < 0) {
perror("read() failed");
goto out10;
}
}
if (close(fdfile) < 0) {
perror("close() failed");
goto out10;
}
archive_entry_free(entry);
}
free(filename);
}
if (closedir(dir) < 0) {
perror("closedir() failed");
goto out10;
}
if (archive_write_close(archive) != ARCHIVE_OK) {
fprintf(stderr, "archive_write_close() failed");
goto out10;
}
if (archive_write_free(archive) != ARCHIVE_OK) {
fprintf(stderr, "archive_write_free() failed");
goto out10;
}
if (access(CPIOFILE, F_OK) == 0 && unlink(CPIOFILE) < 0) {
perror("unkink() failed");
goto out10;
}
if (rename(cpiotmpfile, CPIOFILE) < 0) {
perror("rename() failed");
goto out10;
}
rc = EXIT_SUCCESS;
out10:
if (access(cpiotmpfile, F_OK) == 0)
unlink(cpiotmpfile);
return rc;
}
static void mfu_flist_archive_create_libcircle(mfu_flist flist, const char* archivefile, mfu_archive_options_t* opts)
{
DTAR_flist = flist;
DTAR_user_opts = *opts;
MPI_Comm_rank(MPI_COMM_WORLD, &DTAR_rank);
/* TODO: stripe the archive file if on parallel file system */
/* init statistics */
DTAR_statistics.total_dirs = 0;
DTAR_statistics.total_files = 0;
DTAR_statistics.total_links = 0;
DTAR_statistics.total_size = 0;
DTAR_statistics.total_bytes_copied = 0;
time(&(DTAR_statistics.time_started));
DTAR_statistics.wtime_started = MPI_Wtime();
/* create the archive file */
DTAR_writer.name = archivefile;
DTAR_writer.flags = O_WRONLY | O_CREAT | O_CLOEXEC | O_LARGEFILE;
DTAR_writer.fd_tar = open(archivefile, DTAR_writer.flags, 0664);
/* get number of items in our portion of the list */
DTAR_count = mfu_flist_size(DTAR_flist);
/* allocate memory for file sizes and offsets */
uint64_t* fsizes = (uint64_t*) MFU_MALLOC(DTAR_count * sizeof(uint64_t));
DTAR_offsets = (uint64_t*) MFU_MALLOC(DTAR_count * sizeof(uint64_t));
/* compute local offsets for each item and total
* bytes we're contributing to the archive */
uint64_t idx;
uint64_t offset = 0;
for (idx = 0; idx < DTAR_count; idx++) {
/* assume the item takes no space */
fsizes[idx] = 0;
/* identify item type to compute its size in the archive */
mfu_filetype type = mfu_flist_file_get_type(DTAR_flist, idx);
if (type == MFU_TYPE_DIR || type == MFU_TYPE_LINK) {
/* directories and symlinks only need the header */
fsizes[idx] = DTAR_HDR_LENGTH;
} else if (type == MFU_TYPE_FILE) {
/* regular file requires a header, plus file content,
* and things are packed into blocks of 512 bytes */
uint64_t fsize = mfu_flist_file_get_size(DTAR_flist, idx);
/* determine whether file size is integer multiple of 512 bytes */
uint64_t rem = fsize % 512;
if (rem == 0) {
/* file content is multiple of 512 bytes, so perfect fit */
fsizes[idx] = fsize + DTAR_HDR_LENGTH;
} else {
/* TODO: check and explain this math */
fsizes[idx] = (fsize / 512 + 4) * 512;
}
}
/* increment our local offset for this item */
DTAR_offsets[idx] = offset;
offset += fsizes[idx];
}
/* execute scan to figure our global base offset in the archive file */
uint64_t global_offset = 0;
MPI_Scan(&offset, &global_offset, 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);
global_offset -= offset;
/* update offsets for each of our file to their global offset */
for (idx = 0; idx < DTAR_count; idx++) {
DTAR_offsets[idx] += global_offset;
}
/* create an archive */
struct archive* ar = archive_write_new();
archive_write_set_format_pax(ar);
int r = archive_write_open_fd(ar, DTAR_writer.fd_tar);
if (r != ARCHIVE_OK) {
MFU_LOG(MFU_LOG_ERR, "archive_write_open_fd(): %s", archive_error_string(ar));
DTAR_abort(EXIT_FAILURE);
}
/* write headers for our files */
for (idx = 0; idx < DTAR_count; idx++) {
mfu_filetype type = mfu_flist_file_get_type(DTAR_flist, idx);
if (type == MFU_TYPE_FILE || type == MFU_TYPE_DIR || type == MFU_TYPE_LINK) {
DTAR_write_header(ar, idx, DTAR_offsets[idx]);
}
}
/* prepare libcircle */
CIRCLE_init(0, NULL, CIRCLE_SPLIT_EQUAL | CIRCLE_CREATE_GLOBAL);
CIRCLE_loglevel loglevel = CIRCLE_LOG_WARN;
CIRCLE_enable_logging(loglevel);
/* register callbacks */
CIRCLE_cb_create(&DTAR_enqueue_copy);
CIRCLE_cb_process(&DTAR_perform_copy);
/* run the libcircle job to copy data into archive file */
CIRCLE_begin();
CIRCLE_finalize();
/* compute total bytes copied */
uint64_t archive_size = 0;
MPI_Allreduce(&offset, &archive_size, 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);
DTAR_statistics.total_size = archive_size;
DTAR_statistics.wtime_ended = MPI_Wtime();
time(&(DTAR_statistics.time_ended));
/* print stats */
double rel_time = DTAR_statistics.wtime_ended - \
DTAR_statistics.wtime_started;
if (DTAR_rank == 0) {
char starttime_str[256];
struct tm* localstart = localtime(&(DTAR_statistics.time_started));
strftime(starttime_str, 256, "%b-%d-%Y, %H:%M:%S", localstart);
char endtime_str[256];
struct tm* localend = localtime(&(DTAR_statistics.time_ended));
strftime(endtime_str, 256, "%b-%d-%Y, %H:%M:%S", localend);
/* add two 512 blocks at the end */
DTAR_statistics.total_size += 512 * 2;
/* convert bandwidth to unit */
double agg_rate_tmp;
double agg_rate = (double) DTAR_statistics.total_size / rel_time;
const char* agg_rate_units;
mfu_format_bytes(agg_rate, &agg_rate_tmp, &agg_rate_units);
MFU_LOG(MFU_LOG_INFO, "Started: %s", starttime_str);
MFU_LOG(MFU_LOG_INFO, "Completed: %s", endtime_str);
MFU_LOG(MFU_LOG_INFO, "Total archive size: %" PRIu64, DTAR_statistics.total_size);
MFU_LOG(MFU_LOG_INFO, "Rate: %.3lf %s " \
"(%.3" PRIu64 " bytes in %.3lf seconds)", \
agg_rate_tmp, agg_rate_units, DTAR_statistics.total_size, rel_time);
}
/* clean up */
mfu_free(&fsizes);
mfu_free(&DTAR_offsets);
/* close archive file */
archive_write_free(ar);
mfu_close(DTAR_writer.name, DTAR_writer.fd_tar);
}
void
tar_mode_u(struct bsdtar *bsdtar)
{
off_t end_offset;
struct archive *a;
struct archive_entry *entry;
int format;
struct archive_dir_entry *p;
struct archive_dir archive_dir;
bsdtar->archive_dir = &archive_dir;
memset(&archive_dir, 0, sizeof(archive_dir));
format = ARCHIVE_FORMAT_TAR_PAX_RESTRICTED;
/* Sanity-test some arguments and the file. */
test_for_append(bsdtar);
bsdtar->fd = open(bsdtar->filename, O_RDWR);
if (bsdtar->fd < 0)
bsdtar_errc(bsdtar, 1, errno,
"Cannot open %s", bsdtar->filename);
a = archive_read_new();
archive_read_support_compression_all(a);
archive_read_support_format_tar(a);
archive_read_support_format_gnutar(a);
if (archive_read_open_fd(a, bsdtar->fd,
bsdtar->bytes_per_block != 0 ? bsdtar->bytes_per_block :
DEFAULT_BYTES_PER_BLOCK) != ARCHIVE_OK) {
bsdtar_errc(bsdtar, 1, 0,
"Can't open %s: %s", bsdtar->filename,
archive_error_string(a));
}
/* Build a list of all entries and their recorded mod times. */
while (0 == archive_read_next_header(a, &entry)) {
if (archive_compression(a) != ARCHIVE_COMPRESSION_NONE) {
archive_read_finish(a);
close(bsdtar->fd);
bsdtar_errc(bsdtar, 1, 0,
"Cannot append to compressed archive.");
}
add_dir_list(bsdtar, archive_entry_pathname(entry),
archive_entry_mtime(entry),
archive_entry_mtime_nsec(entry));
/* Record the last format determination we see */
format = archive_format(a);
/* Keep going until we hit end-of-archive */
}
end_offset = archive_read_header_position(a);
archive_read_finish(a);
/* Re-open archive for writing. */
a = archive_write_new();
archive_write_set_compression_none(a);
/*
* Set format to same one auto-detected above, except that
* we don't write GNU tar format, so use ustar instead.
*/
if (format == ARCHIVE_FORMAT_TAR_GNUTAR)
format = ARCHIVE_FORMAT_TAR_USTAR;
archive_write_set_format(a, format);
if (bsdtar->bytes_per_block != 0) {
archive_write_set_bytes_per_block(a, bsdtar->bytes_per_block);
archive_write_set_bytes_in_last_block(a,
bsdtar->bytes_per_block);
} else
archive_write_set_bytes_per_block(a, DEFAULT_BYTES_PER_BLOCK);
lseek(bsdtar->fd, end_offset, SEEK_SET);
ftruncate(bsdtar->fd, end_offset);
if (ARCHIVE_OK != archive_write_set_options(a, bsdtar->option_options))
bsdtar_errc(bsdtar, 1, 0, archive_error_string(a));
if (ARCHIVE_OK != archive_write_open_fd(a, bsdtar->fd))
bsdtar_errc(bsdtar, 1, 0, archive_error_string(a));
write_archive(a, bsdtar);
close(bsdtar->fd);
bsdtar->fd = -1;
while (bsdtar->archive_dir->head != NULL) {
p = bsdtar->archive_dir->head->next;
free(bsdtar->archive_dir->head->name);
free(bsdtar->archive_dir->head);
bsdtar->archive_dir->head = p;
}
bsdtar->archive_dir->tail = NULL;
}
/*
* Same as 'c', except we only support tar or empty formats in
* uncompressed files on disk.
*/
void
tar_mode_r(struct bsdtar *bsdtar)
{
off_t end_offset;
int format;
struct archive *a;
struct archive_entry *entry;
int r;
/* Sanity-test some arguments and the file. */
test_for_append(bsdtar);
format = ARCHIVE_FORMAT_TAR_PAX_RESTRICTED;
bsdtar->fd = open(bsdtar->filename, O_RDWR | O_CREAT, 0666);
if (bsdtar->fd < 0)
bsdtar_errc(bsdtar, 1, errno,
"Cannot open %s", bsdtar->filename);
a = archive_read_new();
archive_read_support_compression_all(a);
archive_read_support_format_tar(a);
archive_read_support_format_gnutar(a);
r = archive_read_open_fd(a, bsdtar->fd, 10240);
if (r != ARCHIVE_OK)
bsdtar_errc(bsdtar, 1, archive_errno(a),
"Can't read archive %s: %s", bsdtar->filename,
archive_error_string(a));
while (0 == archive_read_next_header(a, &entry)) {
if (archive_compression(a) != ARCHIVE_COMPRESSION_NONE) {
archive_read_finish(a);
close(bsdtar->fd);
bsdtar_errc(bsdtar, 1, 0,
"Cannot append to compressed archive.");
}
/* Keep going until we hit end-of-archive */
format = archive_format(a);
}
end_offset = archive_read_header_position(a);
archive_read_finish(a);
/* Re-open archive for writing */
a = archive_write_new();
archive_write_set_compression_none(a);
/*
* Set the format to be used for writing. To allow people to
* extend empty files, we need to allow them to specify the format,
* which opens the possibility that they will specify a format that
* doesn't match the existing format. Hence, the following bit
* of arcane ugliness.
*/
if (bsdtar->create_format != NULL) {
/* If the user requested a format, use that, but ... */
archive_write_set_format_by_name(a,
bsdtar->create_format);
/* ... complain if it's not compatible. */
format &= ARCHIVE_FORMAT_BASE_MASK;
if (format != (int)(archive_format(a) & ARCHIVE_FORMAT_BASE_MASK)
&& format != ARCHIVE_FORMAT_EMPTY) {
bsdtar_errc(bsdtar, 1, 0,
"Format %s is incompatible with the archive %s.",
bsdtar->create_format, bsdtar->filename);
}
} else {
/*
* Just preserve the current format, with a little care
* for formats that libarchive can't write.
*/
if (format == ARCHIVE_FORMAT_TAR_GNUTAR)
/* TODO: When gtar supports pax, use pax restricted. */
format = ARCHIVE_FORMAT_TAR_USTAR;
if (format == ARCHIVE_FORMAT_EMPTY)
format = ARCHIVE_FORMAT_TAR_PAX_RESTRICTED;
archive_write_set_format(a, format);
}
lseek(bsdtar->fd, end_offset, SEEK_SET); /* XXX check return val XXX */
if (ARCHIVE_OK != archive_write_set_options(a, bsdtar->option_options))
bsdtar_errc(bsdtar, 1, 0, archive_error_string(a));
if (ARCHIVE_OK != archive_write_open_fd(a, bsdtar->fd))
bsdtar_errc(bsdtar, 1, 0, archive_error_string(a));
write_archive(a, bsdtar); /* XXX check return val XXX */
close(bsdtar->fd);
bsdtar->fd = -1;
}
