static void DTAR_write_header(struct archive* ar, uint64_t idx, uint64_t offset)
{
/* allocate and entry for this item */
struct archive_entry* entry = archive_entry_new();
/* get file name for this item */
/* fill up entry, FIXME: the uglyness of removing leading slash */
const char* fname = mfu_flist_file_get_name(DTAR_flist, idx);
archive_entry_copy_pathname(entry, &fname[1]);
if (DTAR_user_opts.preserve) {
struct archive* source = archive_read_disk_new();
archive_read_disk_set_standard_lookup(source);
int fd = open(fname, O_RDONLY);
if (archive_read_disk_entry_from_file(source, entry, fd, NULL) != ARCHIVE_OK) {
MFU_LOG(MFU_LOG_ERR, "archive_read_disk_entry_from_file(): %s", archive_error_string(ar));
}
archive_read_free(source);
close(fd);
} else {
/* TODO: read stat info from mfu_flist */
struct stat stbuf;
mfu_lstat(fname, &stbuf);
archive_entry_copy_stat(entry, &stbuf);
/* set user name of owner */
const char* uname = mfu_flist_file_get_username(DTAR_flist, idx);
archive_entry_set_uname(entry, uname);
/* set group name */
const char* gname = mfu_flist_file_get_groupname(DTAR_flist, idx);
archive_entry_set_gname(entry, gname);
}
/* TODO: Seems to be a bug here potentially leading to corrupted
* archive files. archive_write_free also writes two blocks of
* NULL bytes at the end of an archive file, however, each rank
* will have a different view of the length of the file, so one
* rank may write its NULL blocks over top of the actual data
* written by another rank */
/* write entry info to archive */
struct archive* dest = archive_write_new();
archive_write_set_format_pax(dest);
if (archive_write_open_fd(dest, DTAR_writer.fd_tar) != ARCHIVE_OK) {
MFU_LOG(MFU_LOG_ERR, "archive_write_open_fd(): %s", archive_error_string(ar));
}
/* seek to offset in tar archive for this file */
lseek(DTAR_writer.fd_tar, offset, SEEK_SET);
/* write header for this item */
if (archive_write_header(dest, entry) != ARCHIVE_OK) {
MFU_LOG(MFU_LOG_ERR, "archive_write_header(): %s", archive_error_string(ar));
}
archive_entry_free(entry);
archive_write_free(dest);
}
void Writer::set_format_helper(struct archive *ar, int format)
{
std::string error_msg;
switch(format) {
case Archive::FORMAT_AR_BSD:
archive_write_set_format_ar_bsd(ar);
break;
case Archive::FORMAT_AR_SVR4:
archive_write_set_format_ar_svr4(ar);
break;
case Archive::FORMAT_CPIO:
archive_write_set_format_cpio(ar);
break;
case Archive::FORMAT_CPIO_NEWC:
archive_write_set_format_cpio_newc(ar);
break;
case Archive::FORMAT_MTREE:
archive_write_set_format_mtree(ar);
break;
case Archive::FORMAT_TAR:
case Archive::FORMAT_TAR_PAX_RESTRICTED:
archive_write_set_format_pax_restricted(ar);
break;
case Archive::FORMAT_TAR_PAX_INTERCHANGE:
archive_write_set_format_pax(ar);
break;
case Archive::FORMAT_TAR_USTAR:
archive_write_set_format_ustar(ar);
break;
default:
error_msg = "unknown or unsupported archive format";
throw Error(error_msg);
}
}
/*
* Verify that KOI8-R filenames are not translated to Unicode and UTF-8
* when using hdrcharset=BINARY option.
*/
static void
test_pax_filename_encoding_KOI8R_BINARY(void)
{
struct archive *a;
struct archive_entry *entry;
char buff[4096];
size_t used;
if (NULL == setlocale(LC_ALL, "ru_RU.KOI8-R")) {
skipping("KOI8-R locale not available on this system.");
return;
}
a = archive_write_new();
assertEqualInt(ARCHIVE_OK, archive_write_set_format_pax(a));
/* BINARY mode should be accepted. */
assertEqualInt(ARCHIVE_OK,
archive_write_set_options(a, "hdrcharset=BINARY"));
assertEqualInt(ARCHIVE_OK,
archive_write_open_memory(a, buff, sizeof(buff), &used));
entry = archive_entry_new2(a);
archive_entry_set_pathname(entry, "\xD0\xD2\xC9");
archive_entry_set_filetype(entry, AE_IFREG);
archive_entry_set_size(entry, 0);
assertEqualInt(ARCHIVE_OK, archive_write_header(a, entry));
archive_entry_free(entry);
assertEqualInt(ARCHIVE_OK, archive_write_free(a));
/* "hdrcharset=BINARY" pax attribute should be written. */
assertEqualMem(buff + 512, "21 hdrcharset=BINARY\x0A", 21);
/* Above three characters in KOI8-R should not translate to any
* character-set. */
assertEqualMem(buff + 512+21, "12 path=\xD0\xD2\xC9\x0A", 12);
}
/*
* Verify that CP932/SJIS filenames are correctly translated to Unicode and UTF-8.
*/
static void
test_pax_filename_encoding_CP932(void)
{
struct archive *a;
struct archive_entry *entry;
char buff[4096];
size_t used;
if (NULL == setlocale(LC_ALL, "Japanese_Japan") &&
NULL == setlocale(LC_ALL, "ja_JP.SJIS")) {
skipping("eucJP locale not available on this system.");
return;
}
/* Check if the paltform completely supports the string conversion. */
a = archive_write_new();
assertEqualInt(ARCHIVE_OK, archive_write_set_format_pax(a));
if (archive_write_set_options(a, "hdrcharset=UTF-8") != ARCHIVE_OK) {
skipping("This system cannot convert character-set"
" from CP932/SJIS to UTF-8.");
archive_write_free(a);
return;
}
archive_write_free(a);
/* Re-create a write archive object since filenames should be written
* in UTF-8 by default. */
a = archive_write_new();
assertEqualInt(ARCHIVE_OK, archive_write_set_format_pax(a));
assertEqualInt(ARCHIVE_OK,
archive_write_open_memory(a, buff, sizeof(buff), &used));
entry = archive_entry_new2(a);
archive_entry_set_pathname(entry, "\x95\x5C.txt");
/* Check the Unicode version. */
archive_entry_set_filetype(entry, AE_IFREG);
archive_entry_set_size(entry, 0);
assertEqualInt(ARCHIVE_OK, archive_write_header(a, entry));
archive_entry_free(entry);
assertEqualInt(ARCHIVE_OK, archive_write_free(a));
/* Check UTF-8 version. */
assertEqualMem(buff + 512, "16 path=\xE8\xA1\xA8.txt\x0A", 16);
}
/*
* Verify that CP1251 filenames are correctly translated to Unicode and UTF-8.
*/
static void
test_pax_filename_encoding_CP1251(void)
{
struct archive *a;
struct archive_entry *entry;
char buff[4096];
size_t used;
if (NULL == setlocale(LC_ALL, "Russian_Russia") &&
NULL == setlocale(LC_ALL, "ru_RU.CP1251")) {
skipping("KOI8-R locale not available on this system.");
return;
}
/* Check if the paltform completely supports the string conversion. */
a = archive_write_new();
assertEqualInt(ARCHIVE_OK, archive_write_set_format_pax(a));
if (archive_write_set_options(a, "hdrcharset=UTF-8") != ARCHIVE_OK) {
skipping("This system cannot convert character-set"
" from KOI8-R to UTF-8.");
archive_write_free(a);
return;
}
archive_write_free(a);
/* Re-create a write archive object since filenames should be written
* in UTF-8 by default. */
a = archive_write_new();
assertEqualInt(ARCHIVE_OK, archive_write_set_format_pax(a));
assertEqualInt(ARCHIVE_OK,
archive_write_open_memory(a, buff, sizeof(buff), &used));
entry = archive_entry_new2(a);
archive_entry_set_pathname(entry, "\xef\xf0\xe8");
archive_entry_set_filetype(entry, AE_IFREG);
archive_entry_set_size(entry, 0);
assertEqualInt(ARCHIVE_OK, archive_write_header(a, entry));
archive_entry_free(entry);
assertEqualInt(ARCHIVE_OK, archive_write_free(a));
/* Above three characters in KOI8-R should translate to the following
* three characters (two bytes each) in UTF-8. */
assertEqualMem(buff + 512, "15 path=\xD0\xBF\xD1\x80\xD0\xB8\x0A", 15);
}
/*
* Set output format to 'restricted pax' format.
*
* This is the same as normal 'pax', but tries to suppress
* the pax header whenever possible. This is the default for
* bsdtar, for instance.
*/
int
archive_write_set_format_pax_restricted(struct archive *_a)
{
struct archive_write *a = (struct archive_write *)_a;
int r;
r = archive_write_set_format_pax(&a->archive);
a->archive.archive_format = ARCHIVE_FORMAT_TAR_PAX_RESTRICTED;
a->archive.archive_format_name = "restricted POSIX pax interchange";
return (r);
}
struct archive * DTAR_new_archive() {
struct archive *a = archive_write_new();
archive_write_set_format_pax(a);
int r = archive_write_open_fd(a, DTAR_writer.fd_tar);
if ( r != ARCHIVE_OK) {
MFU_LOG(MFU_LOG_ERR, "archive_write_open_fd(): %s", archive_error_string(a));
return NULL;
}
return a;
}
/*
* Pax format writer only accepts both BINARY and UTF-8.
* If other character-set name is specified, you will get ARCHIVE_FAILED.
*/
static void
test_pax_filename_encoding_KOI8R_CP1251(void)
{
struct archive *a;
if (NULL == setlocale(LC_ALL, "ru_RU.KOI8-R")) {
skipping("KOI8-R locale not available on this system.");
return;
}
a = archive_write_new();
assertEqualInt(ARCHIVE_OK, archive_write_set_format_pax(a));
/* pax format writer only accepts both BINARY and UTF-8. */
assertEqualInt(ARCHIVE_FAILED,
archive_write_set_options(a, "hdrcharset=CP1251"));
assertEqualInt(ARCHIVE_OK, archive_write_free(a));
}
void DTAR_write_header(struct archive *ar, uint64_t idx, uint64_t offset)
{
const char * fname = mfu_flist_file_get_name(DTAR_flist, idx);
/* fill up entry, FIXME: the uglyness of removing leading slash */
struct archive_entry *entry = archive_entry_new();
archive_entry_copy_pathname(entry, &fname[1]);
if (DTAR_user_opts.preserve) {
struct archive * source = archive_read_disk_new();
archive_read_disk_set_standard_lookup(source);
int fd = open(fname, O_RDONLY);
if (archive_read_disk_entry_from_file(source, entry, fd, NULL) != ARCHIVE_OK) {
MFU_LOG(MFU_LOG_ERR, "archive_read_disk_entry_from_file(): %s", archive_error_string(ar));
}
archive_read_free(source);
} else {
/* read stat info from mfu_flist */
struct stat stbuf;
mfu_lstat(fname, &stbuf);
archive_entry_copy_stat(entry, &stbuf);
const char* uname = mfu_flist_file_get_username(DTAR_flist, idx);
archive_entry_set_uname(entry, uname);
const char* gname = mfu_flist_file_get_groupname(DTAR_flist, idx);
archive_entry_set_gname(entry, gname);
}
/* write entry info to archive */
struct archive* dest = archive_write_new();
archive_write_set_format_pax(dest);
if (archive_write_open_fd(dest, DTAR_writer.fd_tar) != ARCHIVE_OK) {
MFU_LOG(MFU_LOG_ERR, "archive_write_open_fd(): %s", archive_error_string(ar));
}
lseek64(DTAR_writer.fd_tar, offset, SEEK_SET);
if (archive_write_header(dest, entry) != ARCHIVE_OK) {
MFU_LOG(MFU_LOG_ERR, "archive_write_header(): %s", archive_error_string(ar));
}
archive_entry_free(entry);
archive_write_free(dest);
}
/* ArchiveWriter::__construct {{{
*
*/
ZEND_METHOD(ArchiveWriter, __construct)
{
archive_file_t *arch = NULL;
int resource_id;
zval *this = getThis();
const char *error_string = NULL;
char *filename;
long error_num, filename_len, result, format=0, compression=0;
zend_error_handling error_handling;
zend_replace_error_handling(EH_THROW, ce_ArchiveException, &error_handling TSRMLS_CC);
if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "s|ll", &filename, &filename_len, &format, &compression) == FAILURE) {
zend_restore_error_handling(&error_handling TSRMLS_CC);
return;
}
#if PHP_API_VERSION < 20100412
if (PG(safe_mode) && (!php_checkuid(filename, NULL, CHECKUID_CHECK_FILE_AND_DIR))) {
zend_restore_error_handling(&error_handling TSRMLS_CC);
return;
}
#endif
if (php_check_open_basedir(filename TSRMLS_CC)) {
zend_restore_error_handling(&error_handling TSRMLS_CC);
return;
}
arch = (archive_file_t *) emalloc(sizeof(archive_file_t));
arch->stream = NULL;
ALLOC_HASHTABLE(arch->entries);
zend_hash_init(arch->entries, 10, NULL, _archive_entries_hash_dtor, 0);
arch->mode = PHP_ARCHIVE_WRITE_MODE;
arch->buf = emalloc(PHP_ARCHIVE_BUF_LEN + 1);
arch->filename = estrndup(filename, filename_len);
arch->arch = archive_write_new();
switch (compression) {
case PHP_ARCHIVE_COMPRESSION_GZIP:
if (archive_write_add_filter_gzip(arch->arch) != ARCHIVE_OK) {
php_error_docref(NULL TSRMLS_CC, E_WARNING, "Gzip compression is not supported in this build");
zend_restore_error_handling(&error_handling TSRMLS_CC);
return;
}
break;
case PHP_ARCHIVE_COMPRESSION_BZIP2:
if (archive_write_add_filter_bzip2(arch->arch) != ARCHIVE_OK) {
php_error_docref(NULL TSRMLS_CC, E_WARNING, "Bzip2 compression is not supported in this build");
zend_restore_error_handling(&error_handling TSRMLS_CC);
return;
}
break;
case 0: /* default value */
case PHP_ARCHIVE_COMPRESSION_NONE:
/* always supported */
break;
default:
php_error_docref(NULL TSRMLS_CC, E_WARNING, "Unsupported compression type %ld", compression);
zend_restore_error_handling(&error_handling TSRMLS_CC);
return;
break;
}
switch (format) {
case 0: /* default value */
case PHP_ARCHIVE_FORMAT_TAR:
case PHP_ARCHIVE_FORMAT_PAX_RESTRICTED:
archive_write_set_format_pax_restricted(arch->arch);
break;
case PHP_ARCHIVE_FORMAT_PAX:
archive_write_set_format_pax(arch->arch);
break;
case PHP_ARCHIVE_FORMAT_CPIO:
archive_write_set_format_cpio(arch->arch);
break;
case PHP_ARCHIVE_FORMAT_SHAR:
archive_write_set_format_shar(arch->arch);
break;
case PHP_ARCHIVE_FORMAT_USTAR:
archive_write_set_format_ustar(arch->arch);
break;
default:
php_error_docref(NULL TSRMLS_CC, E_WARNING, "Unsupported archive format: %ld", format);
zend_restore_error_handling(&error_handling TSRMLS_CC);
return;
break;
}
archive_write_set_bytes_per_block(arch->arch, DEFAULT_BYTES_PER_BLOCK);
result = archive_write_open(arch->arch, arch, _archive_open_clbk, _archive_write_clbk, _archive_close_clbk);
/* do not pad the last block */
archive_write_set_bytes_in_last_block(arch->arch, 1);
if (result) {
error_num = archive_errno(arch->arch);
error_string = archive_error_string(arch->arch);
efree(arch->filename);
efree(arch->buf);
efree(arch);
if (error_num && error_string) {
php_error_docref(NULL TSRMLS_CC, E_WARNING, "Failed to open file %s for writing: error #%ld, %s", filename, error_num, error_string);
}
else {
php_error_docref(NULL TSRMLS_CC, E_WARNING, "Failed to open file %s for writing: unknown error %ld", filename, result);
}
zend_restore_error_handling(&error_handling TSRMLS_CC);
return;
}
resource_id = zend_list_insert(arch,le_archive);
add_property_resource(this, "fd", resource_id);
zend_restore_error_handling(&error_handling TSRMLS_CC);
return;
}
static void
test_1(void)
{
struct archive_entry *ae;
struct archive *a;
size_t used;
size_t blocksize;
int64_t offset, length;
char *buff2;
size_t buff2_size = 0x13000;
char buff3[1024];
long i;
assert((buff2 = malloc(buff2_size)) != NULL);
/* Repeat the following for a variety of odd blocksizes. */
for (blocksize = 1; blocksize < 100000; blocksize += blocksize + 3) {
/* Create a new archive in memory. */
assert((a = archive_write_new()) != NULL);
assertEqualIntA(a, ARCHIVE_OK,
archive_write_set_format_pax(a));
assertEqualIntA(a, ARCHIVE_OK,
archive_write_set_compression_none(a));
assertEqualIntA(a, ARCHIVE_OK,
archive_write_set_bytes_per_block(a, (int)blocksize));
assertEqualIntA(a, ARCHIVE_OK,
archive_write_set_bytes_in_last_block(a, (int)blocksize));
assertEqualInt(blocksize,
archive_write_get_bytes_in_last_block(a));
assertEqualIntA(a, ARCHIVE_OK,
archive_write_open_memory(a, buff, sizeof(buff), &used));
assertEqualInt(blocksize,
archive_write_get_bytes_in_last_block(a));
/*
* Write a file to it.
*/
assert((ae = archive_entry_new()) != NULL);
archive_entry_set_mtime(ae, 1, 10);
assertEqualInt(1, archive_entry_mtime(ae));
assertEqualInt(10, archive_entry_mtime_nsec(ae));
archive_entry_copy_pathname(ae, "file");
assertEqualString("file", archive_entry_pathname(ae));
archive_entry_set_mode(ae, S_IFREG | 0755);
assertEqualInt(S_IFREG | 0755, archive_entry_mode(ae));
archive_entry_set_size(ae, 0x81000);
archive_entry_sparse_add_entry(ae, 0x10000, 0x1000);
archive_entry_sparse_add_entry(ae, 0x80000, 0x1000);
assertEqualIntA(a, ARCHIVE_OK, archive_write_header(a, ae));
archive_entry_free(ae);
memset(buff2, 'a', buff2_size);
for (i = 0; i < 0x81000;) {
size_t ws = buff2_size;
if (i + ws > 0x81000)
ws = 0x81000 - i;
assertEqualInt(ws,
archive_write_data(a, buff2, ws));
i += ws;
}
/* Close out the archive. */
assertEqualIntA(a, ARCHIVE_OK, archive_write_close(a));
assertEqualInt(ARCHIVE_OK, archive_write_free(a));
/* This calculation gives "the smallest multiple of
* the block size that is at least 11264 bytes". */
failure("blocksize=%d", blocksize);
assertEqualInt(((11264 - 1)/blocksize+1)*blocksize, used);
/*
* Now, read the data back.
*/
assert((a = archive_read_new()) != NULL);
assertEqualIntA(a, ARCHIVE_OK,
archive_read_support_format_all(a));
assertEqualIntA(a, ARCHIVE_OK,
archive_read_support_filter_all(a));
assertEqualIntA(a, ARCHIVE_OK,
archive_read_open_memory(a, buff, used));
assertEqualIntA(a, ARCHIVE_OK,
archive_read_next_header(a, &ae));
assertEqualInt(1, archive_entry_mtime(ae));
assertEqualInt(10, archive_entry_mtime_nsec(ae));
assertEqualInt(0, archive_entry_atime(ae));
assertEqualInt(0, archive_entry_ctime(ae));
assertEqualString("file", archive_entry_pathname(ae));
assertEqualInt(AE_IFREG, archive_entry_filetype(ae));
assertEqualInt(AE_IFREG | 0755, archive_entry_mode(ae));
assertEqualInt(0x81000, archive_entry_size(ae));
/* Verify sparse information. */
assertEqualInt(2, archive_entry_sparse_reset(ae));
assertEqualInt(0,
archive_entry_sparse_next(ae, &offset, &length));
assertEqualInt(0x10000, offset);
assertEqualInt(0x1000, length);
assertEqualInt(0,
archive_entry_sparse_next(ae, &offset, &length));
assertEqualInt(0x80000, offset);
assertEqualInt(0x1000, length);
/* Verify file contents. */
memset(buff3, 0, sizeof(buff3));
for (i = 0; i < 0x10000; i += 1024) {
assertEqualInt(1024, archive_read_data(a, buff2, 1024));
failure("Read data(0x%lx - 0x%lx) should be all zero",
i, i + 1024);
assertEqualMem(buff2, buff3, 1024);
}
memset(buff3, 'a', sizeof(buff3));
for (i = 0x10000; i < 0x11000; i += 1024) {
assertEqualInt(1024, archive_read_data(a, buff2, 1024));
failure("Read data(0x%lx - 0x%lx) should be all 'a'",
i, i + 1024);
assertEqualMem(buff2, buff3, 1024);
}
memset(buff3, 0, sizeof(buff3));
for (i = 0x11000; i < 0x80000; i += 1024) {
assertEqualInt(1024, archive_read_data(a, buff2, 1024));
failure("Read data(0x%lx - 0x%lx) should be all zero",
i, i + 1024);
assertEqualMem(buff2, buff3, 1024);
}
memset(buff3, 'a', sizeof(buff3));
for (i = 0x80000; i < 0x81000; i += 1024) {
assertEqualInt(1024, archive_read_data(a, buff2, 1024));
failure("Read data(0x%lx - 0x%lx) should be all 'a'",
i, i + 1024);
assertEqualMem(buff2, buff3, 1024);
}
/* Verify the end of the archive. */
assertEqualIntA(a, ARCHIVE_EOF,
archive_read_next_header(a, &ae));
assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a));
assertEqualInt(ARCHIVE_OK, archive_read_free(a));
}
free(buff2);
}
/*
* Set the locale and write a pathname containing invalid characters.
* This should work; the underlying implementation should automatically
* fall back to storing the pathname in binary.
*/
static void
test_pax_filename_encoding_2(void)
{
char filename[] = "abc\314\214mno\374xyz";
struct archive *a;
struct archive_entry *entry;
char buff[65536];
char longname[] = "abc\314\214mno\374xyz"
"/abc\314\214mno\374xyz/abcdefghijklmnopqrstuvwxyz"
"/abc\314\214mno\374xyz/abcdefghijklmnopqrstuvwxyz"
"/abc\314\214mno\374xyz/abcdefghijklmnopqrstuvwxyz"
"/abc\314\214mno\374xyz/abcdefghijklmnopqrstuvwxyz"
"/abc\314\214mno\374xyz/abcdefghijklmnopqrstuvwxyz"
"/abc\314\214mno\374xyz/abcdefghijklmnopqrstuvwxyz"
;
size_t used;
/*
* We need a starting locale which has invalid sequences.
* en_US.UTF-8 seems to be commonly supported.
*/
/* If it doesn't exist, just warn and return. */
if (NULL == setlocale(LC_ALL, "en_US.UTF-8")) {
skipping("invalid encoding tests require a suitable locale;"
" en_US.UTF-8 not available on this system");
return;
}
assert((a = archive_write_new()) != NULL);
assertEqualIntA(a, 0, archive_write_set_format_pax(a));
assertEqualIntA(a, 0, archive_write_add_filter_none(a));
assertEqualIntA(a, 0, archive_write_set_bytes_per_block(a, 0));
assertEqualInt(0,
archive_write_open_memory(a, buff, sizeof(buff), &used));
assert((entry = archive_entry_new()) != NULL);
/* Set pathname, gname, uname, hardlink to nonconvertible values. */
archive_entry_copy_pathname(entry, filename);
archive_entry_copy_gname(entry, filename);
archive_entry_copy_uname(entry, filename);
archive_entry_copy_hardlink(entry, filename);
archive_entry_set_filetype(entry, AE_IFREG);
failure("This should generate a warning for nonconvertible names.");
assertEqualInt(ARCHIVE_WARN, archive_write_header(a, entry));
archive_entry_free(entry);
assert((entry = archive_entry_new()) != NULL);
/* Set path, gname, uname, and symlink to nonconvertible values. */
archive_entry_copy_pathname(entry, filename);
archive_entry_copy_gname(entry, filename);
archive_entry_copy_uname(entry, filename);
archive_entry_copy_symlink(entry, filename);
archive_entry_set_filetype(entry, AE_IFLNK);
failure("This should generate a warning for nonconvertible names.");
assertEqualInt(ARCHIVE_WARN, archive_write_header(a, entry));
archive_entry_free(entry);
assert((entry = archive_entry_new()) != NULL);
/* Set pathname to a very long nonconvertible value. */
archive_entry_copy_pathname(entry, longname);
archive_entry_set_filetype(entry, AE_IFREG);
failure("This should generate a warning for nonconvertible names.");
assertEqualInt(ARCHIVE_WARN, archive_write_header(a, entry));
archive_entry_free(entry);
assertEqualIntA(a, ARCHIVE_OK, archive_write_close(a));
assertEqualInt(ARCHIVE_OK, archive_write_free(a));
/*
* Now read the entries back.
*/
assert((a = archive_read_new()) != NULL);
assertEqualInt(0, archive_read_support_format_tar(a));
assertEqualInt(0, archive_read_open_memory(a, buff, used));
assertEqualInt(0, archive_read_next_header(a, &entry));
assertEqualString(filename, archive_entry_pathname(entry));
assertEqualString(filename, archive_entry_gname(entry));
assertEqualString(filename, archive_entry_uname(entry));
assertEqualString(filename, archive_entry_hardlink(entry));
assertEqualInt(0, archive_read_next_header(a, &entry));
assertEqualString(filename, archive_entry_pathname(entry));
assertEqualString(filename, archive_entry_gname(entry));
assertEqualString(filename, archive_entry_uname(entry));
assertEqualString(filename, archive_entry_symlink(entry));
assertEqualInt(0, archive_read_next_header(a, &entry));
assertEqualString(longname, archive_entry_pathname(entry));
assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a));
assertEqualInt(ARCHIVE_OK, archive_read_free(a));
}
/*
* Create an entry starting from a wide-character Unicode pathname,
* read it back into "C" locale, which doesn't support the name.
* TODO: Figure out the "right" behavior here.
*/
static void
test_pax_filename_encoding_3(void)
{
wchar_t badname[] = L"xxxAyyyBzzz";
const char badname_utf8[] = "xxx\xE1\x88\xB4yyy\xE5\x99\xB8zzz";
struct archive *a;
struct archive_entry *entry;
char buff[65536];
size_t used;
badname[3] = 0x1234;
badname[7] = 0x5678;
/* If it doesn't exist, just warn and return. */
if (NULL == setlocale(LC_ALL, "C")) {
skipping("Can't set \"C\" locale, so can't exercise "
"certain character-conversion failures");
return;
}
/* If wctomb is broken, warn and return. */
if (wctomb(buff, 0x1234) > 0) {
skipping("Cannot test conversion failures because \"C\" "
"locale on this system has no invalid characters.");
return;
}
/* If wctomb is broken, warn and return. */
if (wctomb(buff, 0x1234) > 0) {
skipping("Cannot test conversion failures because \"C\" "
"locale on this system has no invalid characters.");
return;
}
/* Skip test if archive_entry_update_pathname_utf8() is broken. */
/* In particular, this is currently broken on Win32 because
* setlocale() does not set the default encoding for CP_ACP. */
entry = archive_entry_new();
if (archive_entry_update_pathname_utf8(entry, badname_utf8)) {
archive_entry_free(entry);
skipping("Cannot test conversion failures.");
return;
}
archive_entry_free(entry);
assert((a = archive_write_new()) != NULL);
assertEqualIntA(a, 0, archive_write_set_format_pax(a));
assertEqualIntA(a, 0, archive_write_add_filter_none(a));
assertEqualIntA(a, 0, archive_write_set_bytes_per_block(a, 0));
assertEqualInt(0,
archive_write_open_memory(a, buff, sizeof(buff), &used));
assert((entry = archive_entry_new()) != NULL);
/* Set pathname to non-convertible wide value. */
archive_entry_copy_pathname_w(entry, badname);
archive_entry_set_filetype(entry, AE_IFREG);
assertEqualInt(ARCHIVE_OK, archive_write_header(a, entry));
archive_entry_free(entry);
assert((entry = archive_entry_new()) != NULL);
archive_entry_copy_pathname_w(entry, L"abc");
/* Set gname to non-convertible wide value. */
archive_entry_copy_gname_w(entry, badname);
archive_entry_set_filetype(entry, AE_IFREG);
assertEqualInt(ARCHIVE_OK, archive_write_header(a, entry));
archive_entry_free(entry);
assert((entry = archive_entry_new()) != NULL);
archive_entry_copy_pathname_w(entry, L"abc");
/* Set uname to non-convertible wide value. */
archive_entry_copy_uname_w(entry, badname);
archive_entry_set_filetype(entry, AE_IFREG);
assertEqualInt(ARCHIVE_OK, archive_write_header(a, entry));
archive_entry_free(entry);
assert((entry = archive_entry_new()) != NULL);
archive_entry_copy_pathname_w(entry, L"abc");
/* Set hardlink to non-convertible wide value. */
archive_entry_copy_hardlink_w(entry, badname);
archive_entry_set_filetype(entry, AE_IFREG);
assertEqualInt(ARCHIVE_OK, archive_write_header(a, entry));
archive_entry_free(entry);
assert((entry = archive_entry_new()) != NULL);
archive_entry_copy_pathname_w(entry, L"abc");
/* Set symlink to non-convertible wide value. */
archive_entry_copy_symlink_w(entry, badname);
archive_entry_set_filetype(entry, AE_IFLNK);
assertEqualInt(ARCHIVE_OK, archive_write_header(a, entry));
archive_entry_free(entry);
assertEqualIntA(a, ARCHIVE_OK, archive_write_close(a));
assertEqualInt(ARCHIVE_OK, archive_write_free(a));
/*
* Now read the entries back.
*/
assert((a = archive_read_new()) != NULL);
assertEqualInt(0, archive_read_support_format_tar(a));
assertEqualInt(0, archive_read_open_memory(a, buff, used));
failure("A non-convertible pathname should cause a warning.");
assertEqualInt(ARCHIVE_WARN, archive_read_next_header(a, &entry));
assertEqualWString(badname, archive_entry_pathname_w(entry));
failure("If native locale can't convert, we should get UTF-8 back.");
assertEqualString(badname_utf8, archive_entry_pathname(entry));
failure("A non-convertible gname should cause a warning.");
assertEqualInt(ARCHIVE_WARN, archive_read_next_header(a, &entry));
assertEqualWString(badname, archive_entry_gname_w(entry));
failure("If native locale can't convert, we should get UTF-8 back.");
assertEqualString(badname_utf8, archive_entry_gname(entry));
failure("A non-convertible uname should cause a warning.");
assertEqualInt(ARCHIVE_WARN, archive_read_next_header(a, &entry));
assertEqualWString(badname, archive_entry_uname_w(entry));
failure("If native locale can't convert, we should get UTF-8 back.");
assertEqualString(badname_utf8, archive_entry_uname(entry));
failure("A non-convertible hardlink should cause a warning.");
assertEqualInt(ARCHIVE_WARN, archive_read_next_header(a, &entry));
assertEqualWString(badname, archive_entry_hardlink_w(entry));
failure("If native locale can't convert, we should get UTF-8 back.");
assertEqualString(badname_utf8, archive_entry_hardlink(entry));
failure("A non-convertible symlink should cause a warning.");
assertEqualInt(ARCHIVE_WARN, archive_read_next_header(a, &entry));
assertEqualWString(badname, archive_entry_symlink_w(entry));
assertEqualWString(NULL, archive_entry_hardlink_w(entry));
failure("If native locale can't convert, we should get UTF-8 back.");
assertEqualString(badname_utf8, archive_entry_symlink(entry));
assertEqualInt(ARCHIVE_EOF, archive_read_next_header(a, &entry));
assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a));
assertEqualInt(ARCHIVE_OK, archive_read_free(a));
}
static void mfu_flist_archive_create_libcircle(mfu_flist flist, const char* archivefile, mfu_archive_options_t* opts)
{
DTAR_flist = flist;
DTAR_user_opts = *opts;
MPI_Comm_rank(MPI_COMM_WORLD, &DTAR_rank);
/* TODO: stripe the archive file if on parallel file system */
/* init statistics */
DTAR_statistics.total_dirs = 0;
DTAR_statistics.total_files = 0;
DTAR_statistics.total_links = 0;
DTAR_statistics.total_size = 0;
DTAR_statistics.total_bytes_copied = 0;
time(&(DTAR_statistics.time_started));
DTAR_statistics.wtime_started = MPI_Wtime();
/* create the archive file */
DTAR_writer.name = archivefile;
DTAR_writer.flags = O_WRONLY | O_CREAT | O_CLOEXEC | O_LARGEFILE;
DTAR_writer.fd_tar = open(archivefile, DTAR_writer.flags, 0664);
/* get number of items in our portion of the list */
DTAR_count = mfu_flist_size(DTAR_flist);
/* allocate memory for file sizes and offsets */
uint64_t* fsizes = (uint64_t*) MFU_MALLOC(DTAR_count * sizeof(uint64_t));
DTAR_offsets = (uint64_t*) MFU_MALLOC(DTAR_count * sizeof(uint64_t));
/* compute local offsets for each item and total
* bytes we're contributing to the archive */
uint64_t idx;
uint64_t offset = 0;
for (idx = 0; idx < DTAR_count; idx++) {
/* assume the item takes no space */
fsizes[idx] = 0;
/* identify item type to compute its size in the archive */
mfu_filetype type = mfu_flist_file_get_type(DTAR_flist, idx);
if (type == MFU_TYPE_DIR || type == MFU_TYPE_LINK) {
/* directories and symlinks only need the header */
fsizes[idx] = DTAR_HDR_LENGTH;
} else if (type == MFU_TYPE_FILE) {
/* regular file requires a header, plus file content,
* and things are packed into blocks of 512 bytes */
uint64_t fsize = mfu_flist_file_get_size(DTAR_flist, idx);
/* determine whether file size is integer multiple of 512 bytes */
uint64_t rem = fsize % 512;
if (rem == 0) {
/* file content is multiple of 512 bytes, so perfect fit */
fsizes[idx] = fsize + DTAR_HDR_LENGTH;
} else {
/* TODO: check and explain this math */
fsizes[idx] = (fsize / 512 + 4) * 512;
}
}
/* increment our local offset for this item */
DTAR_offsets[idx] = offset;
offset += fsizes[idx];
}
/* execute scan to figure our global base offset in the archive file */
uint64_t global_offset = 0;
MPI_Scan(&offset, &global_offset, 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);
global_offset -= offset;
/* update offsets for each of our file to their global offset */
for (idx = 0; idx < DTAR_count; idx++) {
DTAR_offsets[idx] += global_offset;
}
/* create an archive */
struct archive* ar = archive_write_new();
archive_write_set_format_pax(ar);
int r = archive_write_open_fd(ar, DTAR_writer.fd_tar);
if (r != ARCHIVE_OK) {
MFU_LOG(MFU_LOG_ERR, "archive_write_open_fd(): %s", archive_error_string(ar));
DTAR_abort(EXIT_FAILURE);
}
/* write headers for our files */
for (idx = 0; idx < DTAR_count; idx++) {
mfu_filetype type = mfu_flist_file_get_type(DTAR_flist, idx);
if (type == MFU_TYPE_FILE || type == MFU_TYPE_DIR || type == MFU_TYPE_LINK) {
DTAR_write_header(ar, idx, DTAR_offsets[idx]);
}
}
/* prepare libcircle */
CIRCLE_init(0, NULL, CIRCLE_SPLIT_EQUAL | CIRCLE_CREATE_GLOBAL);
CIRCLE_loglevel loglevel = CIRCLE_LOG_WARN;
CIRCLE_enable_logging(loglevel);
/* register callbacks */
CIRCLE_cb_create(&DTAR_enqueue_copy);
CIRCLE_cb_process(&DTAR_perform_copy);
/* run the libcircle job to copy data into archive file */
CIRCLE_begin();
CIRCLE_finalize();
/* compute total bytes copied */
uint64_t archive_size = 0;
MPI_Allreduce(&offset, &archive_size, 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);
DTAR_statistics.total_size = archive_size;
DTAR_statistics.wtime_ended = MPI_Wtime();
time(&(DTAR_statistics.time_ended));
/* print stats */
double rel_time = DTAR_statistics.wtime_ended - \
DTAR_statistics.wtime_started;
if (DTAR_rank == 0) {
char starttime_str[256];
struct tm* localstart = localtime(&(DTAR_statistics.time_started));
strftime(starttime_str, 256, "%b-%d-%Y, %H:%M:%S", localstart);
char endtime_str[256];
struct tm* localend = localtime(&(DTAR_statistics.time_ended));
strftime(endtime_str, 256, "%b-%d-%Y, %H:%M:%S", localend);
/* add two 512 blocks at the end */
DTAR_statistics.total_size += 512 * 2;
/* convert bandwidth to unit */
double agg_rate_tmp;
double agg_rate = (double) DTAR_statistics.total_size / rel_time;
const char* agg_rate_units;
mfu_format_bytes(agg_rate, &agg_rate_tmp, &agg_rate_units);
MFU_LOG(MFU_LOG_INFO, "Started: %s", starttime_str);
MFU_LOG(MFU_LOG_INFO, "Completed: %s", endtime_str);
MFU_LOG(MFU_LOG_INFO, "Total archive size: %" PRIu64, DTAR_statistics.total_size);
MFU_LOG(MFU_LOG_INFO, "Rate: %.3lf %s " \
"(%.3" PRIu64 " bytes in %.3lf seconds)", \
agg_rate_tmp, agg_rate_units, DTAR_statistics.total_size, rel_time);
}
/* clean up */
mfu_free(&fsizes);
mfu_free(&DTAR_offsets);
/* close archive file */
archive_write_free(ar);
mfu_close(DTAR_writer.name, DTAR_writer.fd_tar);
}
const gchar* archive_create(const char* archive_name, GSList* files,
COMPRESS_METHOD method, ARCHIVE_FORMAT format) {
struct archive* arch;
struct archive_entry* entry;
char* buf = NULL;
ssize_t len;
int fd;
struct stat st;
struct file_info* file;
gchar* filename = NULL;
gchar* msg = NULL;
#ifndef _TEST
gint num = 0;
gint total = g_slist_length (files);
#endif
g_return_val_if_fail(files != NULL, "No files for archiving");
debug_print("File: %s\n", archive_name);
arch = archive_write_new();
switch (method) {
case ZIP:
if (archive_write_set_compression_gzip(arch) != ARCHIVE_OK)
return archive_error_string(arch);
break;
case BZIP2:
if (archive_write_set_compression_bzip2(arch) != ARCHIVE_OK)
return archive_error_string(arch);
break;
#if NEW_ARCHIVE_API
case COMPRESS:
if (archive_write_set_compression_compress(arch) != ARCHIVE_OK)
return archive_error_string(arch);
break;
#endif
case NO_COMPRESS:
if (archive_write_set_compression_none(arch) != ARCHIVE_OK)
return archive_error_string(arch);
break;
}
switch (format) {
case TAR:
if (archive_write_set_format_ustar(arch) != ARCHIVE_OK)
return archive_error_string(arch);
break;
case SHAR:
if (archive_write_set_format_shar(arch) != ARCHIVE_OK)
return archive_error_string(arch);
break;
case PAX:
if (archive_write_set_format_pax(arch) != ARCHIVE_OK)
return archive_error_string(arch);
break;
case CPIO:
if (archive_write_set_format_cpio(arch) != ARCHIVE_OK)
return archive_error_string(arch);
break;
case NO_FORMAT:
return "Missing archive format";
}
if (archive_write_open_file(arch, archive_name) != ARCHIVE_OK)
return archive_error_string(arch);
while (files && ! stop_action) {
#ifndef _TEST
set_progress_print_all(num++, total, 30);
#endif
file = (struct file_info *) files->data;
if (!file)
continue;
filename = get_full_path(file);
/* libarchive will crash if instructed to add archive to it self */
if (g_utf8_collate(archive_name, filename) == 0) {
buf = NULL;
buf = g_strdup_printf(
"%s: Not dumping to %s", archive_name, filename);
g_warning("%s\n", buf);
#ifndef _TEST
debug_print("%s\n", buf);
#endif
g_free(buf);
}
else {
#ifndef _TEST
debug_print("Adding: %s\n", filename);
msg = g_strdup_printf("%s", filename);
set_progress_file_label(msg);
g_free(msg);
#endif
entry = archive_entry_new();
lstat(filename, &st);
if ((fd = open(filename, O_RDONLY)) == -1) {
perror("open file");
}
else {
archive_entry_copy_stat(entry, &st);
archive_entry_set_pathname(entry, filename);
if (S_ISLNK(st.st_mode)) {
buf = NULL;
buf = malloc(PATH_MAX + 1);
if ((len = readlink(filename, buf, PATH_MAX)) < 0)
perror("error in readlink");
else
buf[len] = '\0';
archive_entry_set_symlink(entry, buf);
g_free(buf);
archive_entry_set_size(entry, 0);
archive_write_header(arch, entry);
}
else {
if (archive_write_header(arch, entry) != ARCHIVE_OK)
g_warning("%s", archive_error_string(arch));
buf = NULL;
buf = malloc(READ_BLOCK_SIZE);
len = read(fd, buf, READ_BLOCK_SIZE);
while (len > 0) {
if (archive_write_data(arch, buf, len) == -1)
g_warning("%s", archive_error_string(arch));
memset(buf, 0, READ_BLOCK_SIZE);
len = read(fd, buf, READ_BLOCK_SIZE);
}
g_free(buf);
}
close(fd);
archive_entry_free(entry);
}
}
g_free(filename);
files = g_slist_next(files);
}
#ifndef _TEST
if (stop_action)
unlink(archive_name);
stop_action = FALSE;
#endif
archive_write_close(arch);
archive_write_finish(arch);
return NULL;
}
