int
main(void)
{
test_equal();
test_overflow();
lzma_index *i = create_empty();
test_many(i);
lzma_index_end(i, NULL);
i = create_small();
test_many(i);
lzma_index_end(i, NULL);
i = create_big();
test_many(i);
lzma_index_end(i, NULL);
test_cat();
test_locate();
test_corrupt();
return 0;
}
static void
test_copy(const lzma_index *i)
{
lzma_index *d = lzma_index_dup(i, NULL);
expect(d != NULL);
expect(is_equal(i, d));
lzma_index_end(d, NULL);
}
static void
test_equal(void)
{
lzma_index *a = create_empty();
lzma_index *b = create_small();
lzma_index *c = create_big();
expect(a && b && c);
expect(is_equal(a, a));
expect(is_equal(b, b));
expect(is_equal(c, c));
expect(!is_equal(a, b));
expect(!is_equal(a, c));
expect(!is_equal(b, c));
lzma_index_end(a, NULL);
lzma_index_end(b, NULL);
lzma_index_end(c, NULL);
}
/*
* Small wrapper to extract total length of a file
*/
off_t
unxz_len(int fd)
{
xz_file_info xfi = XZ_FILE_INFO_INIT;
if (!parse_indexes(&xfi, fd)) {
off_t res = lzma_index_uncompressed_size(xfi.idx);
lzma_index_end(xfi.idx, NULL);
return res;
}
return 0;
}
int
main(void)
{
test_equal();
test_overflow();
lzma_index *i = create_empty();
test_many(i);
lzma_index_end(i, NULL);
i = create_small();
test_many(i);
lzma_index_end(i, NULL);
i = create_big();
test_many(i);
lzma_index_end(i, NULL);
test_cat();
test_locate();
test_corrupt();
// Test for the bug fix 21515d79d778b8730a434f151b07202d52a04611:
// liblzma: Fix lzma_index_dup() for empty Streams.
i = create_empty();
expect(lzma_index_stream_padding(i, 4) == LZMA_OK);
test_copy(i);
lzma_index_end(i, NULL);
// Test for the bug fix 3bf857edfef51374f6f3fffae3d817f57d3264a0:
// liblzma: Fix a memory leak in error path of lzma_index_dup().
// Use Valgrind to see that there are no leaks.
i = create_small();
expect(lzma_index_dup(i, &my_allocator) == NULL);
lzma_index_end(i, NULL);
return 0;
}
static void
test_overflow(void)
{
// Integer overflow tests
lzma_index *i = create_empty();
expect(lzma_index_append(i, NULL, LZMA_VLI_MAX - 5, 1234)
== LZMA_DATA_ERROR);
// TODO
lzma_index_end(i, NULL);
}
static void
test_corrupt(void)
{
const size_t alloc_size = 128 * 1024;
uint8_t *buf = malloc(alloc_size);
expect(buf != NULL);
lzma_stream strm = LZMA_STREAM_INIT;
lzma_index *i = create_empty();
expect(lzma_index_append(i, NULL, 0, 1) == LZMA_PROG_ERROR);
lzma_index_end(i, NULL);
// Create a valid Index and corrupt it in different ways.
i = create_small();
expect(lzma_index_encoder(&strm, i) == LZMA_OK);
succeed(coder_loop(&strm, NULL, 0, buf, 20,
LZMA_STREAM_END, LZMA_RUN));
lzma_index_end(i, NULL);
// Wrong Index Indicator
buf[0] ^= 1;
expect(lzma_index_decoder(&strm, &i, MEMLIMIT) == LZMA_OK);
succeed(decoder_loop_ret(&strm, buf, 1, LZMA_DATA_ERROR));
buf[0] ^= 1;
// Wrong Number of Records and thus CRC32 fails.
--buf[1];
expect(lzma_index_decoder(&strm, &i, MEMLIMIT) == LZMA_OK);
succeed(decoder_loop_ret(&strm, buf, 10, LZMA_DATA_ERROR));
++buf[1];
// Padding not NULs
buf[15] ^= 1;
expect(lzma_index_decoder(&strm, &i, MEMLIMIT) == LZMA_OK);
succeed(decoder_loop_ret(&strm, buf, 16, LZMA_DATA_ERROR));
lzma_end(&strm);
free(buf);
}
static int
lzma_close (struct bfd *nbfd,
void *stream)
{
struct lzma_stream *lstream = stream;
lzma_index_end (lstream->index, &gdb_lzma_allocator);
xfree (lstream->data);
xfree (lstream);
/* Zero means success. */
return 0;
}
static void
stream_encoder_end(lzma_coder *coder, const lzma_allocator *allocator)
{
lzma_next_end(&coder->block_encoder, allocator);
lzma_next_end(&coder->index_encoder, allocator);
lzma_index_end(coder->index, allocator);
for (size_t i = 0; coder->filters[i].id != LZMA_VLI_UNKNOWN; ++i)
lzma_free(coder->filters[i].options, allocator);
lzma_free(coder, allocator);
return;
}
extern void
list_file(const char *filename)
{
if (opt_format != FORMAT_XZ && opt_format != FORMAT_AUTO)
message_fatal(_("--list works only on .xz files "
"(--format=xz or --format=auto)"));
message_filename(filename);
if (filename == stdin_filename) {
message_error(_("--list does not support reading from "
"standard input"));
return;
}
// Unset opt_stdout so that io_open_src() won't accept special files.
// Set opt_force so that io_open_src() will follow symlinks.
opt_stdout = false;
opt_force = true;
file_pair *pair = io_open_src(filename);
if (pair == NULL)
return;
xz_file_info xfi = XZ_FILE_INFO_INIT;
if (!parse_indexes(&xfi, pair)) {
bool fail;
// We have three main modes:
// - --robot, which has submodes if --verbose is specified
// once or twice
// - Normal --list without --verbose
// - --list with one or two --verbose
if (opt_robot)
fail = print_info_robot(&xfi, pair);
else if (message_verbosity_get() <= V_WARNING)
fail = print_info_basic(&xfi, pair);
else
fail = print_info_adv(&xfi, pair);
// Update the totals that are displayed after all
// the individual files have been listed. Don't count
// broken files.
if (!fail)
update_totals(&xfi);
lzma_index_end(xfi.idx, NULL);
}
io_close(pair, false);
return;
}
/// \brief Parse the Index(es) from the given .xz file
///
/// \param xfi Pointer to structure where the decoded information
/// is stored.
/// \param pair Input file
///
/// \return On success, false is returned. On error, true is returned.
///
// TODO: This function is pretty big. liblzma should have a function that
// takes a callback function to parse the Index(es) from a .xz file to make
// it easy for applications.
static bool
parse_indexes(xz_file_info *xfi, file_pair *pair)
{
if (pair->src_st.st_size <= 0) {
message_error(_("%s: File is empty"), pair->src_name);
return true;
}
if (pair->src_st.st_size < 2 * LZMA_STREAM_HEADER_SIZE) {
message_error(_("%s: Too small to be a valid .xz file"),
pair->src_name);
return true;
}
io_buf buf;
lzma_stream_flags header_flags;
lzma_stream_flags footer_flags;
lzma_ret ret;
// lzma_stream for the Index decoder
lzma_stream strm = LZMA_STREAM_INIT;
// All Indexes decoded so far
lzma_index *combined_index = NULL;
// The Index currently being decoded
lzma_index *this_index = NULL;
// Current position in the file. We parse the file backwards so
// initialize it to point to the end of the file.
off_t pos = pair->src_st.st_size;
// Each loop iteration decodes one Index.
do {
// Check that there is enough data left to contain at least
// the Stream Header and Stream Footer. This check cannot
// fail in the first pass of this loop.
if (pos < 2 * LZMA_STREAM_HEADER_SIZE) {
message_error("%s: %s", pair->src_name,
message_strm(LZMA_DATA_ERROR));
goto error;
}
pos -= LZMA_STREAM_HEADER_SIZE;
lzma_vli stream_padding = 0;
// Locate the Stream Footer. There may be Stream Padding which
// we must skip when reading backwards.
while (true) {
if (pos < LZMA_STREAM_HEADER_SIZE) {
message_error("%s: %s", pair->src_name,
message_strm(
LZMA_DATA_ERROR));
goto error;
}
if (io_pread(pair, &buf,
LZMA_STREAM_HEADER_SIZE, pos))
goto error;
// Stream Padding is always a multiple of four bytes.
int i = 2;
if (buf.u32[i] != 0)
break;
// To avoid calling io_pread() for every four bytes
// of Stream Padding, take advantage that we read
// 12 bytes (LZMA_STREAM_HEADER_SIZE) already and
// check them too before calling io_pread() again.
do {
stream_padding += 4;
pos -= 4;
--i;
} while (i >= 0 && buf.u32[i] == 0);
}
// Decode the Stream Footer.
ret = lzma_stream_footer_decode(&footer_flags, buf.u8);
if (ret != LZMA_OK) {
message_error("%s: %s", pair->src_name,
message_strm(ret));
goto error;
}
// Check that the size of the Index field looks sane.
lzma_vli index_size = footer_flags.backward_size;
if ((lzma_vli)(pos) < index_size + LZMA_STREAM_HEADER_SIZE) {
message_error("%s: %s", pair->src_name,
message_strm(LZMA_DATA_ERROR));
goto error;
}
// Set pos to the beginning of the Index.
pos -= index_size;
// See how much memory we can use for decoding this Index.
uint64_t memlimit = hardware_memlimit_get(MODE_LIST);
uint64_t memused = 0;
if (combined_index != NULL) {
memused = lzma_index_memused(combined_index);
if (memused > memlimit)
message_bug();
memlimit -= memused;
}
// Decode the Index.
ret = lzma_index_decoder(&strm, &this_index, memlimit);
if (ret != LZMA_OK) {
message_error("%s: %s", pair->src_name,
message_strm(ret));
goto error;
}
do {
// Don't give the decoder more input than the
// Index size.
strm.avail_in = my_min(IO_BUFFER_SIZE, index_size);
if (io_pread(pair, &buf, strm.avail_in, pos))
goto error;
pos += strm.avail_in;
index_size -= strm.avail_in;
strm.next_in = buf.u8;
ret = lzma_code(&strm, LZMA_RUN);
} while (ret == LZMA_OK);
// If the decoding seems to be successful, check also that
// the Index decoder consumed as much input as indicated
// by the Backward Size field.
if (ret == LZMA_STREAM_END)
if (index_size != 0 || strm.avail_in != 0)
ret = LZMA_DATA_ERROR;
if (ret != LZMA_STREAM_END) {
// LZMA_BUFFER_ERROR means that the Index decoder
// would have liked more input than what the Index
// size should be according to Stream Footer.
// The message for LZMA_DATA_ERROR makes more
// sense in that case.
if (ret == LZMA_BUF_ERROR)
ret = LZMA_DATA_ERROR;
message_error("%s: %s", pair->src_name,
message_strm(ret));
// If the error was too low memory usage limit,
// show also how much memory would have been needed.
if (ret == LZMA_MEMLIMIT_ERROR) {
uint64_t needed = lzma_memusage(&strm);
if (UINT64_MAX - needed < memused)
needed = UINT64_MAX;
else
needed += memused;
message_mem_needed(V_ERROR, needed);
}
goto error;
}
// Decode the Stream Header and check that its Stream Flags
// match the Stream Footer.
pos -= footer_flags.backward_size + LZMA_STREAM_HEADER_SIZE;
if ((lzma_vli)(pos) < lzma_index_total_size(this_index)) {
message_error("%s: %s", pair->src_name,
message_strm(LZMA_DATA_ERROR));
goto error;
}
pos -= lzma_index_total_size(this_index);
if (io_pread(pair, &buf, LZMA_STREAM_HEADER_SIZE, pos))
goto error;
ret = lzma_stream_header_decode(&header_flags, buf.u8);
if (ret != LZMA_OK) {
message_error("%s: %s", pair->src_name,
message_strm(ret));
goto error;
}
ret = lzma_stream_flags_compare(&header_flags, &footer_flags);
if (ret != LZMA_OK) {
message_error("%s: %s", pair->src_name,
message_strm(ret));
goto error;
}
// Store the decoded Stream Flags into this_index. This is
// needed so that we can print which Check is used in each
// Stream.
ret = lzma_index_stream_flags(this_index, &footer_flags);
if (ret != LZMA_OK)
message_bug();
// Store also the size of the Stream Padding field. It is
// needed to show the offsets of the Streams correctly.
ret = lzma_index_stream_padding(this_index, stream_padding);
if (ret != LZMA_OK)
message_bug();
if (combined_index != NULL) {
// Append the earlier decoded Indexes
// after this_index.
ret = lzma_index_cat(
this_index, combined_index, NULL);
if (ret != LZMA_OK) {
message_error("%s: %s", pair->src_name,
message_strm(ret));
goto error;
}
}
combined_index = this_index;
this_index = NULL;
xfi->stream_padding += stream_padding;
} while (pos > 0);
lzma_end(&strm);
// All OK. Make combined_index available to the caller.
xfi->idx = combined_index;
return false;
error:
// Something went wrong, free the allocated memory.
lzma_end(&strm);
lzma_index_end(combined_index, NULL);
lzma_index_end(this_index, NULL);
return true;
}
lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check,
lzma_allocator *allocator, const uint8_t *in, size_t in_size,
uint8_t *out, size_t *out_pos_ptr, size_t out_size)
{
// Sanity checks
if (filters == NULL || (unsigned int)(check) > LZMA_CHECK_ID_MAX
|| (in == NULL && in_size != 0) || out == NULL
|| out_pos_ptr == NULL || *out_pos_ptr > out_size)
return LZMA_PROG_ERROR;
if (!lzma_check_is_supported(check))
return LZMA_UNSUPPORTED_CHECK;
// Note for the paranoids: Index encoder prevents the Stream from
// getting too big and still being accepted with LZMA_OK, and Block
// encoder catches if the input is too big. So we don't need to
// separately check if the buffers are too big.
// Use a local copy. We update *out_pos_ptr only if everything
// succeeds.
size_t out_pos = *out_pos_ptr;
// Check that there's enough space for both Stream Header and
// Stream Footer.
if (out_size - out_pos <= 2 * LZMA_STREAM_HEADER_SIZE)
return LZMA_BUF_ERROR;
// Reserve space for Stream Footer so we don't need to check for
// available space again before encoding Stream Footer.
out_size -= LZMA_STREAM_HEADER_SIZE;
// Encode the Stream Header.
lzma_stream_flags stream_flags = {
.version = 0,
.check = check,
};
if (lzma_stream_header_encode(&stream_flags, out + out_pos)
!= LZMA_OK)
return LZMA_PROG_ERROR;
out_pos += LZMA_STREAM_HEADER_SIZE;
// Encode a Block but only if there is at least one byte of input.
lzma_block block = {
.version = 0,
.check = check,
.filters = filters,
};
if (in_size > 0)
return_if_error(lzma_block_buffer_encode(&block, allocator,
in, in_size, out, &out_pos, out_size));
// Index
{
// Create an Index. It will have one Record if there was
// at least one byte of input to encode. Otherwise the
// Index will be empty.
lzma_index *i = lzma_index_init(allocator);
if (i == NULL)
return LZMA_MEM_ERROR;
lzma_ret ret = LZMA_OK;
if (in_size > 0)
ret = lzma_index_append(i, allocator,
lzma_block_unpadded_size(&block),
block.uncompressed_size);
// If adding the Record was successful, encode the Index
// and get its size which will be stored into Stream Footer.
if (ret == LZMA_OK) {
ret = lzma_index_buffer_encode(
i, out, &out_pos, out_size);
stream_flags.backward_size = lzma_index_size(i);
}
lzma_index_end(i, allocator);
if (ret != LZMA_OK)
return ret;
}
// Stream Footer. We have already reserved space for this.
if (lzma_stream_footer_encode(&stream_flags, out + out_pos)
!= LZMA_OK)
return LZMA_PROG_ERROR;
out_pos += LZMA_STREAM_HEADER_SIZE;
// Everything went fine, make the new output position available
// to the application.
*out_pos_ptr = out_pos;
return LZMA_OK;
}
/// \brief Parse the Index(es) from the given .xz file
///
/// \param xfi Pointer to structure where the decoded information
/// is stored.
/// \param pair Input file
///
/// \return On success, false is returned. On error, true is returned.
///
// TODO: This function is pretty big. liblzma should have a function that
// takes a callback function to parse the Index(es) from a .xz file to make
// it easy for applications.
static bool
parse_indexes(xz_file_info *xfi, int src_fd)
{
struct stat st;
fstat(src_fd, &st);
if (st.st_size <= 0) {
return true;
}
if (st.st_size < 2 * LZMA_STREAM_HEADER_SIZE) {
return true;
}
io_buf buf;
lzma_stream_flags header_flags;
lzma_stream_flags footer_flags;
lzma_ret ret;
// lzma_stream for the Index decoder
lzma_stream strm = LZMA_STREAM_INIT;
// All Indexes decoded so far
lzma_index *combined_index = NULL;
// The Index currently being decoded
lzma_index *this_index = NULL;
// Current position in the file. We parse the file backwards so
// initialize it to point to the end of the file.
off_t pos = st.st_size;
// Each loop iteration decodes one Index.
do {
// Check that there is enough data left to contain at least
// the Stream Header and Stream Footer. This check cannot
// fail in the first pass of this loop.
if (pos < 2 * LZMA_STREAM_HEADER_SIZE) {
goto error;
}
pos -= LZMA_STREAM_HEADER_SIZE;
lzma_vli stream_padding = 0;
// Locate the Stream Footer. There may be Stream Padding which
// we must skip when reading backwards.
while (true) {
if (pos < LZMA_STREAM_HEADER_SIZE) {
goto error;
}
if (io_pread(src_fd, &buf,
LZMA_STREAM_HEADER_SIZE, pos))
goto error;
// Stream Padding is always a multiple of four bytes.
int i = 2;
if (buf.u32[i] != 0)
break;
// To avoid calling io_pread() for every four bytes
// of Stream Padding, take advantage that we read
// 12 bytes (LZMA_STREAM_HEADER_SIZE) already and
// check them too before calling io_pread() again.
do {
stream_padding += 4;
pos -= 4;
--i;
} while (i >= 0 && buf.u32[i] == 0);
}
// Decode the Stream Footer.
ret = lzma_stream_footer_decode(&footer_flags, buf.u8);
if (ret != LZMA_OK) {
goto error;
}
// Check that the Stream Footer doesn't specify something
// that we don't support. This can only happen if the xz
// version is older than liblzma and liblzma supports
// something new.
//
// It is enough to check Stream Footer. Stream Header must
// match when it is compared against Stream Footer with
// lzma_stream_flags_compare().
if (footer_flags.version != 0) {
goto error;
}
// Check that the size of the Index field looks sane.
lzma_vli index_size = footer_flags.backward_size;
if ((lzma_vli)(pos) < index_size + LZMA_STREAM_HEADER_SIZE) {
goto error;
}
// Set pos to the beginning of the Index.
pos -= index_size;
// Decode the Index.
ret = lzma_index_decoder(&strm, &this_index, UINT64_MAX);
if (ret != LZMA_OK) {
goto error;
}
do {
// Don't give the decoder more input than the
// Index size.
strm.avail_in = my_min(IO_BUFFER_SIZE, index_size);
if (io_pread(src_fd, &buf, strm.avail_in, pos))
goto error;
pos += strm.avail_in;
index_size -= strm.avail_in;
strm.next_in = buf.u8;
ret = lzma_code(&strm, LZMA_RUN);
} while (ret == LZMA_OK);
// If the decoding seems to be successful, check also that
// the Index decoder consumed as much input as indicated
// by the Backward Size field.
if (ret == LZMA_STREAM_END)
if (index_size != 0 || strm.avail_in != 0)
ret = LZMA_DATA_ERROR;
if (ret != LZMA_STREAM_END) {
// LZMA_BUFFER_ERROR means that the Index decoder
// would have liked more input than what the Index
// size should be according to Stream Footer.
// The message for LZMA_DATA_ERROR makes more
// sense in that case.
if (ret == LZMA_BUF_ERROR)
ret = LZMA_DATA_ERROR;
goto error;
}
// Decode the Stream Header and check that its Stream Flags
// match the Stream Footer.
pos -= footer_flags.backward_size + LZMA_STREAM_HEADER_SIZE;
if ((lzma_vli)(pos) < lzma_index_total_size(this_index)) {
goto error;
}
pos -= lzma_index_total_size(this_index);
if (io_pread(src_fd, &buf, LZMA_STREAM_HEADER_SIZE, pos))
goto error;
ret = lzma_stream_header_decode(&header_flags, buf.u8);
if (ret != LZMA_OK) {
goto error;
}
ret = lzma_stream_flags_compare(&header_flags, &footer_flags);
if (ret != LZMA_OK) {
goto error;
}
// Store the decoded Stream Flags into this_index. This is
// needed so that we can print which Check is used in each
// Stream.
ret = lzma_index_stream_flags(this_index, &footer_flags);
if (ret != LZMA_OK)
goto error;
// Store also the size of the Stream Padding field. It is
// needed to show the offsets of the Streams correctly.
ret = lzma_index_stream_padding(this_index, stream_padding);
if (ret != LZMA_OK)
goto error;
if (combined_index != NULL) {
// Append the earlier decoded Indexes
// after this_index.
ret = lzma_index_cat(
this_index, combined_index, NULL);
if (ret != LZMA_OK) {
goto error;
}
}
combined_index = this_index;
this_index = NULL;
xfi->stream_padding += stream_padding;
} while (pos > 0);
lzma_end(&strm);
// All OK. Make combined_index available to the caller.
xfi->idx = combined_index;
return false;
error:
// Something went wrong, free the allocated memory.
lzma_end(&strm);
lzma_index_end(combined_index, NULL);
lzma_index_end(this_index, NULL);
return true;
}
/* adapted from official xz source: xz/src/xz/list.c */
static lzma_index* read_index(File file, uint64_t memlimit, std::string &error) {
union {
unsigned char buf[4096];
unsigned char u8[LZMA_STREAM_HEADER_SIZE];
uint32_t u32[LZMA_STREAM_HEADER_SIZE/4];
} buf;
lzma_stream strm = LZMA_STREAM_INIT;
lzma_index *cur_index = nullptr;
lzma_index *col_index = nullptr;
FileReaderState *filestate = nullptr;
lzma_stream_flags header_flags;
lzma_stream_flags footer_flags;
lzma_ret ret;
// Current position in the file. We parse the file backwards so
// initialize it to point to the end of the file.
int64_t pos = file->filesize();;
// Each loop iteration decodes one Index.
do {
lzma_vli stream_padding, index_size;
uint64_t memused;
// Check that there is enough data left to contain at least
// the Stream Header and Stream Footer. This check cannot
// fail in the first pass of this loop.
if (pos < 2 * LZMA_STREAM_HEADER_SIZE) {
error.assign("file too small for xz archive");
goto failed;
}
pos -= LZMA_STREAM_HEADER_SIZE;
stream_padding = 0;
// Locate the Stream Footer. There may be Stream Padding which
// we must skip when reading backwards.
for (;;) {
if (pos < LZMA_STREAM_HEADER_SIZE) {
error.assign("file too small for xz archive");
goto failed;
}
if (!file->readInto(filestate, pos, LZMA_STREAM_HEADER_SIZE, buf.buf, error)) goto failed;
/* padding must be a multiple of 4 */
if (buf.u32[2] != 0) break;
pos -= 4; stream_padding += 4;
if (buf.u32[1] != 0) break;
pos -= 4; stream_padding += 4;
if (buf.u32[0] != 0) break;
pos -= 4; stream_padding += 4;
}
// Decode the Stream Footer.
ret = lzma_stream_footer_decode(&footer_flags, buf.u8);
if (LZMA_OK != ret) {
errnoLzmaToStr("invalid footer", ret, error);
goto failed;
}
// Check that the size of the Index field looks sane.
index_size = footer_flags.backward_size;
if ((lzma_vli)(pos) < index_size + LZMA_STREAM_HEADER_SIZE) {
error.assign("invalid index size");
goto failed;
}
// Set pos to the beginning of the Index.
pos -= index_size;
// See how much memory we can use for decoding this Index.
memused = nullptr != col_index ? lzma_index_memused(col_index) : 0;
if (memused > memlimit) {
error.assign("mem limit hit");
goto failed;
}
// Decode the Index.
ret = lzma_index_decoder(&strm, &cur_index, memlimit - memused);
if (ret != LZMA_OK) {
errnoLzmaToStr("couldn't allocate new index", ret, error);
goto failed;
}
do {
ssize_t want = (index_size < sizeof(buf.buf) ? index_size : sizeof(buf.buf));
if (want < 0) { ret = LZMA_DATA_ERROR; break; }
if (!file->readInto(filestate, pos, want, buf.buf, error)) goto failed;
strm.avail_in = want;
strm.next_in = buf.buf;
pos += want;
index_size -= want;
ret = lzma_code(&strm, LZMA_RUN);
} while (ret == LZMA_OK);
// If the decoding seems to be successful, check also that
// the Index decoder consumed as much input as indicated
// by the Backward Size field.
if (ret == LZMA_STREAM_END)
if (index_size != 0 || strm.avail_in != 0)
ret = LZMA_DATA_ERROR;
if (ret != LZMA_STREAM_END) {
// LZMA_BUFFER_ERROR means that the Index decoder
// would have liked more input than what the Index
// size should be according to Stream Footer.
// The message for LZMA_DATA_ERROR makes more
// sense in that case.
if (ret == LZMA_BUF_ERROR)
ret = LZMA_DATA_ERROR;
errnoLzmaToStr("decoding index failed", ret, error);
goto failed;
}
// Decode the Stream Header and check that its Stream Flags
// match the Stream Footer.
pos -= footer_flags.backward_size + LZMA_STREAM_HEADER_SIZE;
if ((lzma_vli)(pos) < lzma_index_total_size(cur_index)) {
error.assign("invalid archive - index large than available data");
goto failed;
}
pos -= lzma_index_total_size(cur_index);
if (!file->readInto(filestate, pos, LZMA_STREAM_HEADER_SIZE, buf.buf, error)) goto failed;
ret = lzma_stream_header_decode(&header_flags, buf.u8);
if (ret != LZMA_OK) {
errnoLzmaToStr("invalid header", ret, error);
goto failed;
}
ret = lzma_stream_flags_compare(&header_flags, &footer_flags);
if (ret != LZMA_OK) {
errnoLzmaToStr("invalid stream: footer doesn't match header", ret, error);
goto failed;
}
// Store the decoded Stream Flags into this_index. This is
// needed so that we can print which Check is used in each
// Stream.
ret = lzma_index_stream_flags(cur_index, &footer_flags);
if (ret != LZMA_OK) {
errnoLzmaToStr("decoding stream flags failed", ret, error);
goto failed;
}
// Store also the size of the Stream Padding field. It is
// needed to show the offsets of the Streams correctly.
ret = lzma_index_stream_padding(cur_index, stream_padding);
if (ret != LZMA_OK) {
errnoLzmaToStr("storing stream padding failed", ret, error);
goto failed;
}
if (nullptr != col_index) {
// Append the earlier decoded Indexes
// after this_index.
ret = lzma_index_cat(cur_index, col_index, NULL);
col_index = nullptr;
if (ret != LZMA_OK) {
errnoLzmaToStr("failed to concatenate indexes", ret, error);
goto failed;
}
}
col_index = cur_index;
cur_index = nullptr;
} while (pos > 0);
lzma_end(&strm);
file->finish(filestate);
return col_index;
failed:
lzma_end(&strm);
if (nullptr != cur_index) lzma_index_end(cur_index, NULL);
if (nullptr != col_index) lzma_index_end(col_index, NULL);
file->finish(filestate);
return nullptr;
}
XZFile::~XZFile() {
if (nullptr != m_index) {
lzma_index_end(m_index, NULL);
m_index = nullptr;
}
}
static lzma_ret
stream_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator,
const lzma_filter *filters, lzma_check check)
{
lzma_next_coder_init(&stream_encoder_init, next, allocator);
if (filters == NULL)
return LZMA_PROG_ERROR;
if (next->coder == NULL) {
next->coder = lzma_alloc(sizeof(lzma_coder), allocator);
if (next->coder == NULL)
return LZMA_MEM_ERROR;
next->code = &stream_encode;
next->end = &stream_encoder_end;
next->update = &stream_encoder_update;
next->coder->filters[0].id = LZMA_VLI_UNKNOWN;
next->coder->block_encoder = LZMA_NEXT_CODER_INIT;
next->coder->index_encoder = LZMA_NEXT_CODER_INIT;
next->coder->index = NULL;
}
// Basic initializations
next->coder->sequence = SEQ_STREAM_HEADER;
next->coder->block_options.version = 0;
next->coder->block_options.check = check;
// Initialize the Index
lzma_index_end(next->coder->index, allocator);
next->coder->index = lzma_index_init(allocator);
if (next->coder->index == NULL)
return LZMA_MEM_ERROR;
// Encode the Stream Header
lzma_stream_flags stream_flags = {
.version = 0,
.check = check,
};
return_if_error(lzma_stream_header_encode(
&stream_flags, next->coder->buffer));
next->coder->buffer_pos = 0;
next->coder->buffer_size = LZMA_STREAM_HEADER_SIZE;
// Initialize the Block encoder. This way we detect unsupported
// filter chains when initializing the Stream encoder instead of
// giving an error after Stream Header has already written out.
return stream_encoder_update(
next->coder, allocator, filters, NULL);
}
extern LZMA_API(lzma_ret)
lzma_stream_encoder(lzma_stream *strm,
const lzma_filter *filters, lzma_check check)
{
lzma_next_strm_init(stream_encoder_init, strm, filters, check);
strm->internal->supported_actions[LZMA_RUN] = true;
strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true;
strm->internal->supported_actions[LZMA_FULL_FLUSH] = true;
strm->internal->supported_actions[LZMA_FINISH] = true;
return LZMA_OK;
}
static void
test_code(lzma_index *i)
{
const size_t alloc_size = 128 * 1024;
uint8_t *buf = malloc(alloc_size);
expect(buf != NULL);
// Encode
lzma_stream strm = LZMA_STREAM_INIT;
expect(lzma_index_encoder(&strm, i) == LZMA_OK);
const lzma_vli index_size = lzma_index_size(i);
succeed(coder_loop(&strm, NULL, 0, buf, index_size,
LZMA_STREAM_END, LZMA_RUN));
// Decode
lzma_index *d;
expect(lzma_index_decoder(&strm, &d, MEMLIMIT) == LZMA_OK);
expect(d == NULL);
succeed(decoder_loop(&strm, buf, index_size));
expect(is_equal(i, d));
lzma_index_end(d, NULL);
lzma_end(&strm);
// Decode with hashing
lzma_index_hash *h = lzma_index_hash_init(NULL, NULL);
expect(h != NULL);
lzma_index_iter r;
lzma_index_iter_init(&r, i);
while (!lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK))
expect(lzma_index_hash_append(h, r.block.unpadded_size,
r.block.uncompressed_size) == LZMA_OK);
size_t pos = 0;
while (pos < index_size - 1)
expect(lzma_index_hash_decode(h, buf, &pos, pos + 1)
== LZMA_OK);
expect(lzma_index_hash_decode(h, buf, &pos, pos + 1)
== LZMA_STREAM_END);
lzma_index_hash_end(h, NULL);
// Encode buffer
size_t buf_pos = 1;
expect(lzma_index_buffer_encode(i, buf, &buf_pos, index_size)
== LZMA_BUF_ERROR);
expect(buf_pos == 1);
succeed(lzma_index_buffer_encode(i, buf, &buf_pos, index_size + 1));
expect(buf_pos == index_size + 1);
// Decode buffer
buf_pos = 1;
uint64_t memlimit = MEMLIMIT;
expect(lzma_index_buffer_decode(&d, &memlimit, NULL, buf, &buf_pos,
index_size) == LZMA_DATA_ERROR);
expect(buf_pos == 1);
expect(d == NULL);
succeed(lzma_index_buffer_decode(&d, &memlimit, NULL, buf, &buf_pos,
index_size + 1));
expect(buf_pos == index_size + 1);
expect(is_equal(i, d));
lzma_index_end(d, NULL);
free(buf);
}
void* gar_map(char* filename, size_t* out_length)
{
uint8_t* in_data;
size_t in_length;
if (!(in_data = platform_map(filename, &in_length)))
return NULL;
switch (gar_identify(in_data))
{
case gar_uncompressed:
*out_length = in_length;
return in_data;
case gar_xz_compressed:
{
uint8_t* out_data;
size_t in_pos = 0;
size_t out_pos = 0;
uint8_t* ptr;
uint64_t memlimit = 134217728;
lzma_stream_flags flags;
lzma_index *index;
if ((ptr = in_data + in_length - 12) < in_data)
goto error;
if (lzma_stream_footer_decode(&flags, ptr) != LZMA_OK)
goto error;
if ((ptr -= flags.backward_size) < in_data)
goto error;
if (lzma_index_buffer_decode(&index, &memlimit, NULL, ptr, &in_pos, in_length - (ptr - in_data)) != LZMA_OK)
goto error;
memlimit = 134217728;
*out_length = lzma_index_uncompressed_size(index);
if (!(out_data = platform_virtualalloc(*out_length, mem_read | mem_write)))
{
lzma_index_end(index, NULL);
goto error;
}
in_pos = 0;
if (lzma_stream_buffer_decode(&memlimit, 0, NULL, in_data, &in_pos, in_length, out_data, &out_pos, *out_length) == LZMA_OK)
{
lzma_index_end(index, NULL);
platform_unmap(in_data, in_length);
if (gar_identify(out_data) != gar_uncompressed)
{
platform_virtualfree(out_data, *out_length);
return NULL;
}
out_data[0] = 'a';
out_data[1] = 'n';
out_data[2] = 'o';
out_data[3] = 'n';
platform_virtualprotect(out_data, *out_length, mem_read);
return out_data;
}
platform_virtualfree(out_data, *out_length);
error:
platform_unmap(in_data, in_length);
return NULL;
}
default:
platform_unmap(in_data, in_length);
return NULL;
}
}
static void
test_locate(void)
{
lzma_index *i = lzma_index_init(NULL);
expect(i != NULL);
lzma_index_iter r;
lzma_index_iter_init(&r, i);
// Cannot locate anything from an empty Index.
expect(lzma_index_iter_locate(&r, 0));
expect(lzma_index_iter_locate(&r, 555));
// One empty Record: nothing is found since there's no uncompressed
// data.
expect(lzma_index_append(i, NULL, 16, 0) == LZMA_OK);
expect(lzma_index_iter_locate(&r, 0));
// Non-empty Record and we can find something.
expect(lzma_index_append(i, NULL, 32, 5) == LZMA_OK);
expect(!lzma_index_iter_locate(&r, 0));
expect(r.block.total_size == 32);
expect(r.block.uncompressed_size == 5);
expect(r.block.compressed_file_offset
== LZMA_STREAM_HEADER_SIZE + 16);
expect(r.block.uncompressed_file_offset == 0);
// Still cannot find anything past the end.
expect(lzma_index_iter_locate(&r, 5));
// Add the third Record.
expect(lzma_index_append(i, NULL, 40, 11) == LZMA_OK);
expect(!lzma_index_iter_locate(&r, 0));
expect(r.block.total_size == 32);
expect(r.block.uncompressed_size == 5);
expect(r.block.compressed_file_offset
== LZMA_STREAM_HEADER_SIZE + 16);
expect(r.block.uncompressed_file_offset == 0);
expect(!lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK));
expect(r.block.total_size == 40);
expect(r.block.uncompressed_size == 11);
expect(r.block.compressed_file_offset
== LZMA_STREAM_HEADER_SIZE + 16 + 32);
expect(r.block.uncompressed_file_offset == 5);
expect(!lzma_index_iter_locate(&r, 2));
expect(r.block.total_size == 32);
expect(r.block.uncompressed_size == 5);
expect(r.block.compressed_file_offset
== LZMA_STREAM_HEADER_SIZE + 16);
expect(r.block.uncompressed_file_offset == 0);
expect(!lzma_index_iter_locate(&r, 5));
expect(r.block.total_size == 40);
expect(r.block.uncompressed_size == 11);
expect(r.block.compressed_file_offset
== LZMA_STREAM_HEADER_SIZE + 16 + 32);
expect(r.block.uncompressed_file_offset == 5);
expect(!lzma_index_iter_locate(&r, 5 + 11 - 1));
expect(r.block.total_size == 40);
expect(r.block.uncompressed_size == 11);
expect(r.block.compressed_file_offset
== LZMA_STREAM_HEADER_SIZE + 16 + 32);
expect(r.block.uncompressed_file_offset == 5);
expect(lzma_index_iter_locate(&r, 5 + 11));
expect(lzma_index_iter_locate(&r, 5 + 15));
// Large Index
lzma_index_end(i, NULL);
i = lzma_index_init(NULL);
expect(i != NULL);
lzma_index_iter_init(&r, i);
for (size_t n = 4; n <= 4 * 5555; n += 4)
expect(lzma_index_append(i, NULL, n + 8, n) == LZMA_OK);
expect(lzma_index_block_count(i) == 5555);
// First Record
expect(!lzma_index_iter_locate(&r, 0));
expect(r.block.total_size == 4 + 8);
expect(r.block.uncompressed_size == 4);
expect(r.block.compressed_file_offset == LZMA_STREAM_HEADER_SIZE);
expect(r.block.uncompressed_file_offset == 0);
expect(!lzma_index_iter_locate(&r, 3));
expect(r.block.total_size == 4 + 8);
expect(r.block.uncompressed_size == 4);
expect(r.block.compressed_file_offset == LZMA_STREAM_HEADER_SIZE);
expect(r.block.uncompressed_file_offset == 0);
// Second Record
expect(!lzma_index_iter_locate(&r, 4));
expect(r.block.total_size == 2 * 4 + 8);
expect(r.block.uncompressed_size == 2 * 4);
expect(r.block.compressed_file_offset
== LZMA_STREAM_HEADER_SIZE + 4 + 8);
expect(r.block.uncompressed_file_offset == 4);
// Last Record
expect(!lzma_index_iter_locate(
&r, lzma_index_uncompressed_size(i) - 1));
expect(r.block.total_size == 4 * 5555 + 8);
expect(r.block.uncompressed_size == 4 * 5555);
expect(r.block.compressed_file_offset == lzma_index_total_size(i)
+ LZMA_STREAM_HEADER_SIZE - 4 * 5555 - 8);
expect(r.block.uncompressed_file_offset
== lzma_index_uncompressed_size(i) - 4 * 5555);
// Allocation chunk boundaries. See INDEX_GROUP_SIZE in
// liblzma/common/index.c.
const size_t group_multiple = 256 * 4;
const size_t radius = 8;
const size_t start = group_multiple - radius;
lzma_vli ubase = 0;
lzma_vli tbase = 0;
size_t n;
for (n = 1; n < start; ++n) {
ubase += n * 4;
tbase += n * 4 + 8;
}
while (n < start + 2 * radius) {
expect(!lzma_index_iter_locate(&r, ubase + n * 4));
expect(r.block.compressed_file_offset == tbase + n * 4 + 8
+ LZMA_STREAM_HEADER_SIZE);
expect(r.block.uncompressed_file_offset == ubase + n * 4);
tbase += n * 4 + 8;
ubase += n * 4;
++n;
expect(r.block.total_size == n * 4 + 8);
expect(r.block.uncompressed_size == n * 4);
}
// Do it also backwards.
while (n > start) {
expect(!lzma_index_iter_locate(&r, ubase + (n - 1) * 4));
expect(r.block.total_size == n * 4 + 8);
expect(r.block.uncompressed_size == n * 4);
--n;
tbase -= n * 4 + 8;
ubase -= n * 4;
expect(r.block.compressed_file_offset == tbase + n * 4 + 8
+ LZMA_STREAM_HEADER_SIZE);
expect(r.block.uncompressed_file_offset == ubase + n * 4);
}
// Test locating in concatenated Index.
lzma_index_end(i, NULL);
i = lzma_index_init(NULL);
expect(i != NULL);
lzma_index_iter_init(&r, i);
for (n = 0; n < group_multiple; ++n)
expect(lzma_index_append(i, NULL, 8, 0) == LZMA_OK);
expect(lzma_index_append(i, NULL, 16, 1) == LZMA_OK);
expect(!lzma_index_iter_locate(&r, 0));
expect(r.block.total_size == 16);
expect(r.block.uncompressed_size == 1);
expect(r.block.compressed_file_offset
== LZMA_STREAM_HEADER_SIZE + group_multiple * 8);
expect(r.block.uncompressed_file_offset == 0);
lzma_index_end(i, NULL);
}
static void
test_cat(void)
{
lzma_index *a, *b, *c;
lzma_index_iter r;
// Empty Indexes
a = create_empty();
b = create_empty();
expect(lzma_index_cat(a, b, NULL) == LZMA_OK);
expect(lzma_index_block_count(a) == 0);
expect(lzma_index_stream_size(a) == 2 * LZMA_STREAM_HEADER_SIZE + 8);
expect(lzma_index_file_size(a)
== 2 * (2 * LZMA_STREAM_HEADER_SIZE + 8));
lzma_index_iter_init(&r, a);
expect(lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK));
b = create_empty();
expect(lzma_index_cat(a, b, NULL) == LZMA_OK);
expect(lzma_index_block_count(a) == 0);
expect(lzma_index_stream_size(a) == 2 * LZMA_STREAM_HEADER_SIZE + 8);
expect(lzma_index_file_size(a)
== 3 * (2 * LZMA_STREAM_HEADER_SIZE + 8));
b = create_empty();
c = create_empty();
expect(lzma_index_stream_padding(b, 4) == LZMA_OK);
expect(lzma_index_cat(b, c, NULL) == LZMA_OK);
expect(lzma_index_block_count(b) == 0);
expect(lzma_index_stream_size(b) == 2 * LZMA_STREAM_HEADER_SIZE + 8);
expect(lzma_index_file_size(b)
== 2 * (2 * LZMA_STREAM_HEADER_SIZE + 8) + 4);
expect(lzma_index_stream_padding(a, 8) == LZMA_OK);
expect(lzma_index_cat(a, b, NULL) == LZMA_OK);
expect(lzma_index_block_count(a) == 0);
expect(lzma_index_stream_size(a) == 2 * LZMA_STREAM_HEADER_SIZE + 8);
expect(lzma_index_file_size(a)
== 5 * (2 * LZMA_STREAM_HEADER_SIZE + 8) + 4 + 8);
expect(lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK));
lzma_index_iter_rewind(&r);
expect(lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK));
lzma_index_end(a, NULL);
// Small Indexes
a = create_small();
lzma_vli stream_size = lzma_index_stream_size(a);
lzma_index_iter_init(&r, a);
for (int i = SMALL_COUNT; i >= 0; --i)
expect(!lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK)
^ (i == 0));
b = create_small();
expect(lzma_index_stream_padding(a, 4) == LZMA_OK);
expect(lzma_index_cat(a, b, NULL) == LZMA_OK);
expect(lzma_index_file_size(a) == stream_size * 2 + 4);
expect(lzma_index_stream_size(a) > stream_size);
expect(lzma_index_stream_size(a) < stream_size * 2);
for (int i = SMALL_COUNT; i >= 0; --i)
expect(!lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK)
^ (i == 0));
lzma_index_iter_rewind(&r);
for (int i = SMALL_COUNT * 2; i >= 0; --i)
expect(!lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK)
^ (i == 0));
b = create_small();
c = create_small();
expect(lzma_index_stream_padding(b, 8) == LZMA_OK);
expect(lzma_index_cat(b, c, NULL) == LZMA_OK);
expect(lzma_index_stream_padding(a, 12) == LZMA_OK);
expect(lzma_index_cat(a, b, NULL) == LZMA_OK);
expect(lzma_index_file_size(a) == stream_size * 4 + 4 + 8 + 12);
expect(lzma_index_block_count(a) == SMALL_COUNT * 4);
for (int i = SMALL_COUNT * 2; i >= 0; --i)
expect(!lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK)
^ (i == 0));
lzma_index_iter_rewind(&r);
for (int i = SMALL_COUNT * 4; i >= 0; --i)
expect(!lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK)
^ (i == 0));
lzma_index_end(a, NULL);
// Mix of empty and small
a = create_empty();
b = create_small();
expect(lzma_index_stream_padding(a, 4) == LZMA_OK);
expect(lzma_index_cat(a, b, NULL) == LZMA_OK);
lzma_index_iter_init(&r, a);
for (int i = SMALL_COUNT; i >= 0; --i)
expect(!lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK)
^ (i == 0));
lzma_index_end(a, NULL);
// Big Indexes
a = create_big();
stream_size = lzma_index_stream_size(a);
b = create_big();
expect(lzma_index_stream_padding(a, 4) == LZMA_OK);
expect(lzma_index_cat(a, b, NULL) == LZMA_OK);
expect(lzma_index_file_size(a) == stream_size * 2 + 4);
expect(lzma_index_stream_size(a) > stream_size);
expect(lzma_index_stream_size(a) < stream_size * 2);
b = create_big();
c = create_big();
expect(lzma_index_stream_padding(b, 8) == LZMA_OK);
expect(lzma_index_cat(b, c, NULL) == LZMA_OK);
expect(lzma_index_stream_padding(a, 12) == LZMA_OK);
expect(lzma_index_cat(a, b, NULL) == LZMA_OK);
expect(lzma_index_file_size(a) == stream_size * 4 + 4 + 8 + 12);
lzma_index_iter_init(&r, a);
for (int i = BIG_COUNT * 4; i >= 0; --i)
expect(!lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK)
^ (i == 0));
lzma_index_end(a, NULL);
}
