void *decode_block_start(off_t block_seek) {
if (fseeko(gInFile, block_seek, SEEK_SET) == -1)
die("Error seeking to block");
// Some memory in which to keep the discovered filters safe
block_wrapper_t *bw = malloc(sizeof(block_wrapper_t));
bw->block = (lzma_block){ .check = gCheck, .filters = bw->filters,
.version = 0 };
int b = fgetc(gInFile);
if (b == EOF || b == 0)
die("Error reading block size");
bw->block.header_size = lzma_block_header_size_decode(b);
uint8_t hdrbuf[bw->block.header_size];
hdrbuf[0] = (uint8_t)b;
if (fread(hdrbuf + 1, bw->block.header_size - 1, 1, gInFile) != 1)
die("Error reading block header");
if (lzma_block_header_decode(&bw->block, NULL, hdrbuf) != LZMA_OK)
die("Error decoding file index block header");
if (lzma_block_decoder(&gStream, &bw->block) != LZMA_OK)
die("Error initializing file index stream");
return bw;
}
bool is_multi_header(const char *name) {
size_t i = strlen(name);
while (i != 0 && name[i - 1] != '/')
--i;
return strncmp(name + i, "._", 2) == 0;
}
bool loadBlock(std::string &error) {
clearFilters();
position = -1;
//LOG_VERBOSE("seeking to offset %i", (int) iter->block.compressed_file_offset);
reader.seek(iter.block.compressed_file_offset);
strm.avail_in = 0; /* make sure we read new data after lseek */
if (!fill_input_buffer(error)) return false;
if (0 == strm.avail_in) {
error.assign("Unexpected end of file while trying to read block header");
return false;
}
block.version = 0;
block.check = iter.stream.flags->check;
block.header_size = lzma_block_header_size_decode(strm.next_in[0]);
if (block.header_size > strm.avail_in) {
error.assign("Unexpected end of file while trying to read block header");
return false;
}
// Decode the Block Header.
lzma_ret ret = lzma_block_header_decode(&block, NULL, strm.next_in);
if (LZMA_OK != ret) {
errnoLzmaToStr("decoding block header failed", ret, error);
return false;
}
ret = lzma_block_compressed_size(&block, iter.block.unpadded_size);
if (LZMA_OK != ret) {
errnoLzmaToStr("decoding block header failed, invalid compressed size", ret, error);
return false;
}
strm.next_in += block.header_size;
strm.avail_in -= block.header_size;
lzma_end(&strm);
ret = lzma_block_decoder(&strm, &block);
if (LZMA_OK != ret) {
errnoLzmaToStr("couldn't initialize block decoder", ret, error);
return false;
}
position = iter.block.uncompressed_file_offset;
return true;
}
static file_ptr
lzma_pread (struct bfd *nbfd, void *stream, void *buf, file_ptr nbytes,
file_ptr offset)
{
struct lzma_stream *lstream = stream;
bfd_size_type chunk_size;
lzma_index_iter iter;
gdb_byte *compressed, *uncompressed;
file_ptr block_offset;
lzma_filter filters[LZMA_FILTERS_MAX + 1];
lzma_block block;
size_t compressed_pos, uncompressed_pos;
file_ptr res;
res = 0;
while (nbytes > 0)
{
if (lstream->data == NULL
|| lstream->data_start > offset || offset >= lstream->data_end)
{
asection *section = lstream->section;
lzma_index_iter_init (&iter, lstream->index);
if (lzma_index_iter_locate (&iter, offset))
break;
compressed = xmalloc (iter.block.total_size);
block_offset = section->filepos + iter.block.compressed_file_offset;
if (bfd_seek (section->owner, block_offset, SEEK_SET) != 0
|| bfd_bread (compressed, iter.block.total_size, section->owner)
!= iter.block.total_size)
{
xfree (compressed);
break;
}
uncompressed = xmalloc (iter.block.uncompressed_size);
memset (&block, 0, sizeof (block));
block.filters = filters;
block.header_size = lzma_block_header_size_decode (compressed[0]);
if (lzma_block_header_decode (&block, &gdb_lzma_allocator, compressed)
!= LZMA_OK)
{
xfree (compressed);
xfree (uncompressed);
break;
}
compressed_pos = block.header_size;
uncompressed_pos = 0;
if (lzma_block_buffer_decode (&block, &gdb_lzma_allocator,
compressed, &compressed_pos,
iter.block.total_size,
uncompressed, &uncompressed_pos,
iter.block.uncompressed_size)
!= LZMA_OK)
{
xfree (compressed);
xfree (uncompressed);
break;
}
xfree (compressed);
xfree (lstream->data);
lstream->data = uncompressed;
lstream->data_start = iter.block.uncompressed_file_offset;
lstream->data_end = (iter.block.uncompressed_file_offset
+ iter.block.uncompressed_size);
}
chunk_size = min (nbytes, lstream->data_end - offset);
memcpy (buf, lstream->data + offset - lstream->data_start, chunk_size);
buf = (gdb_byte *) buf + chunk_size;
offset += chunk_size;
nbytes -= chunk_size;
res += chunk_size;
}
return res;
}
static void decode_thread(size_t thnum) {
lzma_stream stream = LZMA_STREAM_INIT;
lzma_filter filters[LZMA_FILTERS_MAX + 1];
lzma_block block = { .filters = filters, .check = gCheck, .version = 0 };
pipeline_item_t *pi;
io_block_t *ib;
while (PIPELINE_STOP != queue_pop(gPipelineSplitQ, (void**)&pi)) {
ib = (io_block_t*)(pi->data);
block.header_size = lzma_block_header_size_decode(*(ib->input));
if (lzma_block_header_decode(&block, NULL, ib->input) != LZMA_OK)
die("Error decoding block header");
if (lzma_block_decoder(&stream, &block) != LZMA_OK)
die("Error initializing block decode");
stream.avail_in = ib->insize - block.header_size;
stream.next_in = ib->input + block.header_size;
stream.avail_out = gBlockOutSize;
stream.next_out = ib->output;
lzma_ret err = LZMA_OK;
while (err != LZMA_STREAM_END) {
if (err != LZMA_OK)
die("Error decoding block");
err = lzma_code(&stream, LZMA_FINISH);
}
ib->outsize = stream.next_out - ib->output;
queue_push(gPipelineMergeQ, PIPELINE_ITEM, pi);
}
lzma_end(&stream);
}
#pragma mark ARCHIVE
static int tar_ok(struct archive *ar, void *ref) {
return ARCHIVE_OK;
}
static bool tar_next_block(void) {
if (gArItem && !gArNextItem && gArWanted) {
io_block_t *ib = (io_block_t*)(gArItem->data);
if (gArWanted->start < ib->uoffset + ib->outsize)
return true; // No need
}
if (gArLastItem)
queue_push(gPipelineStartQ, PIPELINE_ITEM, gArLastItem);
gArLastItem = gArItem;
gArItem = pipeline_merged();
gArNextItem = false;
return gArItem;
}
static void tar_write_last(void) {
if (gArItem) {
io_block_t *ib = (io_block_t*)(gArItem->data);
fwrite(ib->output + gArLastOffset, gArLastSize, 1, gOutFile);
gArLastSize = 0;
}
}
static ssize_t tar_read(struct archive *ar, void *ref, const void **bufp) {
// If we got here, the last bit of archive is ok to write
tar_write_last();
// Write the first wanted file
if (!tar_next_block())
return 0;
off_t off;
size_t size;
io_block_t *ib = (io_block_t*)(gArItem->data);
if (gWantedFiles) {
debug("tar want: %s", gArWanted->name);
off = gArWanted->start - ib->uoffset;
size = gArWanted->size;
if (off < 0) {
size += off;
off = 0;
}
if (off + size > ib->outsize) {
size = ib->outsize - off;
gArNextItem = true; // force the end of this block
} else {
gArWanted = gArWanted->next;
}
} else {
off = 0;
size = ib->outsize;
}
debug("tar off = %llu, size = %zu", (unsigned long long)off, size);
gArLastOffset = off;
gArLastSize = size;
if (bufp)
*bufp = ib->output + off;
return size;
}
/// \brief Parse the Block Header
///
/// The result is stored into *bhi. The caller takes care of initializing it.
///
/// \return False on success, true on error.
static bool
parse_block_header(file_pair *pair, const lzma_index_iter *iter,
block_header_info *bhi, xz_file_info *xfi)
{
#if IO_BUFFER_SIZE < LZMA_BLOCK_HEADER_SIZE_MAX
# error IO_BUFFER_SIZE < LZMA_BLOCK_HEADER_SIZE_MAX
#endif
// Get the whole Block Header with one read, but don't read past
// the end of the Block (or even its Check field).
const uint32_t size = my_min(iter->block.total_size
- lzma_check_size(iter->stream.flags->check),
LZMA_BLOCK_HEADER_SIZE_MAX);
io_buf buf;
if (io_pread(pair, &buf, size, iter->block.compressed_file_offset))
return true;
// Zero would mean Index Indicator and thus not a valid Block.
if (buf.u8[0] == 0)
goto data_error;
// Initialize the block structure and decode Block Header Size.
lzma_filter filters[LZMA_FILTERS_MAX + 1];
lzma_block block;
block.version = 0;
block.check = iter->stream.flags->check;
block.filters = filters;
block.header_size = lzma_block_header_size_decode(buf.u8[0]);
if (block.header_size > size)
goto data_error;
// Decode the Block Header.
switch (lzma_block_header_decode(&block, NULL, buf.u8)) {
case LZMA_OK:
break;
case LZMA_OPTIONS_ERROR:
message_error("%s: %s", pair->src_name,
message_strm(LZMA_OPTIONS_ERROR));
return true;
case LZMA_DATA_ERROR:
goto data_error;
default:
message_bug();
}
// Check the Block Flags. These must be done before calling
// lzma_block_compressed_size(), because it overwrites
// block.compressed_size.
bhi->flags[0] = block.compressed_size != LZMA_VLI_UNKNOWN
? 'c' : '-';
bhi->flags[1] = block.uncompressed_size != LZMA_VLI_UNKNOWN
? 'u' : '-';
bhi->flags[2] = '\0';
// Collect information if all Blocks have both Compressed Size
// and Uncompressed Size fields. They can be useful e.g. for
// multi-threaded decompression so it can be useful to know it.
xfi->all_have_sizes &= block.compressed_size != LZMA_VLI_UNKNOWN
&& block.uncompressed_size != LZMA_VLI_UNKNOWN;
// Validate or set block.compressed_size.
switch (lzma_block_compressed_size(&block,
iter->block.unpadded_size)) {
case LZMA_OK:
// Validate also block.uncompressed_size if it is present.
// If it isn't present, there's no need to set it since
// we aren't going to actually decompress the Block; if
// we were decompressing, then we should set it so that
// the Block decoder could validate the Uncompressed Size
// that was stored in the Index.
if (block.uncompressed_size == LZMA_VLI_UNKNOWN
|| block.uncompressed_size
== iter->block.uncompressed_size)
break;
// If the above fails, the file is corrupt so
// LZMA_DATA_ERROR is a good error code.
// Fall through
case LZMA_DATA_ERROR:
// Free the memory allocated by lzma_block_header_decode().
for (size_t i = 0; filters[i].id != LZMA_VLI_UNKNOWN; ++i)
free(filters[i].options);
goto data_error;
default:
message_bug();
}
// Copy the known sizes.
bhi->header_size = block.header_size;
bhi->compressed_size = block.compressed_size;
// Calculate the decoder memory usage and update the maximum
// memory usage of this Block.
bhi->memusage = lzma_raw_decoder_memusage(filters);
if (xfi->memusage_max < bhi->memusage)
xfi->memusage_max = bhi->memusage;
// Determine the minimum XZ Utils version that supports this Block.
//
// Currently the only thing that 5.0.0 doesn't support is empty
// LZMA2 Block. This decoder bug was fixed in 5.0.2.
{
size_t i = 0;
while (filters[i + 1].id != LZMA_VLI_UNKNOWN)
++i;
if (filters[i].id == LZMA_FILTER_LZMA2
&& iter->block.uncompressed_size == 0
&& xfi->min_version < 50000022U)
xfi->min_version = 50000022U;
}
// Convert the filter chain to human readable form.
message_filters_to_str(bhi->filter_chain, filters, false);
// Free the memory allocated by lzma_block_header_decode().
for (size_t i = 0; filters[i].id != LZMA_VLI_UNKNOWN; ++i)
free(filters[i].options);
return false;
data_error:
// Show the error message.
message_error("%s: %s", pair->src_name,
message_strm(LZMA_DATA_ERROR));
return true;
}
static bool read_block(bool force_stream, lzma_check check, off_t uoffset) {
lzma_filter filters[LZMA_FILTERS_MAX + 1];
lzma_block block = { .filters = filters, .check = check, .version = 0 };
if (rbuf_read(1) != RBUF_FULL)
die("Error reading block header size");
if (gRbuf->input[0] == 0)
return false;
block.header_size = lzma_block_header_size_decode(gRbuf->input[0]);
if (block.header_size > LZMA_BLOCK_HEADER_SIZE_MAX)
die("Block header size too large");
if (rbuf_read(block.header_size) != RBUF_FULL)
die("Error reading block header");
if (lzma_block_header_decode(&block, NULL, gRbuf->input) != LZMA_OK)
die("Error decoding block header");
size_t comp = block.compressed_size, outsize = block.uncompressed_size;
bool sized = (comp != LZMA_VLI_UNKNOWN && outsize != LZMA_VLI_UNKNOWN);
if (force_stream || !sized || outsize > MAXSPLITSIZE) {
read_streaming(&block, sized ? BLOCK_SIZED : BLOCK_UNSIZED, uoffset);
} else {
block_capacity(gRbuf, 0, outsize);
gRbuf->outsize = outsize;
gRbuf->check = check;
gRbuf->btype = BLOCK_SIZED;
if (rbuf_read(lzma_block_total_size(&block)) != RBUF_FULL)
die("Error reading block contents");
rbuf_dispatch();
}
return true;
}
static void read_streaming(lzma_block *block, block_type sized, off_t uoffset) {
lzma_stream stream = LZMA_STREAM_INIT;
if (lzma_block_decoder(&stream, block) != LZMA_OK)
die("Error initializing streaming block decode");
rbuf_cycle(&stream, true, block->header_size);
stream.avail_out = 0;
bool first = true;
pipeline_item_t *pi = NULL;
io_block_t *ib = NULL;
lzma_ret err = LZMA_OK;
while (err != LZMA_STREAM_END) {
if (err != LZMA_OK)
die("Error decoding streaming block");
if (stream.avail_out == 0) {
if (ib) {
ib->outsize = ib->outcap;
ib->uoffset = uoffset;
uoffset += ib->outsize;
pipeline_dispatch(pi, gPipelineMergeQ);
first = false;
}
queue_pop(gPipelineStartQ, (void**)&pi);
ib = (io_block_t*)pi->data;
ib->btype = (first ? sized : BLOCK_CONTINUATION);
block_capacity(ib, 0, STREAMSIZE);
stream.next_out = ib->output;
stream.avail_out = ib->outcap;
}
if (stream.avail_in == 0 && !rbuf_cycle(&stream, false, 0))
die("Error reading streaming block");
err = lzma_code(&stream, LZMA_RUN);
}
if (ib && stream.avail_out != ib->outcap) {
ib->outsize = ib->outcap - stream.avail_out;
pipeline_dispatch(pi, gPipelineMergeQ);
}
rbuf_consume(gRbuf->insize - stream.avail_in);
lzma_end(&stream);
}
/// \brief Parse the Block Header
///
/// The result is stored into *bhi. The caller takes care of initializing it.
///
/// \return False on success, true on error.
static bool
parse_block_header(file_pair *pair, const lzma_index_iter *iter,
block_header_info *bhi, xz_file_info *xfi)
{
#if IO_BUFFER_SIZE < LZMA_BLOCK_HEADER_SIZE_MAX
# error IO_BUFFER_SIZE < LZMA_BLOCK_HEADER_SIZE_MAX
#endif
// Get the whole Block Header with one read, but don't read past
// the end of the Block (or even its Check field).
const uint32_t size = my_min(iter->block.total_size
- lzma_check_size(iter->stream.flags->check),
LZMA_BLOCK_HEADER_SIZE_MAX);
io_buf buf;
if (io_pread(pair, &buf, size, iter->block.compressed_file_offset))
return true;
// Zero would mean Index Indicator and thus not a valid Block.
if (buf.u8[0] == 0)
goto data_error;
lzma_block block;
lzma_filter filters[LZMA_FILTERS_MAX + 1];
// Initialize the pointers so that they can be passed to free().
for (size_t i = 0; i < ARRAY_SIZE(filters); ++i)
filters[i].options = NULL;
// Initialize the block structure and decode Block Header Size.
block.version = 0;
block.check = iter->stream.flags->check;
block.filters = filters;
block.header_size = lzma_block_header_size_decode(buf.u8[0]);
if (block.header_size > size)
goto data_error;
// Decode the Block Header.
switch (lzma_block_header_decode(&block, NULL, buf.u8)) {
case LZMA_OK:
break;
case LZMA_OPTIONS_ERROR:
message_error("%s: %s", pair->src_name,
message_strm(LZMA_OPTIONS_ERROR));
return true;
case LZMA_DATA_ERROR:
goto data_error;
default:
message_bug();
}
// Check the Block Flags. These must be done before calling
// lzma_block_compressed_size(), because it overwrites
// block.compressed_size.
bhi->flags[0] = block.compressed_size != LZMA_VLI_UNKNOWN
? 'c' : '-';
bhi->flags[1] = block.uncompressed_size != LZMA_VLI_UNKNOWN
? 'u' : '-';
bhi->flags[2] = '\0';
// Collect information if all Blocks have both Compressed Size
// and Uncompressed Size fields. They can be useful e.g. for
// multi-threaded decompression so it can be useful to know it.
xfi->all_have_sizes &= block.compressed_size != LZMA_VLI_UNKNOWN
&& block.uncompressed_size != LZMA_VLI_UNKNOWN;
// Validate or set block.compressed_size.
switch (lzma_block_compressed_size(&block,
iter->block.unpadded_size)) {
case LZMA_OK:
break;
case LZMA_DATA_ERROR:
goto data_error;
default:
message_bug();
}
// Copy the known sizes.
bhi->header_size = block.header_size;
bhi->compressed_size = block.compressed_size;
// Calculate the decoder memory usage and update the maximum
// memory usage of this Block.
bhi->memusage = lzma_raw_decoder_memusage(filters);
if (xfi->memusage_max < bhi->memusage)
xfi->memusage_max = bhi->memusage;
// Convert the filter chain to human readable form.
message_filters_to_str(bhi->filter_chain, filters, false);
// Free the memory allocated by lzma_block_header_decode().
for (size_t i = 0; filters[i].id != LZMA_VLI_UNKNOWN; ++i)
free(filters[i].options);
return false;
data_error:
// Show the error message.
message_error("%s: %s", pair->src_name,
message_strm(LZMA_DATA_ERROR));
// Free the memory allocated by lzma_block_header_decode().
// This is truly needed only if we get here after a succcessful
// call to lzma_block_header_decode() but it doesn't hurt to
// always do it.
for (size_t i = 0; filters[i].id != LZMA_VLI_UNKNOWN; ++i)
free(filters[i].options);
return true;
}