void decode_index(void) { if (fseek(gInFile, -LZMA_STREAM_HEADER_SIZE, SEEK_END) == -1) die("Error seeking to stream footer"); uint8_t hdrbuf[LZMA_STREAM_HEADER_SIZE]; if (fread(hdrbuf, LZMA_STREAM_HEADER_SIZE, 1, gInFile) != 1) die("Error reading stream footer"); lzma_stream_flags flags; if (lzma_stream_footer_decode(&flags, hdrbuf) != LZMA_OK) die("Error decoding stream footer"); gCheck = flags.check; size_t index_seek = -LZMA_STREAM_HEADER_SIZE - flags.backward_size; if (fseek(gInFile, index_seek, SEEK_CUR) == -1) die("Error seeking to index"); if (lzma_index_decoder(&gStream, &gIndex, MEMLIMIT) != LZMA_OK) die("Error creating index decoder"); uint8_t ibuf[CHUNKSIZE]; gStream.avail_in = 0; lzma_ret err = LZMA_OK; while (err != LZMA_STREAM_END) { if (gStream.avail_in == 0) { gStream.avail_in = fread(ibuf, 1, CHUNKSIZE, gInFile); if (ferror(gInFile)) die("Error reading index"); gStream.next_in = ibuf; } err = lzma_code(&gStream, LZMA_RUN); if (err != LZMA_OK && err != LZMA_STREAM_END) die("Error decoding index"); } }
static bool test_footer_decoder(lzma_ret expected_ret) { memcrap(&decoded_flags, sizeof(decoded_flags)); if (lzma_stream_footer_decode(&decoded_flags, buffer) != expected_ret) return true; if (expected_ret != LZMA_OK) return false; return validate(); }
static void * lzma_open (struct bfd *nbfd, void *open_closure) { asection *section = open_closure; bfd_size_type size, offset; lzma_stream_flags options; gdb_byte footer[LZMA_STREAM_HEADER_SIZE]; gdb_byte *indexdata; lzma_index *index; int ret; uint64_t memlimit = UINT64_MAX; struct lzma_stream *lstream; size_t pos; size = bfd_get_section_size (section); offset = section->filepos + size - LZMA_STREAM_HEADER_SIZE; if (size < LZMA_STREAM_HEADER_SIZE || bfd_seek (section->owner, offset, SEEK_SET) != 0 || bfd_bread (footer, LZMA_STREAM_HEADER_SIZE, section->owner) != LZMA_STREAM_HEADER_SIZE || lzma_stream_footer_decode (&options, footer) != LZMA_OK || offset < options.backward_size) { bfd_set_error (bfd_error_wrong_format); return NULL; } offset -= options.backward_size; indexdata = xmalloc (options.backward_size); index = NULL; pos = 0; if (bfd_seek (section->owner, offset, SEEK_SET) != 0 || bfd_bread (indexdata, options.backward_size, section->owner) != options.backward_size || lzma_index_buffer_decode (&index, &memlimit, &gdb_lzma_allocator, indexdata, &pos, options.backward_size) != LZMA_OK || lzma_index_size (index) != options.backward_size) { xfree (indexdata); bfd_set_error (bfd_error_wrong_format); return NULL; } xfree (indexdata); lstream = xzalloc (sizeof (struct lzma_stream)); lstream->section = section; lstream->index = index; return lstream; }
static void read_footer(void) { lzma_stream_flags stream_flags; if (rbuf_read(LZMA_STREAM_HEADER_SIZE) != RBUF_FULL) die("Error reading stream footer"); if (lzma_stream_footer_decode(&stream_flags, gRbuf->input) != LZMA_OK) die("Error decoding XZ footer"); rbuf_consume(LZMA_STREAM_HEADER_SIZE); char zeros[4] = "\0\0\0\0"; while (true) { rbuf_read_status st = rbuf_read(4); if (st == RBUF_EOF) return; if (st != RBUF_FULL) die("Footer must be multiple of four bytes"); if (memcmp(zeros, gRbuf->input, 4) != 0) return; rbuf_consume(4); } }
/// \brief Parse the Index(es) from the given .xz file /// /// \param xfi Pointer to structure where the decoded information /// is stored. /// \param pair Input file /// /// \return On success, false is returned. On error, true is returned. /// // TODO: This function is pretty big. liblzma should have a function that // takes a callback function to parse the Index(es) from a .xz file to make // it easy for applications. static bool parse_indexes(xz_file_info *xfi, int src_fd) { struct stat st; fstat(src_fd, &st); if (st.st_size <= 0) { return true; } if (st.st_size < 2 * LZMA_STREAM_HEADER_SIZE) { return true; } io_buf buf; lzma_stream_flags header_flags; lzma_stream_flags footer_flags; lzma_ret ret; // lzma_stream for the Index decoder lzma_stream strm = LZMA_STREAM_INIT; // All Indexes decoded so far lzma_index *combined_index = NULL; // The Index currently being decoded lzma_index *this_index = NULL; // Current position in the file. We parse the file backwards so // initialize it to point to the end of the file. off_t pos = st.st_size; // Each loop iteration decodes one Index. do { // Check that there is enough data left to contain at least // the Stream Header and Stream Footer. This check cannot // fail in the first pass of this loop. if (pos < 2 * LZMA_STREAM_HEADER_SIZE) { goto error; } pos -= LZMA_STREAM_HEADER_SIZE; lzma_vli stream_padding = 0; // Locate the Stream Footer. There may be Stream Padding which // we must skip when reading backwards. while (true) { if (pos < LZMA_STREAM_HEADER_SIZE) { goto error; } if (io_pread(src_fd, &buf, LZMA_STREAM_HEADER_SIZE, pos)) goto error; // Stream Padding is always a multiple of four bytes. int i = 2; if (buf.u32[i] != 0) break; // To avoid calling io_pread() for every four bytes // of Stream Padding, take advantage that we read // 12 bytes (LZMA_STREAM_HEADER_SIZE) already and // check them too before calling io_pread() again. do { stream_padding += 4; pos -= 4; --i; } while (i >= 0 && buf.u32[i] == 0); } // Decode the Stream Footer. ret = lzma_stream_footer_decode(&footer_flags, buf.u8); if (ret != LZMA_OK) { goto error; } // Check that the Stream Footer doesn't specify something // that we don't support. This can only happen if the xz // version is older than liblzma and liblzma supports // something new. // // It is enough to check Stream Footer. Stream Header must // match when it is compared against Stream Footer with // lzma_stream_flags_compare(). if (footer_flags.version != 0) { goto error; } // Check that the size of the Index field looks sane. lzma_vli index_size = footer_flags.backward_size; if ((lzma_vli)(pos) < index_size + LZMA_STREAM_HEADER_SIZE) { goto error; } // Set pos to the beginning of the Index. pos -= index_size; // Decode the Index. ret = lzma_index_decoder(&strm, &this_index, UINT64_MAX); if (ret != LZMA_OK) { goto error; } do { // Don't give the decoder more input than the // Index size. strm.avail_in = my_min(IO_BUFFER_SIZE, index_size); if (io_pread(src_fd, &buf, strm.avail_in, pos)) goto error; pos += strm.avail_in; index_size -= strm.avail_in; strm.next_in = buf.u8; ret = lzma_code(&strm, LZMA_RUN); } while (ret == LZMA_OK); // If the decoding seems to be successful, check also that // the Index decoder consumed as much input as indicated // by the Backward Size field. if (ret == LZMA_STREAM_END) if (index_size != 0 || strm.avail_in != 0) ret = LZMA_DATA_ERROR; if (ret != LZMA_STREAM_END) { // LZMA_BUFFER_ERROR means that the Index decoder // would have liked more input than what the Index // size should be according to Stream Footer. // The message for LZMA_DATA_ERROR makes more // sense in that case. if (ret == LZMA_BUF_ERROR) ret = LZMA_DATA_ERROR; goto error; } // Decode the Stream Header and check that its Stream Flags // match the Stream Footer. pos -= footer_flags.backward_size + LZMA_STREAM_HEADER_SIZE; if ((lzma_vli)(pos) < lzma_index_total_size(this_index)) { goto error; } pos -= lzma_index_total_size(this_index); if (io_pread(src_fd, &buf, LZMA_STREAM_HEADER_SIZE, pos)) goto error; ret = lzma_stream_header_decode(&header_flags, buf.u8); if (ret != LZMA_OK) { goto error; } ret = lzma_stream_flags_compare(&header_flags, &footer_flags); if (ret != LZMA_OK) { goto error; } // Store the decoded Stream Flags into this_index. This is // needed so that we can print which Check is used in each // Stream. ret = lzma_index_stream_flags(this_index, &footer_flags); if (ret != LZMA_OK) goto error; // Store also the size of the Stream Padding field. It is // needed to show the offsets of the Streams correctly. ret = lzma_index_stream_padding(this_index, stream_padding); if (ret != LZMA_OK) goto error; if (combined_index != NULL) { // Append the earlier decoded Indexes // after this_index. ret = lzma_index_cat( this_index, combined_index, NULL); if (ret != LZMA_OK) { goto error; } } combined_index = this_index; this_index = NULL; xfi->stream_padding += stream_padding; } while (pos > 0); lzma_end(&strm); // All OK. Make combined_index available to the caller. xfi->idx = combined_index; return false; error: // Something went wrong, free the allocated memory. lzma_end(&strm); lzma_index_end(combined_index, NULL); lzma_index_end(this_index, NULL); return true; }
/* adapted from official xz source: xz/src/xz/list.c */ static lzma_index* read_index(File file, uint64_t memlimit, std::string &error) { union { unsigned char buf[4096]; unsigned char u8[LZMA_STREAM_HEADER_SIZE]; uint32_t u32[LZMA_STREAM_HEADER_SIZE/4]; } buf; lzma_stream strm = LZMA_STREAM_INIT; lzma_index *cur_index = nullptr; lzma_index *col_index = nullptr; FileReaderState *filestate = nullptr; lzma_stream_flags header_flags; lzma_stream_flags footer_flags; lzma_ret ret; // Current position in the file. We parse the file backwards so // initialize it to point to the end of the file. int64_t pos = file->filesize();; // Each loop iteration decodes one Index. do { lzma_vli stream_padding, index_size; uint64_t memused; // Check that there is enough data left to contain at least // the Stream Header and Stream Footer. This check cannot // fail in the first pass of this loop. if (pos < 2 * LZMA_STREAM_HEADER_SIZE) { error.assign("file too small for xz archive"); goto failed; } pos -= LZMA_STREAM_HEADER_SIZE; stream_padding = 0; // Locate the Stream Footer. There may be Stream Padding which // we must skip when reading backwards. for (;;) { if (pos < LZMA_STREAM_HEADER_SIZE) { error.assign("file too small for xz archive"); goto failed; } if (!file->readInto(filestate, pos, LZMA_STREAM_HEADER_SIZE, buf.buf, error)) goto failed; /* padding must be a multiple of 4 */ if (buf.u32[2] != 0) break; pos -= 4; stream_padding += 4; if (buf.u32[1] != 0) break; pos -= 4; stream_padding += 4; if (buf.u32[0] != 0) break; pos -= 4; stream_padding += 4; } // Decode the Stream Footer. ret = lzma_stream_footer_decode(&footer_flags, buf.u8); if (LZMA_OK != ret) { errnoLzmaToStr("invalid footer", ret, error); goto failed; } // Check that the size of the Index field looks sane. index_size = footer_flags.backward_size; if ((lzma_vli)(pos) < index_size + LZMA_STREAM_HEADER_SIZE) { error.assign("invalid index size"); goto failed; } // Set pos to the beginning of the Index. pos -= index_size; // See how much memory we can use for decoding this Index. memused = nullptr != col_index ? lzma_index_memused(col_index) : 0; if (memused > memlimit) { error.assign("mem limit hit"); goto failed; } // Decode the Index. ret = lzma_index_decoder(&strm, &cur_index, memlimit - memused); if (ret != LZMA_OK) { errnoLzmaToStr("couldn't allocate new index", ret, error); goto failed; } do { ssize_t want = (index_size < sizeof(buf.buf) ? index_size : sizeof(buf.buf)); if (want < 0) { ret = LZMA_DATA_ERROR; break; } if (!file->readInto(filestate, pos, want, buf.buf, error)) goto failed; strm.avail_in = want; strm.next_in = buf.buf; pos += want; index_size -= want; ret = lzma_code(&strm, LZMA_RUN); } while (ret == LZMA_OK); // If the decoding seems to be successful, check also that // the Index decoder consumed as much input as indicated // by the Backward Size field. if (ret == LZMA_STREAM_END) if (index_size != 0 || strm.avail_in != 0) ret = LZMA_DATA_ERROR; if (ret != LZMA_STREAM_END) { // LZMA_BUFFER_ERROR means that the Index decoder // would have liked more input than what the Index // size should be according to Stream Footer. // The message for LZMA_DATA_ERROR makes more // sense in that case. if (ret == LZMA_BUF_ERROR) ret = LZMA_DATA_ERROR; errnoLzmaToStr("decoding index failed", ret, error); goto failed; } // Decode the Stream Header and check that its Stream Flags // match the Stream Footer. pos -= footer_flags.backward_size + LZMA_STREAM_HEADER_SIZE; if ((lzma_vli)(pos) < lzma_index_total_size(cur_index)) { error.assign("invalid archive - index large than available data"); goto failed; } pos -= lzma_index_total_size(cur_index); if (!file->readInto(filestate, pos, LZMA_STREAM_HEADER_SIZE, buf.buf, error)) goto failed; ret = lzma_stream_header_decode(&header_flags, buf.u8); if (ret != LZMA_OK) { errnoLzmaToStr("invalid header", ret, error); goto failed; } ret = lzma_stream_flags_compare(&header_flags, &footer_flags); if (ret != LZMA_OK) { errnoLzmaToStr("invalid stream: footer doesn't match header", ret, error); goto failed; } // Store the decoded Stream Flags into this_index. This is // needed so that we can print which Check is used in each // Stream. ret = lzma_index_stream_flags(cur_index, &footer_flags); if (ret != LZMA_OK) { errnoLzmaToStr("decoding stream flags failed", ret, error); goto failed; } // Store also the size of the Stream Padding field. It is // needed to show the offsets of the Streams correctly. ret = lzma_index_stream_padding(cur_index, stream_padding); if (ret != LZMA_OK) { errnoLzmaToStr("storing stream padding failed", ret, error); goto failed; } if (nullptr != col_index) { // Append the earlier decoded Indexes // after this_index. ret = lzma_index_cat(cur_index, col_index, NULL); col_index = nullptr; if (ret != LZMA_OK) { errnoLzmaToStr("failed to concatenate indexes", ret, error); goto failed; } } col_index = cur_index; cur_index = nullptr; } while (pos > 0); lzma_end(&strm); file->finish(filestate); return col_index; failed: lzma_end(&strm); if (nullptr != cur_index) lzma_index_end(cur_index, NULL); if (nullptr != col_index) lzma_index_end(col_index, NULL); file->finish(filestate); return nullptr; }
void* gar_map(char* filename, size_t* out_length) { uint8_t* in_data; size_t in_length; if (!(in_data = platform_map(filename, &in_length))) return NULL; switch (gar_identify(in_data)) { case gar_uncompressed: *out_length = in_length; return in_data; case gar_xz_compressed: { uint8_t* out_data; size_t in_pos = 0; size_t out_pos = 0; uint8_t* ptr; uint64_t memlimit = 134217728; lzma_stream_flags flags; lzma_index *index; if ((ptr = in_data + in_length - 12) < in_data) goto error; if (lzma_stream_footer_decode(&flags, ptr) != LZMA_OK) goto error; if ((ptr -= flags.backward_size) < in_data) goto error; if (lzma_index_buffer_decode(&index, &memlimit, NULL, ptr, &in_pos, in_length - (ptr - in_data)) != LZMA_OK) goto error; memlimit = 134217728; *out_length = lzma_index_uncompressed_size(index); if (!(out_data = platform_virtualalloc(*out_length, mem_read | mem_write))) { lzma_index_end(index, NULL); goto error; } in_pos = 0; if (lzma_stream_buffer_decode(&memlimit, 0, NULL, in_data, &in_pos, in_length, out_data, &out_pos, *out_length) == LZMA_OK) { lzma_index_end(index, NULL); platform_unmap(in_data, in_length); if (gar_identify(out_data) != gar_uncompressed) { platform_virtualfree(out_data, *out_length); return NULL; } out_data[0] = 'a'; out_data[1] = 'n'; out_data[2] = 'o'; out_data[3] = 'n'; platform_virtualprotect(out_data, *out_length, mem_read); return out_data; } platform_virtualfree(out_data, *out_length); error: platform_unmap(in_data, in_length); return NULL; } default: platform_unmap(in_data, in_length); return NULL; } }
/// \brief Parse the Index(es) from the given .xz file /// /// \param xfi Pointer to structure where the decoded information /// is stored. /// \param pair Input file /// /// \return On success, false is returned. On error, true is returned. /// // TODO: This function is pretty big. liblzma should have a function that // takes a callback function to parse the Index(es) from a .xz file to make // it easy for applications. static bool parse_indexes(xz_file_info *xfi, file_pair *pair) { if (pair->src_st.st_size <= 0) { message_error(_("%s: File is empty"), pair->src_name); return true; } if (pair->src_st.st_size < 2 * LZMA_STREAM_HEADER_SIZE) { message_error(_("%s: Too small to be a valid .xz file"), pair->src_name); return true; } io_buf buf; lzma_stream_flags header_flags; lzma_stream_flags footer_flags; lzma_ret ret; // lzma_stream for the Index decoder lzma_stream strm = LZMA_STREAM_INIT; // All Indexes decoded so far lzma_index *combined_index = NULL; // The Index currently being decoded lzma_index *this_index = NULL; // Current position in the file. We parse the file backwards so // initialize it to point to the end of the file. off_t pos = pair->src_st.st_size; // Each loop iteration decodes one Index. do { // Check that there is enough data left to contain at least // the Stream Header and Stream Footer. This check cannot // fail in the first pass of this loop. if (pos < 2 * LZMA_STREAM_HEADER_SIZE) { message_error("%s: %s", pair->src_name, message_strm(LZMA_DATA_ERROR)); goto error; } pos -= LZMA_STREAM_HEADER_SIZE; lzma_vli stream_padding = 0; // Locate the Stream Footer. There may be Stream Padding which // we must skip when reading backwards. while (true) { if (pos < LZMA_STREAM_HEADER_SIZE) { message_error("%s: %s", pair->src_name, message_strm( LZMA_DATA_ERROR)); goto error; } if (io_pread(pair, &buf, LZMA_STREAM_HEADER_SIZE, pos)) goto error; // Stream Padding is always a multiple of four bytes. int i = 2; if (buf.u32[i] != 0) break; // To avoid calling io_pread() for every four bytes // of Stream Padding, take advantage that we read // 12 bytes (LZMA_STREAM_HEADER_SIZE) already and // check them too before calling io_pread() again. do { stream_padding += 4; pos -= 4; --i; } while (i >= 0 && buf.u32[i] == 0); } // Decode the Stream Footer. ret = lzma_stream_footer_decode(&footer_flags, buf.u8); if (ret != LZMA_OK) { message_error("%s: %s", pair->src_name, message_strm(ret)); goto error; } // Check that the size of the Index field looks sane. lzma_vli index_size = footer_flags.backward_size; if ((lzma_vli)(pos) < index_size + LZMA_STREAM_HEADER_SIZE) { message_error("%s: %s", pair->src_name, message_strm(LZMA_DATA_ERROR)); goto error; } // Set pos to the beginning of the Index. pos -= index_size; // See how much memory we can use for decoding this Index. uint64_t memlimit = hardware_memlimit_get(MODE_LIST); uint64_t memused = 0; if (combined_index != NULL) { memused = lzma_index_memused(combined_index); if (memused > memlimit) message_bug(); memlimit -= memused; } // Decode the Index. ret = lzma_index_decoder(&strm, &this_index, memlimit); if (ret != LZMA_OK) { message_error("%s: %s", pair->src_name, message_strm(ret)); goto error; } do { // Don't give the decoder more input than the // Index size. strm.avail_in = my_min(IO_BUFFER_SIZE, index_size); if (io_pread(pair, &buf, strm.avail_in, pos)) goto error; pos += strm.avail_in; index_size -= strm.avail_in; strm.next_in = buf.u8; ret = lzma_code(&strm, LZMA_RUN); } while (ret == LZMA_OK); // If the decoding seems to be successful, check also that // the Index decoder consumed as much input as indicated // by the Backward Size field. if (ret == LZMA_STREAM_END) if (index_size != 0 || strm.avail_in != 0) ret = LZMA_DATA_ERROR; if (ret != LZMA_STREAM_END) { // LZMA_BUFFER_ERROR means that the Index decoder // would have liked more input than what the Index // size should be according to Stream Footer. // The message for LZMA_DATA_ERROR makes more // sense in that case. if (ret == LZMA_BUF_ERROR) ret = LZMA_DATA_ERROR; message_error("%s: %s", pair->src_name, message_strm(ret)); // If the error was too low memory usage limit, // show also how much memory would have been needed. if (ret == LZMA_MEMLIMIT_ERROR) { uint64_t needed = lzma_memusage(&strm); if (UINT64_MAX - needed < memused) needed = UINT64_MAX; else needed += memused; message_mem_needed(V_ERROR, needed); } goto error; } // Decode the Stream Header and check that its Stream Flags // match the Stream Footer. pos -= footer_flags.backward_size + LZMA_STREAM_HEADER_SIZE; if ((lzma_vli)(pos) < lzma_index_total_size(this_index)) { message_error("%s: %s", pair->src_name, message_strm(LZMA_DATA_ERROR)); goto error; } pos -= lzma_index_total_size(this_index); if (io_pread(pair, &buf, LZMA_STREAM_HEADER_SIZE, pos)) goto error; ret = lzma_stream_header_decode(&header_flags, buf.u8); if (ret != LZMA_OK) { message_error("%s: %s", pair->src_name, message_strm(ret)); goto error; } ret = lzma_stream_flags_compare(&header_flags, &footer_flags); if (ret != LZMA_OK) { message_error("%s: %s", pair->src_name, message_strm(ret)); goto error; } // Store the decoded Stream Flags into this_index. This is // needed so that we can print which Check is used in each // Stream. ret = lzma_index_stream_flags(this_index, &footer_flags); if (ret != LZMA_OK) message_bug(); // Store also the size of the Stream Padding field. It is // needed to show the offsets of the Streams correctly. ret = lzma_index_stream_padding(this_index, stream_padding); if (ret != LZMA_OK) message_bug(); if (combined_index != NULL) { // Append the earlier decoded Indexes // after this_index. ret = lzma_index_cat( this_index, combined_index, NULL); if (ret != LZMA_OK) { message_error("%s: %s", pair->src_name, message_strm(ret)); goto error; } } combined_index = this_index; this_index = NULL; xfi->stream_padding += stream_padding; } while (pos > 0); lzma_end(&strm); // All OK. Make combined_index available to the caller. xfi->idx = combined_index; return false; error: // Something went wrong, free the allocated memory. lzma_end(&strm); lzma_index_end(combined_index, NULL); lzma_index_end(this_index, NULL); return true; }