static lzma_index * create_big(void) { lzma_index *i = lzma_index_init(NULL); expect(i != NULL); lzma_vli total_size = 0; lzma_vli uncompressed_size = 0; // Add pseudo-random sizes (but always the same size values). uint32_t n = 11; for (size_t j = 0; j < BIG_COUNT; ++j) { n = 7019 * n + 7607; const uint32_t t = n * 3011; expect(lzma_index_append(i, NULL, t, n) == LZMA_OK); total_size += (t + 3) & ~LZMA_VLI_C(3); uncompressed_size += n; } expect(lzma_index_block_count(i) == BIG_COUNT); expect(lzma_index_total_size(i) == total_size); expect(lzma_index_uncompressed_size(i) == uncompressed_size); expect(lzma_index_total_size(i) + lzma_index_size(i) + 2 * LZMA_STREAM_HEADER_SIZE == lzma_index_stream_size(i)); return i; }
static int lzma_stat (struct bfd *abfd, void *stream, struct stat *sb) { struct lzma_stream *lstream = stream; sb->st_size = lzma_index_uncompressed_size (lstream->index); return 0; }
/* * Small wrapper to extract total length of a file */ off_t unxz_len(int fd) { xz_file_info xfi = XZ_FILE_INFO_INIT; if (!parse_indexes(&xfi, fd)) { off_t res = lzma_index_uncompressed_size(xfi.idx); lzma_index_end(xfi.idx, NULL); return res; } return 0; }
static bool print_info_basic(const xz_file_info *xfi, file_pair *pair) { static bool headings_displayed = false; if (!headings_displayed) { headings_displayed = true; // TRANSLATORS: These are column headings. From Strms (Streams) // to Ratio, the columns are right aligned. Check and Filename // are left aligned. If you need longer words, it's OK to // use two lines here. Test with "xz -l foo.xz". puts(_("Strms Blocks Compressed Uncompressed Ratio " "Check Filename")); } char checks[CHECKS_STR_SIZE]; get_check_names(checks, lzma_index_checks(xfi->idx), false); const char *cols[7] = { uint64_to_str(lzma_index_stream_count(xfi->idx), 0), uint64_to_str(lzma_index_block_count(xfi->idx), 1), uint64_to_nicestr(lzma_index_file_size(xfi->idx), NICESTR_B, NICESTR_TIB, false, 2), uint64_to_nicestr(lzma_index_uncompressed_size(xfi->idx), NICESTR_B, NICESTR_TIB, false, 3), get_ratio(lzma_index_file_size(xfi->idx), lzma_index_uncompressed_size(xfi->idx)), checks, pair->src_name, }; printf("%*s %*s %*s %*s %*s %-*s %s\n", tuklib_mbstr_fw(cols[0], 5), cols[0], tuklib_mbstr_fw(cols[1], 7), cols[1], tuklib_mbstr_fw(cols[2], 11), cols[2], tuklib_mbstr_fw(cols[3], 11), cols[3], tuklib_mbstr_fw(cols[4], 5), cols[4], tuklib_mbstr_fw(cols[5], 7), cols[5], cols[6]); return false; }
static void update_totals(const xz_file_info *xfi) { // TODO: Integer overflow checks ++totals.files; totals.streams += lzma_index_stream_count(xfi->idx); totals.blocks += lzma_index_block_count(xfi->idx); totals.compressed_size += lzma_index_file_size(xfi->idx); totals.uncompressed_size += lzma_index_uncompressed_size(xfi->idx); totals.stream_padding += xfi->stream_padding; totals.checks |= lzma_index_checks(xfi->idx); if (totals.memusage_max < xfi->memusage_max) totals.memusage_max = xfi->memusage_max; totals.all_have_sizes &= xfi->all_have_sizes; return; }
static void test_read(lzma_index *i) { lzma_index_iter r; lzma_index_iter_init(&r, i); // Try twice so we see that rewinding works. for (size_t j = 0; j < 2; ++j) { lzma_vli total_size = 0; lzma_vli uncompressed_size = 0; lzma_vli stream_offset = LZMA_STREAM_HEADER_SIZE; lzma_vli uncompressed_offset = 0; uint32_t count = 0; while (!lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK)) { ++count; total_size += r.block.total_size; uncompressed_size += r.block.uncompressed_size; expect(r.block.compressed_file_offset == stream_offset); expect(r.block.uncompressed_file_offset == uncompressed_offset); stream_offset += r.block.total_size; uncompressed_offset += r.block.uncompressed_size; } expect(lzma_index_total_size(i) == total_size); expect(lzma_index_uncompressed_size(i) == uncompressed_size); expect(lzma_index_block_count(i) == count); lzma_index_iter_rewind(&r); } }
lzma_vli find_file_index(void **bdatap) { if (!gIndex) decode_index(); // find the last block lzma_index_iter iter; lzma_index_iter_init(&iter, gIndex); lzma_vli loc = lzma_index_uncompressed_size(gIndex) - 1; if (lzma_index_iter_locate(&iter, loc)) die("Can't locate file index block"); void *bdata = decode_block_start(iter.block.compressed_file_offset); gFileIndexBuf = malloc(gFIBSize); gStream.avail_out = gFIBSize; gStream.avail_in = 0; // Check if this is really an index read_file_index_data(); lzma_vli ret = iter.block.compressed_file_offset; if (xle64dec(gFileIndexBuf + gFIBPos) != PIXZ_INDEX_MAGIC) ret = 0; gFIBPos += sizeof(uint64_t); if (bdatap && ret) { *bdatap = bdata; } else { // Just looking, don't keep things around if (bdatap) *bdatap = NULL; free(bdata); free(gFileIndexBuf); gLastFile = gFileIndex = NULL; lzma_end(&gStream); } return ret; }
int64_t XZFile::filesize() { return (nullptr != m_index) ? lzma_index_uncompressed_size(m_index) : 0; }
static bool print_info_robot(xz_file_info *xfi, file_pair *pair) { char checks[CHECKS_STR_SIZE]; get_check_names(checks, lzma_index_checks(xfi->idx), false); printf("name\t%s\n", pair->src_name); printf("file\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%s\t%s\t%" PRIu64 "\n", lzma_index_stream_count(xfi->idx), lzma_index_block_count(xfi->idx), lzma_index_file_size(xfi->idx), lzma_index_uncompressed_size(xfi->idx), get_ratio(lzma_index_file_size(xfi->idx), lzma_index_uncompressed_size(xfi->idx)), checks, xfi->stream_padding); if (message_verbosity_get() >= V_VERBOSE) { lzma_index_iter iter; lzma_index_iter_init(&iter, xfi->idx); while (!lzma_index_iter_next(&iter, LZMA_INDEX_ITER_STREAM)) printf("stream\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%s\t%s\t%" PRIu64 "\n", iter.stream.number, iter.stream.block_count, iter.stream.compressed_offset, iter.stream.uncompressed_offset, iter.stream.compressed_size, iter.stream.uncompressed_size, get_ratio(iter.stream.compressed_size, iter.stream.uncompressed_size), check_names[iter.stream.flags->check], iter.stream.padding); lzma_index_iter_rewind(&iter); block_header_info bhi; while (!lzma_index_iter_next(&iter, LZMA_INDEX_ITER_BLOCK)) { if (message_verbosity_get() >= V_DEBUG && parse_details( pair, &iter, &bhi, xfi)) return true; printf("block\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%s\t%s", iter.stream.number, iter.block.number_in_stream, iter.block.number_in_file, iter.block.compressed_file_offset, iter.block.uncompressed_file_offset, iter.block.total_size, iter.block.uncompressed_size, get_ratio(iter.block.total_size, iter.block.uncompressed_size), check_names[iter.stream.flags->check]); if (message_verbosity_get() >= V_DEBUG) printf("\t%s\t%" PRIu32 "\t%s\t%" PRIu64 "\t%" PRIu64 "\t%s", check_value, bhi.header_size, bhi.flags, bhi.compressed_size, bhi.memusage, bhi.filter_chain); putchar('\n'); } } if (message_verbosity_get() >= V_DEBUG) printf("summary\t%" PRIu64 "\t%s\t%" PRIu32 "\n", xfi->memusage_max, xfi->all_have_sizes ? "yes" : "no", xfi->min_version); return false; }
static bool print_info_adv(xz_file_info *xfi, file_pair *pair) { // Print the overall information. print_adv_helper(lzma_index_stream_count(xfi->idx), lzma_index_block_count(xfi->idx), lzma_index_file_size(xfi->idx), lzma_index_uncompressed_size(xfi->idx), lzma_index_checks(xfi->idx), xfi->stream_padding); // Size of the biggest Check. This is used to calculate the width // of the CheckVal field. The table would get insanely wide if // we always reserved space for 64-byte Check (128 chars as hex). uint32_t check_max = 0; // Print information about the Streams. // // TRANSLATORS: The second line is column headings. All except // Check are right aligned; Check is left aligned. Test with // "xz -lv foo.xz". puts(_(" Streams:\n Stream Blocks" " CompOffset UncompOffset" " CompSize UncompSize Ratio" " Check Padding")); lzma_index_iter iter; lzma_index_iter_init(&iter, xfi->idx); while (!lzma_index_iter_next(&iter, LZMA_INDEX_ITER_STREAM)) { const char *cols1[4] = { uint64_to_str(iter.stream.number, 0), uint64_to_str(iter.stream.block_count, 1), uint64_to_str(iter.stream.compressed_offset, 2), uint64_to_str(iter.stream.uncompressed_offset, 3), }; printf(" %*s %*s %*s %*s ", tuklib_mbstr_fw(cols1[0], 6), cols1[0], tuklib_mbstr_fw(cols1[1], 9), cols1[1], tuklib_mbstr_fw(cols1[2], 15), cols1[2], tuklib_mbstr_fw(cols1[3], 15), cols1[3]); const char *cols2[5] = { uint64_to_str(iter.stream.compressed_size, 0), uint64_to_str(iter.stream.uncompressed_size, 1), get_ratio(iter.stream.compressed_size, iter.stream.uncompressed_size), _(check_names[iter.stream.flags->check]), uint64_to_str(iter.stream.padding, 2), }; printf("%*s %*s %*s %-*s %*s\n", tuklib_mbstr_fw(cols2[0], 15), cols2[0], tuklib_mbstr_fw(cols2[1], 15), cols2[1], tuklib_mbstr_fw(cols2[2], 5), cols2[2], tuklib_mbstr_fw(cols2[3], 10), cols2[3], tuklib_mbstr_fw(cols2[4], 7), cols2[4]); // Update the maximum Check size. if (lzma_check_size(iter.stream.flags->check) > check_max) check_max = lzma_check_size(iter.stream.flags->check); } // Cache the verbosity level to a local variable. const bool detailed = message_verbosity_get() >= V_DEBUG; // Information collected from Block Headers block_header_info bhi; // Print information about the Blocks but only if there is // at least one Block. if (lzma_index_block_count(xfi->idx) > 0) { // Calculate the width of the CheckVal field. const int checkval_width = my_max(8, 2 * check_max); // TRANSLATORS: The second line is column headings. All // except Check are right aligned; Check is left aligned. printf(_(" Blocks:\n Stream Block" " CompOffset UncompOffset" " TotalSize UncompSize Ratio Check")); if (detailed) { // TRANSLATORS: These are additional column headings // for the most verbose listing mode. CheckVal // (Check value), Flags, and Filters are left aligned. // Header (Block Header Size), CompSize, and MemUsage // are right aligned. %*s is replaced with 0-120 // spaces to make the CheckVal column wide enough. // Test with "xz -lvv foo.xz". printf(_(" CheckVal %*s Header Flags " "CompSize MemUsage Filters"), checkval_width - 8, ""); } putchar('\n'); lzma_index_iter_init(&iter, xfi->idx); // Iterate over the Blocks. while (!lzma_index_iter_next(&iter, LZMA_INDEX_ITER_BLOCK)) { if (detailed && parse_details(pair, &iter, &bhi, xfi)) return true; const char *cols1[4] = { uint64_to_str(iter.stream.number, 0), uint64_to_str( iter.block.number_in_stream, 1), uint64_to_str( iter.block.compressed_file_offset, 2), uint64_to_str( iter.block.uncompressed_file_offset, 3) }; printf(" %*s %*s %*s %*s ", tuklib_mbstr_fw(cols1[0], 6), cols1[0], tuklib_mbstr_fw(cols1[1], 9), cols1[1], tuklib_mbstr_fw(cols1[2], 15), cols1[2], tuklib_mbstr_fw(cols1[3], 15), cols1[3]); const char *cols2[4] = { uint64_to_str(iter.block.total_size, 0), uint64_to_str(iter.block.uncompressed_size, 1), get_ratio(iter.block.total_size, iter.block.uncompressed_size), _(check_names[iter.stream.flags->check]) }; printf("%*s %*s %*s %-*s", tuklib_mbstr_fw(cols2[0], 15), cols2[0], tuklib_mbstr_fw(cols2[1], 15), cols2[1], tuklib_mbstr_fw(cols2[2], 5), cols2[2], tuklib_mbstr_fw(cols2[3], detailed ? 11 : 1), cols2[3]); if (detailed) { const lzma_vli compressed_size = iter.block.unpadded_size - bhi.header_size - lzma_check_size( iter.stream.flags->check); const char *cols3[6] = { check_value, uint64_to_str(bhi.header_size, 0), bhi.flags, uint64_to_str(compressed_size, 1), uint64_to_str( round_up_to_mib(bhi.memusage), 2), bhi.filter_chain }; // Show MiB for memory usage, because it // is the only size which is not in bytes. printf("%-*s %*s %-5s %*s %*s MiB %s", checkval_width, cols3[0], tuklib_mbstr_fw(cols3[1], 6), cols3[1], cols3[2], tuklib_mbstr_fw(cols3[3], 15), cols3[3], tuklib_mbstr_fw(cols3[4], 7), cols3[4], cols3[5]); } putchar('\n'); } } if (detailed) { printf(_(" Memory needed: %s MiB\n"), uint64_to_str( round_up_to_mib(xfi->memusage_max), 0)); printf(_(" Sizes in headers: %s\n"), xfi->all_have_sizes ? _("Yes") : _("No")); printf(_(" Minimum XZ Utils version: %s\n"), xz_ver_to_str(xfi->min_version)); } return false; }
static void test_locate(void) { lzma_index *i = lzma_index_init(NULL); expect(i != NULL); lzma_index_iter r; lzma_index_iter_init(&r, i); // Cannot locate anything from an empty Index. expect(lzma_index_iter_locate(&r, 0)); expect(lzma_index_iter_locate(&r, 555)); // One empty Record: nothing is found since there's no uncompressed // data. expect(lzma_index_append(i, NULL, 16, 0) == LZMA_OK); expect(lzma_index_iter_locate(&r, 0)); // Non-empty Record and we can find something. expect(lzma_index_append(i, NULL, 32, 5) == LZMA_OK); expect(!lzma_index_iter_locate(&r, 0)); expect(r.block.total_size == 32); expect(r.block.uncompressed_size == 5); expect(r.block.compressed_file_offset == LZMA_STREAM_HEADER_SIZE + 16); expect(r.block.uncompressed_file_offset == 0); // Still cannot find anything past the end. expect(lzma_index_iter_locate(&r, 5)); // Add the third Record. expect(lzma_index_append(i, NULL, 40, 11) == LZMA_OK); expect(!lzma_index_iter_locate(&r, 0)); expect(r.block.total_size == 32); expect(r.block.uncompressed_size == 5); expect(r.block.compressed_file_offset == LZMA_STREAM_HEADER_SIZE + 16); expect(r.block.uncompressed_file_offset == 0); expect(!lzma_index_iter_next(&r, LZMA_INDEX_ITER_BLOCK)); expect(r.block.total_size == 40); expect(r.block.uncompressed_size == 11); expect(r.block.compressed_file_offset == LZMA_STREAM_HEADER_SIZE + 16 + 32); expect(r.block.uncompressed_file_offset == 5); expect(!lzma_index_iter_locate(&r, 2)); expect(r.block.total_size == 32); expect(r.block.uncompressed_size == 5); expect(r.block.compressed_file_offset == LZMA_STREAM_HEADER_SIZE + 16); expect(r.block.uncompressed_file_offset == 0); expect(!lzma_index_iter_locate(&r, 5)); expect(r.block.total_size == 40); expect(r.block.uncompressed_size == 11); expect(r.block.compressed_file_offset == LZMA_STREAM_HEADER_SIZE + 16 + 32); expect(r.block.uncompressed_file_offset == 5); expect(!lzma_index_iter_locate(&r, 5 + 11 - 1)); expect(r.block.total_size == 40); expect(r.block.uncompressed_size == 11); expect(r.block.compressed_file_offset == LZMA_STREAM_HEADER_SIZE + 16 + 32); expect(r.block.uncompressed_file_offset == 5); expect(lzma_index_iter_locate(&r, 5 + 11)); expect(lzma_index_iter_locate(&r, 5 + 15)); // Large Index lzma_index_end(i, NULL); i = lzma_index_init(NULL); expect(i != NULL); lzma_index_iter_init(&r, i); for (size_t n = 4; n <= 4 * 5555; n += 4) expect(lzma_index_append(i, NULL, n + 8, n) == LZMA_OK); expect(lzma_index_block_count(i) == 5555); // First Record expect(!lzma_index_iter_locate(&r, 0)); expect(r.block.total_size == 4 + 8); expect(r.block.uncompressed_size == 4); expect(r.block.compressed_file_offset == LZMA_STREAM_HEADER_SIZE); expect(r.block.uncompressed_file_offset == 0); expect(!lzma_index_iter_locate(&r, 3)); expect(r.block.total_size == 4 + 8); expect(r.block.uncompressed_size == 4); expect(r.block.compressed_file_offset == LZMA_STREAM_HEADER_SIZE); expect(r.block.uncompressed_file_offset == 0); // Second Record expect(!lzma_index_iter_locate(&r, 4)); expect(r.block.total_size == 2 * 4 + 8); expect(r.block.uncompressed_size == 2 * 4); expect(r.block.compressed_file_offset == LZMA_STREAM_HEADER_SIZE + 4 + 8); expect(r.block.uncompressed_file_offset == 4); // Last Record expect(!lzma_index_iter_locate( &r, lzma_index_uncompressed_size(i) - 1)); expect(r.block.total_size == 4 * 5555 + 8); expect(r.block.uncompressed_size == 4 * 5555); expect(r.block.compressed_file_offset == lzma_index_total_size(i) + LZMA_STREAM_HEADER_SIZE - 4 * 5555 - 8); expect(r.block.uncompressed_file_offset == lzma_index_uncompressed_size(i) - 4 * 5555); // Allocation chunk boundaries. See INDEX_GROUP_SIZE in // liblzma/common/index.c. const size_t group_multiple = 256 * 4; const size_t radius = 8; const size_t start = group_multiple - radius; lzma_vli ubase = 0; lzma_vli tbase = 0; size_t n; for (n = 1; n < start; ++n) { ubase += n * 4; tbase += n * 4 + 8; } while (n < start + 2 * radius) { expect(!lzma_index_iter_locate(&r, ubase + n * 4)); expect(r.block.compressed_file_offset == tbase + n * 4 + 8 + LZMA_STREAM_HEADER_SIZE); expect(r.block.uncompressed_file_offset == ubase + n * 4); tbase += n * 4 + 8; ubase += n * 4; ++n; expect(r.block.total_size == n * 4 + 8); expect(r.block.uncompressed_size == n * 4); } // Do it also backwards. while (n > start) { expect(!lzma_index_iter_locate(&r, ubase + (n - 1) * 4)); expect(r.block.total_size == n * 4 + 8); expect(r.block.uncompressed_size == n * 4); --n; tbase -= n * 4 + 8; ubase -= n * 4; expect(r.block.compressed_file_offset == tbase + n * 4 + 8 + LZMA_STREAM_HEADER_SIZE); expect(r.block.uncompressed_file_offset == ubase + n * 4); } // Test locating in concatenated Index. lzma_index_end(i, NULL); i = lzma_index_init(NULL); expect(i != NULL); lzma_index_iter_init(&r, i); for (n = 0; n < group_multiple; ++n) expect(lzma_index_append(i, NULL, 8, 0) == LZMA_OK); expect(lzma_index_append(i, NULL, 16, 1) == LZMA_OK); expect(!lzma_index_iter_locate(&r, 0)); expect(r.block.total_size == 16); expect(r.block.uncompressed_size == 1); expect(r.block.compressed_file_offset == LZMA_STREAM_HEADER_SIZE + group_multiple * 8); expect(r.block.uncompressed_file_offset == 0); lzma_index_end(i, NULL); }
void* gar_map(char* filename, size_t* out_length) { uint8_t* in_data; size_t in_length; if (!(in_data = platform_map(filename, &in_length))) return NULL; switch (gar_identify(in_data)) { case gar_uncompressed: *out_length = in_length; return in_data; case gar_xz_compressed: { uint8_t* out_data; size_t in_pos = 0; size_t out_pos = 0; uint8_t* ptr; uint64_t memlimit = 134217728; lzma_stream_flags flags; lzma_index *index; if ((ptr = in_data + in_length - 12) < in_data) goto error; if (lzma_stream_footer_decode(&flags, ptr) != LZMA_OK) goto error; if ((ptr -= flags.backward_size) < in_data) goto error; if (lzma_index_buffer_decode(&index, &memlimit, NULL, ptr, &in_pos, in_length - (ptr - in_data)) != LZMA_OK) goto error; memlimit = 134217728; *out_length = lzma_index_uncompressed_size(index); if (!(out_data = platform_virtualalloc(*out_length, mem_read | mem_write))) { lzma_index_end(index, NULL); goto error; } in_pos = 0; if (lzma_stream_buffer_decode(&memlimit, 0, NULL, in_data, &in_pos, in_length, out_data, &out_pos, *out_length) == LZMA_OK) { lzma_index_end(index, NULL); platform_unmap(in_data, in_length); if (gar_identify(out_data) != gar_uncompressed) { platform_virtualfree(out_data, *out_length); return NULL; } out_data[0] = 'a'; out_data[1] = 'n'; out_data[2] = 'o'; out_data[3] = 'n'; platform_virtualprotect(out_data, *out_length, mem_read); return out_data; } platform_virtualfree(out_data, *out_length); error: platform_unmap(in_data, in_length); return NULL; } default: platform_unmap(in_data, in_length); return NULL; } }