static void
print_adv_helper(uint64_t stream_count, uint64_t block_count,
uint64_t compressed_size, uint64_t uncompressed_size,
uint32_t checks, uint64_t stream_padding)
{
char checks_str[CHECKS_STR_SIZE];
get_check_names(checks_str, checks, true);
printf(_(" Streams: %s\n"),
uint64_to_str(stream_count, 0));
printf(_(" Blocks: %s\n"),
uint64_to_str(block_count, 0));
printf(_(" Compressed size: %s\n"),
uint64_to_nicestr(compressed_size,
NICESTR_B, NICESTR_TIB, true, 0));
printf(_(" Uncompressed size: %s\n"),
uint64_to_nicestr(uncompressed_size,
NICESTR_B, NICESTR_TIB, true, 0));
printf(_(" Ratio: %s\n"),
get_ratio(compressed_size, uncompressed_size));
printf(_(" Check: %s\n"), checks_str);
printf(_(" Stream padding: %s\n"),
uint64_to_nicestr(stream_padding,
NICESTR_B, NICESTR_TIB, true, 0));
return;
}
Status ClientImpl::zrrange(const std::string &name,
uint64_t offset, uint64_t limit,
std::vector<std::string> *ret)
{
std::string s_offset = uint64_to_str(offset);
std::string s_limit = uint64_to_str(limit);
const std::vector<std::string> *resp;
resp = this->request("zrrange", name, s_offset, s_limit);
return _read_list(resp, ret);
}
// store a block in another block store named by the block's hash.
void export_exo_inputs(void *ptr, int size, hash_t *digest) {
unsigned char hash[20];
char hexstring[42];
// compute hash of the data
__hashbits(digest, ptr, size);
int rank;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
char tmp_db_file_path[BUFLEN];
snprintf(tmp_db_file_path, BUFLEN - 1, "%s/%s/prover_%d_tmp", FOLDER_STATE, ROOT_FOLDER_BLOCK_STORES, rank);
FILE *fp = fopen(tmp_db_file_path, "wb");
fwrite(ptr, 1, size, fp);
fclose(fp);
string input_name = uint64_to_str(NUM_HASH_CHUNKS, digest->bit);
sha1::calc(input_name.c_str(), strlen(input_name.c_str()), hash);
sha1::toHexString(hash, hexstring);
hexstring[41] = '\0';
char com_mv_db_file_path[BUFLEN];
snprintf(com_mv_db_file_path, BUFLEN - 1, "mv %s %s/%s/%s", tmp_db_file_path, FOLDER_STATE, ROOT_FOLDER_BLOCK_STORES, hexstring);
if (system(com_mv_db_file_path) != 0) {
printf("ERROR executing %s\n", com_mv_db_file_path);
}
//char db_file_path[BUFLEN];
//snprintf(db_file_path, BUFLEN - 1, "%s/%s/%s", FOLDER_STATE, ROOT_FOLDER_BLOCK_STORES, hexstring);
//string db_file_path_str(db_file_path);
//HashBlockStore* _block_store = new ConfigurableBlockStore(db_file_path_str);
//__hashput(_block_store, digest, ptr, size);
//delete _block_store;
}
Status ClientImpl::rscan(const std::string &key_start, const std::string &key_end,
uint64_t limit, std::vector<std::string> *ret)
{
std::string s_limit = uint64_to_str(limit);
const std::vector<std::string> *resp;
resp = this->request("rscan", key_start, key_end, s_limit);
return _read_list(resp, ret);
}
Status ClientImpl::zrandom(const std::string &name, const std::string &key_start,
int64_t *score_start, int64_t *score_end,
uint64_t limit, std::vector<std::string> *ret)
{
std::string s_score_start = score_start? int_to_str(*score_start) : "";
std::string s_score_end = score_end? int_to_str(*score_end) : "";
std::string s_limit = uint64_to_str(limit);
const std::vector<std::string> *resp;
resp = this->request("zrandom", name, key_start, s_score_start, s_score_end, s_limit);
return _read_list(resp, ret);
}
static void
print_totals_adv(void)
{
putchar('\n');
puts(_("Totals:"));
printf(_(" Number of files: %s\n"),
uint64_to_str(totals.files, 0));
print_adv_helper(totals.streams, totals.blocks,
totals.compressed_size, totals.uncompressed_size,
totals.checks, totals.stream_padding);
if (message_verbosity_get() >= V_DEBUG) {
printf(_(" Memory needed: %s MiB\n"), uint64_to_str(
round_up_to_mib(totals.memusage_max), 0));
printf(_(" Sizes in headers: %s\n"),
totals.all_have_sizes ? _("Yes") : _("No"));
}
return;
}
static bool
print_info_basic(const xz_file_info *xfi, file_pair *pair)
{
static bool headings_displayed = false;
if (!headings_displayed) {
headings_displayed = true;
// TRANSLATORS: These are column headings. From Strms (Streams)
// to Ratio, the columns are right aligned. Check and Filename
// are left aligned. If you need longer words, it's OK to
// use two lines here. Test with "xz -l foo.xz".
puts(_("Strms Blocks Compressed Uncompressed Ratio "
"Check Filename"));
}
char checks[CHECKS_STR_SIZE];
get_check_names(checks, lzma_index_checks(xfi->idx), false);
const char *cols[7] = {
uint64_to_str(lzma_index_stream_count(xfi->idx), 0),
uint64_to_str(lzma_index_block_count(xfi->idx), 1),
uint64_to_nicestr(lzma_index_file_size(xfi->idx),
NICESTR_B, NICESTR_TIB, false, 2),
uint64_to_nicestr(lzma_index_uncompressed_size(xfi->idx),
NICESTR_B, NICESTR_TIB, false, 3),
get_ratio(lzma_index_file_size(xfi->idx),
lzma_index_uncompressed_size(xfi->idx)),
checks,
pair->src_name,
};
printf("%*s %*s %*s %*s %*s %-*s %s\n",
tuklib_mbstr_fw(cols[0], 5), cols[0],
tuklib_mbstr_fw(cols[1], 7), cols[1],
tuklib_mbstr_fw(cols[2], 11), cols[2],
tuklib_mbstr_fw(cols[3], 11), cols[3],
tuklib_mbstr_fw(cols[4], 5), cols[4],
tuklib_mbstr_fw(cols[5], 7), cols[5],
cols[6]);
return false;
}
static void
print_totals_basic(void)
{
// Print a separator line.
char line[80];
memset(line, '-', sizeof(line));
line[sizeof(line) - 1] = '\0';
puts(line);
// Get the check names.
char checks[CHECKS_STR_SIZE];
get_check_names(checks, totals.checks, false);
// Print the totals except the file count, which needs
// special handling.
printf("%5s %7s %11s %11s %5s %-7s ",
uint64_to_str(totals.streams, 0),
uint64_to_str(totals.blocks, 1),
uint64_to_nicestr(totals.compressed_size,
NICESTR_B, NICESTR_TIB, false, 2),
uint64_to_nicestr(totals.uncompressed_size,
NICESTR_B, NICESTR_TIB, false, 3),
get_ratio(totals.compressed_size,
totals.uncompressed_size),
checks);
// Since we print totals only when there are at least two files,
// the English message will always use "%s files". But some other
// languages need different forms for different plurals so we
// have to translate this with ngettext().
//
// TRANSLATORS: %s is an integer. Only the plural form of this
// message is used (e.g. "2 files"). Test with "xz -l foo.xz bar.xz".
printf(ngettext("%s file\n", "%s files\n",
totals.files <= ULONG_MAX ? totals.files
: (totals.files % 1000000) + 1000000),
uint64_to_str(totals.files, 0));
return;
}
// read a block by hash from a block store named by the hash and store it in bs.
void import_exo_inputs_to_bs(HashBlockStore *bs, void *ptr, int size, hash_t *digest) {
string input_name = uint64_to_str(NUM_HASH_CHUNKS, digest->bit);
unsigned char hash[20];
char hexstring[42];
sha1::calc(input_name.c_str(), strlen(input_name.c_str()), hash);
sha1::toHexString(hash, hexstring);
hexstring[41] = '\0';
char db_file_path[BUFLEN];
snprintf(db_file_path, BUFLEN - 1, "%s/%s/%s", FOLDER_STATE, ROOT_FOLDER_BLOCK_STORES, hexstring);
FILE *fp = fopen(db_file_path, "rb");
if (fread(ptr, 1, size, fp) != (size_t)size) {
printf("error reading %d bytes from %s\n", size, db_file_path);
}
fclose(fp);
//string db_file_path_str(db_file_path);
//HashBlockStore* _block_store = new ConfigurableBlockStore(db_file_path_str);
//__hashget(_block_store, ptr, digest, size);
//delete _block_store;
__hashput(bs, digest, ptr, size);
}
extern void
coder_set_compression_settings(void)
{
// The default check type is CRC64, but fallback to CRC32
// if CRC64 isn't supported by the copy of liblzma we are
// using. CRC32 is always supported.
if (check_default) {
check = LZMA_CHECK_CRC64;
if (!lzma_check_is_supported(check))
check = LZMA_CHECK_CRC32;
}
// Options for LZMA1 or LZMA2 in case we are using a preset.
static lzma_options_lzma opt_lzma;
if (filters_count == 0) {
// We are using a preset. This is not a good idea in raw mode
// except when playing around with things. Different versions
// of this software may use different options in presets, and
// thus make uncompressing the raw data difficult.
if (opt_format == FORMAT_RAW) {
// The message is shown only if warnings are allowed
// but the exit status isn't changed.
message(V_WARNING, _("Using a preset in raw mode "
"is discouraged."));
message(V_WARNING, _("The exact options of the "
"presets may vary between software "
"versions."));
}
// Get the preset for LZMA1 or LZMA2.
if (lzma_lzma_preset(&opt_lzma, preset_number))
message_bug();
// Use LZMA2 except with --format=lzma we use LZMA1.
filters[0].id = opt_format == FORMAT_LZMA
? LZMA_FILTER_LZMA1 : LZMA_FILTER_LZMA2;
filters[0].options = &opt_lzma;
filters_count = 1;
}
// Terminate the filter options array.
filters[filters_count].id = LZMA_VLI_UNKNOWN;
// If we are using the .lzma format, allow exactly one filter
// which has to be LZMA1.
if (opt_format == FORMAT_LZMA && (filters_count != 1
|| filters[0].id != LZMA_FILTER_LZMA1))
message_fatal(_("The .lzma format supports only "
"the LZMA1 filter"));
// If we are using the .xz format, make sure that there is no LZMA1
// filter to prevent LZMA_PROG_ERROR.
if (opt_format == FORMAT_XZ)
for (size_t i = 0; i < filters_count; ++i)
if (filters[i].id == LZMA_FILTER_LZMA1)
message_fatal(_("LZMA1 cannot be used "
"with the .xz format"));
// Print the selected filter chain.
message_filters_show(V_DEBUG, filters);
// Get the memory usage. Note that if --format=raw was used,
// we can be decompressing.
const uint64_t memory_limit = hardware_memlimit_get(opt_mode);
uint64_t memory_usage;
if (opt_mode == MODE_COMPRESS) {
#ifdef MYTHREAD_ENABLED
if (opt_format == FORMAT_XZ && hardware_threads_get() > 1) {
mt_options.threads = hardware_threads_get();
mt_options.block_size = opt_block_size;
mt_options.check = check;
memory_usage = lzma_stream_encoder_mt_memusage(
&mt_options);
if (memory_usage != UINT64_MAX)
message(V_DEBUG, _("Using up to %" PRIu32
" threads."),
mt_options.threads);
} else
#endif
{
memory_usage = lzma_raw_encoder_memusage(filters);
}
} else {
memory_usage = lzma_raw_decoder_memusage(filters);
}
if (memory_usage == UINT64_MAX)
message_fatal(_("Unsupported filter chain or filter options"));
// Print memory usage info before possible dictionary
// size auto-adjusting.
message_mem_needed(V_DEBUG, memory_usage);
if (opt_mode == MODE_COMPRESS) {
const uint64_t decmem = lzma_raw_decoder_memusage(filters);
if (decmem != UINT64_MAX)
message(V_DEBUG, _("Decompression will need "
"%s MiB of memory."), uint64_to_str(
round_up_to_mib(decmem), 0));
}
if (memory_usage <= memory_limit)
return;
// If --no-auto-adjust was used or we didn't find LZMA1 or
// LZMA2 as the last filter, give an error immediately.
// --format=raw implies --no-auto-adjust.
if (!opt_auto_adjust || opt_format == FORMAT_RAW)
memlimit_too_small(memory_usage);
assert(opt_mode == MODE_COMPRESS);
#ifdef MYTHREAD_ENABLED
if (opt_format == FORMAT_XZ && mt_options.threads > 1) {
// Try to reduce the number of threads before
// adjusting the compression settings down.
do {
// FIXME? The real single-threaded mode has
// lower memory usage, but it's not comparable
// because it doesn't write the size info
// into Block Headers.
if (--mt_options.threads == 0)
memlimit_too_small(memory_usage);
memory_usage = lzma_stream_encoder_mt_memusage(
&mt_options);
if (memory_usage == UINT64_MAX)
message_bug();
} while (memory_usage > memory_limit);
message(V_WARNING, _("Adjusted the number of threads "
"from %s to %s to not exceed "
"the memory usage limit of %s MiB"),
uint64_to_str(hardware_threads_get(), 0),
uint64_to_str(mt_options.threads, 1),
uint64_to_str(round_up_to_mib(
memory_limit), 2));
}
#endif
if (memory_usage <= memory_limit) {
return;
}
// Look for the last filter if it is LZMA2 or LZMA1, so we can make
// it use less RAM. With other filters we don't know what to do.
size_t i = 0;
while (filters[i].id != LZMA_FILTER_LZMA2
&& filters[i].id != LZMA_FILTER_LZMA1) {
if (filters[i].id == LZMA_VLI_UNKNOWN)
memlimit_too_small(memory_usage);
++i;
}
// Decrease the dictionary size until we meet the memory
// usage limit. First round down to full mebibytes.
lzma_options_lzma *opt = filters[i].options;
const uint32_t orig_dict_size = opt->dict_size;
opt->dict_size &= ~((UINT32_C(1) << 20) - 1);
while (true) {
// If it is below 1 MiB, auto-adjusting failed. We could be
// more sophisticated and scale it down even more, but let's
// see if many complain about this version.
//
// FIXME: Displays the scaled memory usage instead
// of the original.
if (opt->dict_size < (UINT32_C(1) << 20))
memlimit_too_small(memory_usage);
memory_usage = lzma_raw_encoder_memusage(filters);
if (memory_usage == UINT64_MAX)
message_bug();
// Accept it if it is low enough.
if (memory_usage <= memory_limit)
break;
// Otherwise 1 MiB down and try again. I hope this
// isn't too slow method for cases where the original
// dict_size is very big.
opt->dict_size -= UINT32_C(1) << 20;
}
// Tell the user that we decreased the dictionary size.
message(V_WARNING, _("Adjusted LZMA%c dictionary size "
"from %s MiB to %s MiB to not exceed "
"the memory usage limit of %s MiB"),
filters[i].id == LZMA_FILTER_LZMA2
? '2' : '1',
uint64_to_str(orig_dict_size >> 20, 0),
uint64_to_str(opt->dict_size >> 20, 1),
uint64_to_str(round_up_to_mib(memory_limit), 2));
return;
}
static bool
print_info_adv(xz_file_info *xfi, file_pair *pair)
{
// Print the overall information.
print_adv_helper(lzma_index_stream_count(xfi->idx),
lzma_index_block_count(xfi->idx),
lzma_index_file_size(xfi->idx),
lzma_index_uncompressed_size(xfi->idx),
lzma_index_checks(xfi->idx),
xfi->stream_padding);
// Size of the biggest Check. This is used to calculate the width
// of the CheckVal field. The table would get insanely wide if
// we always reserved space for 64-byte Check (128 chars as hex).
uint32_t check_max = 0;
// Print information about the Streams.
//
// TRANSLATORS: The second line is column headings. All except
// Check are right aligned; Check is left aligned. Test with
// "xz -lv foo.xz".
puts(_(" Streams:\n Stream Blocks"
" CompOffset UncompOffset"
" CompSize UncompSize Ratio"
" Check Padding"));
lzma_index_iter iter;
lzma_index_iter_init(&iter, xfi->idx);
while (!lzma_index_iter_next(&iter, LZMA_INDEX_ITER_STREAM)) {
const char *cols1[4] = {
uint64_to_str(iter.stream.number, 0),
uint64_to_str(iter.stream.block_count, 1),
uint64_to_str(iter.stream.compressed_offset, 2),
uint64_to_str(iter.stream.uncompressed_offset, 3),
};
printf(" %*s %*s %*s %*s ",
tuklib_mbstr_fw(cols1[0], 6), cols1[0],
tuklib_mbstr_fw(cols1[1], 9), cols1[1],
tuklib_mbstr_fw(cols1[2], 15), cols1[2],
tuklib_mbstr_fw(cols1[3], 15), cols1[3]);
const char *cols2[5] = {
uint64_to_str(iter.stream.compressed_size, 0),
uint64_to_str(iter.stream.uncompressed_size, 1),
get_ratio(iter.stream.compressed_size,
iter.stream.uncompressed_size),
_(check_names[iter.stream.flags->check]),
uint64_to_str(iter.stream.padding, 2),
};
printf("%*s %*s %*s %-*s %*s\n",
tuklib_mbstr_fw(cols2[0], 15), cols2[0],
tuklib_mbstr_fw(cols2[1], 15), cols2[1],
tuklib_mbstr_fw(cols2[2], 5), cols2[2],
tuklib_mbstr_fw(cols2[3], 10), cols2[3],
tuklib_mbstr_fw(cols2[4], 7), cols2[4]);
// Update the maximum Check size.
if (lzma_check_size(iter.stream.flags->check) > check_max)
check_max = lzma_check_size(iter.stream.flags->check);
}
// Cache the verbosity level to a local variable.
const bool detailed = message_verbosity_get() >= V_DEBUG;
// Information collected from Block Headers
block_header_info bhi;
// Print information about the Blocks but only if there is
// at least one Block.
if (lzma_index_block_count(xfi->idx) > 0) {
// Calculate the width of the CheckVal field.
const int checkval_width = my_max(8, 2 * check_max);
// TRANSLATORS: The second line is column headings. All
// except Check are right aligned; Check is left aligned.
printf(_(" Blocks:\n Stream Block"
" CompOffset UncompOffset"
" TotalSize UncompSize Ratio Check"));
if (detailed) {
// TRANSLATORS: These are additional column headings
// for the most verbose listing mode. CheckVal
// (Check value), Flags, and Filters are left aligned.
// Header (Block Header Size), CompSize, and MemUsage
// are right aligned. %*s is replaced with 0-120
// spaces to make the CheckVal column wide enough.
// Test with "xz -lvv foo.xz".
printf(_(" CheckVal %*s Header Flags "
"CompSize MemUsage Filters"),
checkval_width - 8, "");
}
putchar('\n');
lzma_index_iter_init(&iter, xfi->idx);
// Iterate over the Blocks.
while (!lzma_index_iter_next(&iter, LZMA_INDEX_ITER_BLOCK)) {
if (detailed && parse_details(pair, &iter, &bhi, xfi))
return true;
const char *cols1[4] = {
uint64_to_str(iter.stream.number, 0),
uint64_to_str(
iter.block.number_in_stream, 1),
uint64_to_str(
iter.block.compressed_file_offset, 2),
uint64_to_str(
iter.block.uncompressed_file_offset, 3)
};
printf(" %*s %*s %*s %*s ",
tuklib_mbstr_fw(cols1[0], 6), cols1[0],
tuklib_mbstr_fw(cols1[1], 9), cols1[1],
tuklib_mbstr_fw(cols1[2], 15), cols1[2],
tuklib_mbstr_fw(cols1[3], 15), cols1[3]);
const char *cols2[4] = {
uint64_to_str(iter.block.total_size, 0),
uint64_to_str(iter.block.uncompressed_size,
1),
get_ratio(iter.block.total_size,
iter.block.uncompressed_size),
_(check_names[iter.stream.flags->check])
};
printf("%*s %*s %*s %-*s",
tuklib_mbstr_fw(cols2[0], 15), cols2[0],
tuklib_mbstr_fw(cols2[1], 15), cols2[1],
tuklib_mbstr_fw(cols2[2], 5), cols2[2],
tuklib_mbstr_fw(cols2[3], detailed ? 11 : 1),
cols2[3]);
if (detailed) {
const lzma_vli compressed_size
= iter.block.unpadded_size
- bhi.header_size
- lzma_check_size(
iter.stream.flags->check);
const char *cols3[6] = {
check_value,
uint64_to_str(bhi.header_size, 0),
bhi.flags,
uint64_to_str(compressed_size, 1),
uint64_to_str(
round_up_to_mib(bhi.memusage),
2),
bhi.filter_chain
};
// Show MiB for memory usage, because it
// is the only size which is not in bytes.
printf("%-*s %*s %-5s %*s %*s MiB %s",
checkval_width, cols3[0],
tuklib_mbstr_fw(cols3[1], 6), cols3[1],
cols3[2],
tuklib_mbstr_fw(cols3[3], 15),
cols3[3],
tuklib_mbstr_fw(cols3[4], 7), cols3[4],
cols3[5]);
}
putchar('\n');
}
}
if (detailed) {
printf(_(" Memory needed: %s MiB\n"), uint64_to_str(
round_up_to_mib(xfi->memusage_max), 0));
printf(_(" Sizes in headers: %s\n"),
xfi->all_have_sizes ? _("Yes") : _("No"));
printf(_(" Minimum XZ Utils version: %s\n"),
xz_ver_to_str(xfi->min_version));
}
return false;
}