Esempio n. 1
0
static void
print_adv_helper(uint64_t stream_count, uint64_t block_count,
		uint64_t compressed_size, uint64_t uncompressed_size,
		uint32_t checks, uint64_t stream_padding)
{
	char checks_str[CHECKS_STR_SIZE];
	get_check_names(checks_str, checks, true);

	printf(_("  Streams:            %s\n"),
			uint64_to_str(stream_count, 0));
	printf(_("  Blocks:             %s\n"),
			uint64_to_str(block_count, 0));
	printf(_("  Compressed size:    %s\n"),
			uint64_to_nicestr(compressed_size,
				NICESTR_B, NICESTR_TIB, true, 0));
	printf(_("  Uncompressed size:  %s\n"),
			uint64_to_nicestr(uncompressed_size,
				NICESTR_B, NICESTR_TIB, true, 0));
	printf(_("  Ratio:              %s\n"),
			get_ratio(compressed_size, uncompressed_size));
	printf(_("  Check:              %s\n"), checks_str);
	printf(_("  Stream padding:     %s\n"),
			uint64_to_nicestr(stream_padding,
				NICESTR_B, NICESTR_TIB, true, 0));
	return;
}
Esempio n. 2
0
Status ClientImpl::zrrange(const std::string &name,
		uint64_t offset, uint64_t limit,
		std::vector<std::string> *ret)
{
	std::string s_offset = uint64_to_str(offset);
	std::string s_limit = uint64_to_str(limit);
	const std::vector<std::string> *resp;
	resp = this->request("zrrange", name, s_offset, s_limit);
	return _read_list(resp, ret);
}
Esempio n. 3
0
// store a block in another block store named by the block's hash.
void export_exo_inputs(void *ptr, int size, hash_t *digest) {
  unsigned char hash[20];
  char hexstring[42];
 
  // compute hash of the data
  __hashbits(digest, ptr, size);
  
  int rank;
  MPI_Comm_rank(MPI_COMM_WORLD, &rank);

  char tmp_db_file_path[BUFLEN];
  snprintf(tmp_db_file_path, BUFLEN - 1, "%s/%s/prover_%d_tmp", FOLDER_STATE, ROOT_FOLDER_BLOCK_STORES, rank);
  FILE *fp = fopen(tmp_db_file_path, "wb");
  fwrite(ptr, 1, size, fp);
  fclose(fp);
  
  string input_name = uint64_to_str(NUM_HASH_CHUNKS, digest->bit);
  sha1::calc(input_name.c_str(), strlen(input_name.c_str()), hash);
  sha1::toHexString(hash, hexstring);
  hexstring[41] = '\0';
  
  char com_mv_db_file_path[BUFLEN];
  snprintf(com_mv_db_file_path, BUFLEN - 1, "mv %s %s/%s/%s", tmp_db_file_path, FOLDER_STATE, ROOT_FOLDER_BLOCK_STORES, hexstring);
  if (system(com_mv_db_file_path) != 0) {
    printf("ERROR executing %s\n", com_mv_db_file_path);
  }
  
  //char db_file_path[BUFLEN];
  //snprintf(db_file_path, BUFLEN - 1, "%s/%s/%s", FOLDER_STATE, ROOT_FOLDER_BLOCK_STORES, hexstring);
  //string db_file_path_str(db_file_path);
  //HashBlockStore* _block_store = new ConfigurableBlockStore(db_file_path_str);
  //__hashput(_block_store, digest, ptr, size);
  //delete _block_store;
}
Esempio n. 4
0
Status ClientImpl::rscan(const std::string &key_start, const std::string &key_end,
	uint64_t limit, std::vector<std::string> *ret)
{
	std::string s_limit = uint64_to_str(limit);
	const std::vector<std::string> *resp;
	resp = this->request("rscan", key_start, key_end, s_limit);
	return _read_list(resp, ret);
}
Esempio n. 5
0
Status ClientImpl::zrandom(const std::string &name, const std::string &key_start,
        int64_t *score_start, int64_t *score_end,
        uint64_t limit, std::vector<std::string> *ret)
{
        std::string s_score_start = score_start? int_to_str(*score_start) : "";
        std::string s_score_end = score_end? int_to_str(*score_end) : "";
        std::string s_limit = uint64_to_str(limit);
        const std::vector<std::string> *resp;
        resp = this->request("zrandom", name, key_start, s_score_start, s_score_end, s_limit);
        return _read_list(resp, ret);
}
Esempio n. 6
0
static void
print_totals_adv(void)
{
	putchar('\n');
	puts(_("Totals:"));
	printf(_("  Number of files:    %s\n"),
			uint64_to_str(totals.files, 0));
	print_adv_helper(totals.streams, totals.blocks,
			totals.compressed_size, totals.uncompressed_size,
			totals.checks, totals.stream_padding);

	if (message_verbosity_get() >= V_DEBUG) {
		printf(_("  Memory needed:      %s MiB\n"), uint64_to_str(
				round_up_to_mib(totals.memusage_max), 0));
		printf(_("  Sizes in headers:   %s\n"),
				totals.all_have_sizes ? _("Yes") : _("No"));
	}

	return;
}
Esempio n. 7
0
static bool
print_info_basic(const xz_file_info *xfi, file_pair *pair)
{
	static bool headings_displayed = false;
	if (!headings_displayed) {
		headings_displayed = true;
		// TRANSLATORS: These are column headings. From Strms (Streams)
		// to Ratio, the columns are right aligned. Check and Filename
		// are left aligned. If you need longer words, it's OK to
		// use two lines here. Test with "xz -l foo.xz".
		puts(_("Strms  Blocks   Compressed Uncompressed  Ratio  "
				"Check   Filename"));
	}

	char checks[CHECKS_STR_SIZE];
	get_check_names(checks, lzma_index_checks(xfi->idx), false);

	const char *cols[7] = {
		uint64_to_str(lzma_index_stream_count(xfi->idx), 0),
		uint64_to_str(lzma_index_block_count(xfi->idx), 1),
		uint64_to_nicestr(lzma_index_file_size(xfi->idx),
			NICESTR_B, NICESTR_TIB, false, 2),
		uint64_to_nicestr(lzma_index_uncompressed_size(xfi->idx),
			NICESTR_B, NICESTR_TIB, false, 3),
		get_ratio(lzma_index_file_size(xfi->idx),
			lzma_index_uncompressed_size(xfi->idx)),
		checks,
		pair->src_name,
	};
	printf("%*s %*s  %*s  %*s  %*s  %-*s %s\n",
			tuklib_mbstr_fw(cols[0], 5), cols[0],
			tuklib_mbstr_fw(cols[1], 7), cols[1],
			tuklib_mbstr_fw(cols[2], 11), cols[2],
			tuklib_mbstr_fw(cols[3], 11), cols[3],
			tuklib_mbstr_fw(cols[4], 5), cols[4],
			tuklib_mbstr_fw(cols[5], 7), cols[5],
			cols[6]);

	return false;
}
Esempio n. 8
0
static void
print_totals_basic(void)
{
	// Print a separator line.
	char line[80];
	memset(line, '-', sizeof(line));
	line[sizeof(line) - 1] = '\0';
	puts(line);

	// Get the check names.
	char checks[CHECKS_STR_SIZE];
	get_check_names(checks, totals.checks, false);

	// Print the totals except the file count, which needs
	// special handling.
	printf("%5s %7s  %11s  %11s  %5s  %-7s ",
			uint64_to_str(totals.streams, 0),
			uint64_to_str(totals.blocks, 1),
			uint64_to_nicestr(totals.compressed_size,
				NICESTR_B, NICESTR_TIB, false, 2),
			uint64_to_nicestr(totals.uncompressed_size,
				NICESTR_B, NICESTR_TIB, false, 3),
			get_ratio(totals.compressed_size,
				totals.uncompressed_size),
			checks);

	// Since we print totals only when there are at least two files,
	// the English message will always use "%s files". But some other
	// languages need different forms for different plurals so we
	// have to translate this with ngettext().
	//
	// TRANSLATORS: %s is an integer. Only the plural form of this
	// message is used (e.g. "2 files"). Test with "xz -l foo.xz bar.xz".
	printf(ngettext("%s file\n", "%s files\n",
			totals.files <= ULONG_MAX ? totals.files
				: (totals.files % 1000000) + 1000000),
			uint64_to_str(totals.files, 0));

	return;
}
Esempio n. 9
0
// read a block by hash from a block store named by the hash and store it in bs.
void import_exo_inputs_to_bs(HashBlockStore *bs, void *ptr, int size, hash_t *digest) {
  string input_name = uint64_to_str(NUM_HASH_CHUNKS, digest->bit);
  unsigned char hash[20];
  char hexstring[42];
  sha1::calc(input_name.c_str(), strlen(input_name.c_str()), hash);
  sha1::toHexString(hash, hexstring);
  hexstring[41] = '\0';

  char db_file_path[BUFLEN];
  snprintf(db_file_path, BUFLEN - 1, "%s/%s/%s", FOLDER_STATE, ROOT_FOLDER_BLOCK_STORES, hexstring);
  FILE *fp = fopen(db_file_path, "rb");
  if (fread(ptr, 1, size, fp) != (size_t)size) {
    printf("error reading %d bytes from %s\n", size, db_file_path);
  }
  fclose(fp);
  
  //string db_file_path_str(db_file_path);
  //HashBlockStore* _block_store = new ConfigurableBlockStore(db_file_path_str);
  //__hashget(_block_store, ptr, digest, size);
  //delete _block_store;
  
  __hashput(bs, digest, ptr, size);
}
Esempio n. 10
0
File: coder.c Progetto: Garen/xz
extern void
coder_set_compression_settings(void)
{
	// The default check type is CRC64, but fallback to CRC32
	// if CRC64 isn't supported by the copy of liblzma we are
	// using. CRC32 is always supported.
	if (check_default) {
		check = LZMA_CHECK_CRC64;
		if (!lzma_check_is_supported(check))
			check = LZMA_CHECK_CRC32;
	}

	// Options for LZMA1 or LZMA2 in case we are using a preset.
	static lzma_options_lzma opt_lzma;

	if (filters_count == 0) {
		// We are using a preset. This is not a good idea in raw mode
		// except when playing around with things. Different versions
		// of this software may use different options in presets, and
		// thus make uncompressing the raw data difficult.
		if (opt_format == FORMAT_RAW) {
			// The message is shown only if warnings are allowed
			// but the exit status isn't changed.
			message(V_WARNING, _("Using a preset in raw mode "
					"is discouraged."));
			message(V_WARNING, _("The exact options of the "
					"presets may vary between software "
					"versions."));
		}

		// Get the preset for LZMA1 or LZMA2.
		if (lzma_lzma_preset(&opt_lzma, preset_number))
			message_bug();

		// Use LZMA2 except with --format=lzma we use LZMA1.
		filters[0].id = opt_format == FORMAT_LZMA
				? LZMA_FILTER_LZMA1 : LZMA_FILTER_LZMA2;
		filters[0].options = &opt_lzma;
		filters_count = 1;
	}

	// Terminate the filter options array.
	filters[filters_count].id = LZMA_VLI_UNKNOWN;

	// If we are using the .lzma format, allow exactly one filter
	// which has to be LZMA1.
	if (opt_format == FORMAT_LZMA && (filters_count != 1
			|| filters[0].id != LZMA_FILTER_LZMA1))
		message_fatal(_("The .lzma format supports only "
				"the LZMA1 filter"));

	// If we are using the .xz format, make sure that there is no LZMA1
	// filter to prevent LZMA_PROG_ERROR.
	if (opt_format == FORMAT_XZ)
		for (size_t i = 0; i < filters_count; ++i)
			if (filters[i].id == LZMA_FILTER_LZMA1)
				message_fatal(_("LZMA1 cannot be used "
						"with the .xz format"));

	// Print the selected filter chain.
	message_filters_show(V_DEBUG, filters);

	// Get the memory usage. Note that if --format=raw was used,
	// we can be decompressing.
	const uint64_t memory_limit = hardware_memlimit_get(opt_mode);
	uint64_t memory_usage;
	if (opt_mode == MODE_COMPRESS) {
#ifdef MYTHREAD_ENABLED
		if (opt_format == FORMAT_XZ && hardware_threads_get() > 1) {
			mt_options.threads = hardware_threads_get();
			mt_options.block_size = opt_block_size;
			mt_options.check = check;
			memory_usage = lzma_stream_encoder_mt_memusage(
					&mt_options);
			if (memory_usage != UINT64_MAX)
				message(V_DEBUG, _("Using up to %" PRIu32
						" threads."),
						mt_options.threads);
		} else
#endif
		{
			memory_usage = lzma_raw_encoder_memusage(filters);
		}
	} else {
		memory_usage = lzma_raw_decoder_memusage(filters);
	}

	if (memory_usage == UINT64_MAX)
		message_fatal(_("Unsupported filter chain or filter options"));

	// Print memory usage info before possible dictionary
	// size auto-adjusting.
	message_mem_needed(V_DEBUG, memory_usage);
	if (opt_mode == MODE_COMPRESS) {
		const uint64_t decmem = lzma_raw_decoder_memusage(filters);
		if (decmem != UINT64_MAX)
			message(V_DEBUG, _("Decompression will need "
					"%s MiB of memory."), uint64_to_str(
						round_up_to_mib(decmem), 0));
	}

	if (memory_usage <= memory_limit)
		return;

	// If --no-auto-adjust was used or we didn't find LZMA1 or
	// LZMA2 as the last filter, give an error immediately.
	// --format=raw implies --no-auto-adjust.
	if (!opt_auto_adjust || opt_format == FORMAT_RAW)
		memlimit_too_small(memory_usage);

	assert(opt_mode == MODE_COMPRESS);

#ifdef MYTHREAD_ENABLED
	if (opt_format == FORMAT_XZ && mt_options.threads > 1) {
		// Try to reduce the number of threads before
		// adjusting the compression settings down.
		do {
			// FIXME? The real single-threaded mode has
			// lower memory usage, but it's not comparable
			// because it doesn't write the size info
			// into Block Headers.
			if (--mt_options.threads == 0)
				memlimit_too_small(memory_usage);

			memory_usage = lzma_stream_encoder_mt_memusage(
					&mt_options);
			if (memory_usage == UINT64_MAX)
				message_bug();

		} while (memory_usage > memory_limit);

		message(V_WARNING, _("Adjusted the number of threads "
			"from %s to %s to not exceed "
			"the memory usage limit of %s MiB"),
			uint64_to_str(hardware_threads_get(), 0),
			uint64_to_str(mt_options.threads, 1),
			uint64_to_str(round_up_to_mib(
				memory_limit), 2));
	}
#endif

	if (memory_usage <= memory_limit) {
		return;
	}
	// Look for the last filter if it is LZMA2 or LZMA1, so we can make
	// it use less RAM. With other filters we don't know what to do.
	size_t i = 0;
	while (filters[i].id != LZMA_FILTER_LZMA2
			&& filters[i].id != LZMA_FILTER_LZMA1) {
		if (filters[i].id == LZMA_VLI_UNKNOWN)
			memlimit_too_small(memory_usage);

		++i;
	}

	// Decrease the dictionary size until we meet the memory
	// usage limit. First round down to full mebibytes.
	lzma_options_lzma *opt = filters[i].options;
	const uint32_t orig_dict_size = opt->dict_size;
	opt->dict_size &= ~((UINT32_C(1) << 20) - 1);
	while (true) {
		// If it is below 1 MiB, auto-adjusting failed. We could be
		// more sophisticated and scale it down even more, but let's
		// see if many complain about this version.
		//
		// FIXME: Displays the scaled memory usage instead
		// of the original.
		if (opt->dict_size < (UINT32_C(1) << 20))
			memlimit_too_small(memory_usage);

		memory_usage = lzma_raw_encoder_memusage(filters);
		if (memory_usage == UINT64_MAX)
			message_bug();

		// Accept it if it is low enough.
		if (memory_usage <= memory_limit)
			break;

		// Otherwise 1 MiB down and try again. I hope this
		// isn't too slow method for cases where the original
		// dict_size is very big.
		opt->dict_size -= UINT32_C(1) << 20;
	}

	// Tell the user that we decreased the dictionary size.
	message(V_WARNING, _("Adjusted LZMA%c dictionary size "
			"from %s MiB to %s MiB to not exceed "
			"the memory usage limit of %s MiB"),
			filters[i].id == LZMA_FILTER_LZMA2
				? '2' : '1',
			uint64_to_str(orig_dict_size >> 20, 0),
			uint64_to_str(opt->dict_size >> 20, 1),
			uint64_to_str(round_up_to_mib(memory_limit), 2));

	return;
}
Esempio n. 11
0
static bool
print_info_adv(xz_file_info *xfi, file_pair *pair)
{
	// Print the overall information.
	print_adv_helper(lzma_index_stream_count(xfi->idx),
			lzma_index_block_count(xfi->idx),
			lzma_index_file_size(xfi->idx),
			lzma_index_uncompressed_size(xfi->idx),
			lzma_index_checks(xfi->idx),
			xfi->stream_padding);

	// Size of the biggest Check. This is used to calculate the width
	// of the CheckVal field. The table would get insanely wide if
	// we always reserved space for 64-byte Check (128 chars as hex).
	uint32_t check_max = 0;

	// Print information about the Streams.
	//
	// TRANSLATORS: The second line is column headings. All except
	// Check are right aligned; Check is left aligned. Test with
	// "xz -lv foo.xz".
	puts(_("  Streams:\n    Stream    Blocks"
			"      CompOffset    UncompOffset"
			"        CompSize      UncompSize  Ratio"
			"  Check      Padding"));

	lzma_index_iter iter;
	lzma_index_iter_init(&iter, xfi->idx);

	while (!lzma_index_iter_next(&iter, LZMA_INDEX_ITER_STREAM)) {
		const char *cols1[4] = {
			uint64_to_str(iter.stream.number, 0),
			uint64_to_str(iter.stream.block_count, 1),
			uint64_to_str(iter.stream.compressed_offset, 2),
			uint64_to_str(iter.stream.uncompressed_offset, 3),
		};
		printf("    %*s %*s %*s %*s ",
				tuklib_mbstr_fw(cols1[0], 6), cols1[0],
				tuklib_mbstr_fw(cols1[1], 9), cols1[1],
				tuklib_mbstr_fw(cols1[2], 15), cols1[2],
				tuklib_mbstr_fw(cols1[3], 15), cols1[3]);

		const char *cols2[5] = {
			uint64_to_str(iter.stream.compressed_size, 0),
			uint64_to_str(iter.stream.uncompressed_size, 1),
			get_ratio(iter.stream.compressed_size,
				iter.stream.uncompressed_size),
			_(check_names[iter.stream.flags->check]),
			uint64_to_str(iter.stream.padding, 2),
		};
		printf("%*s %*s  %*s  %-*s %*s\n",
				tuklib_mbstr_fw(cols2[0], 15), cols2[0],
				tuklib_mbstr_fw(cols2[1], 15), cols2[1],
				tuklib_mbstr_fw(cols2[2], 5), cols2[2],
				tuklib_mbstr_fw(cols2[3], 10), cols2[3],
				tuklib_mbstr_fw(cols2[4], 7), cols2[4]);

		// Update the maximum Check size.
		if (lzma_check_size(iter.stream.flags->check) > check_max)
			check_max = lzma_check_size(iter.stream.flags->check);
	}

	// Cache the verbosity level to a local variable.
	const bool detailed = message_verbosity_get() >= V_DEBUG;

	// Information collected from Block Headers
	block_header_info bhi;

	// Print information about the Blocks but only if there is
	// at least one Block.
	if (lzma_index_block_count(xfi->idx) > 0) {
		// Calculate the width of the CheckVal field.
		const int checkval_width = my_max(8, 2 * check_max);

		// TRANSLATORS: The second line is column headings. All
		// except Check are right aligned; Check is left aligned.
		printf(_("  Blocks:\n    Stream     Block"
			"      CompOffset    UncompOffset"
			"       TotalSize      UncompSize  Ratio  Check"));

		if (detailed) {
			// TRANSLATORS: These are additional column headings
			// for the most verbose listing mode. CheckVal
			// (Check value), Flags, and Filters are left aligned.
			// Header (Block Header Size), CompSize, and MemUsage
			// are right aligned. %*s is replaced with 0-120
			// spaces to make the CheckVal column wide enough.
			// Test with "xz -lvv foo.xz".
			printf(_("      CheckVal %*s Header  Flags        "
					"CompSize    MemUsage  Filters"),
					checkval_width - 8, "");
		}

		putchar('\n');

		lzma_index_iter_init(&iter, xfi->idx);

		// Iterate over the Blocks.
		while (!lzma_index_iter_next(&iter, LZMA_INDEX_ITER_BLOCK)) {
			if (detailed && parse_details(pair, &iter, &bhi, xfi))
					return true;

			const char *cols1[4] = {
				uint64_to_str(iter.stream.number, 0),
				uint64_to_str(
					iter.block.number_in_stream, 1),
				uint64_to_str(
					iter.block.compressed_file_offset, 2),
				uint64_to_str(
					iter.block.uncompressed_file_offset, 3)
			};
			printf("    %*s %*s %*s %*s ",
				tuklib_mbstr_fw(cols1[0], 6), cols1[0],
				tuklib_mbstr_fw(cols1[1], 9), cols1[1],
				tuklib_mbstr_fw(cols1[2], 15), cols1[2],
				tuklib_mbstr_fw(cols1[3], 15), cols1[3]);

			const char *cols2[4] = {
				uint64_to_str(iter.block.total_size, 0),
				uint64_to_str(iter.block.uncompressed_size,
						1),
				get_ratio(iter.block.total_size,
					iter.block.uncompressed_size),
				_(check_names[iter.stream.flags->check])
			};
			printf("%*s %*s  %*s  %-*s",
				tuklib_mbstr_fw(cols2[0], 15), cols2[0],
				tuklib_mbstr_fw(cols2[1], 15), cols2[1],
				tuklib_mbstr_fw(cols2[2], 5), cols2[2],
				tuklib_mbstr_fw(cols2[3], detailed ? 11 : 1),
					cols2[3]);

			if (detailed) {
				const lzma_vli compressed_size
						= iter.block.unpadded_size
						- bhi.header_size
						- lzma_check_size(
						iter.stream.flags->check);

				const char *cols3[6] = {
					check_value,
					uint64_to_str(bhi.header_size, 0),
					bhi.flags,
					uint64_to_str(compressed_size, 1),
					uint64_to_str(
						round_up_to_mib(bhi.memusage),
						2),
					bhi.filter_chain
				};
				// Show MiB for memory usage, because it
				// is the only size which is not in bytes.
				printf("%-*s  %*s  %-5s %*s %*s MiB  %s",
					checkval_width, cols3[0],
					tuklib_mbstr_fw(cols3[1], 6), cols3[1],
					cols3[2],
					tuklib_mbstr_fw(cols3[3], 15),
						cols3[3],
					tuklib_mbstr_fw(cols3[4], 7), cols3[4],
					cols3[5]);
			}

			putchar('\n');
		}
	}

	if (detailed) {
		printf(_("  Memory needed:      %s MiB\n"), uint64_to_str(
				round_up_to_mib(xfi->memusage_max), 0));
		printf(_("  Sizes in headers:   %s\n"),
				xfi->all_have_sizes ? _("Yes") : _("No"));
		printf(_("  Minimum XZ Utils version: %s\n"),
				xz_ver_to_str(xfi->min_version));
	}

	return false;
}