extern void
coder_set_compression_settings(void)
{
// The default check type is CRC64, but fallback to CRC32
// if CRC64 isn't supported by the copy of liblzma we are
// using. CRC32 is always supported.
if (check_default) {
check = LZMA_CHECK_CRC64;
if (!lzma_check_is_supported(check))
check = LZMA_CHECK_CRC32;
}
// Options for LZMA1 or LZMA2 in case we are using a preset.
static lzma_options_lzma opt_lzma;
if (filters_count == 0) {
// We are using a preset. This is not a good idea in raw mode
// except when playing around with things. Different versions
// of this software may use different options in presets, and
// thus make uncompressing the raw data difficult.
if (opt_format == FORMAT_RAW) {
// The message is shown only if warnings are allowed
// but the exit status isn't changed.
message(V_WARNING, _("Using a preset in raw mode "
"is discouraged."));
message(V_WARNING, _("The exact options of the "
"presets may vary between software "
"versions."));
}
// Get the preset for LZMA1 or LZMA2.
if (lzma_lzma_preset(&opt_lzma, preset_number))
message_bug();
// Use LZMA2 except with --format=lzma we use LZMA1.
filters[0].id = opt_format == FORMAT_LZMA
? LZMA_FILTER_LZMA1 : LZMA_FILTER_LZMA2;
filters[0].options = &opt_lzma;
filters_count = 1;
}
// Terminate the filter options array.
filters[filters_count].id = LZMA_VLI_UNKNOWN;
// If we are using the .lzma format, allow exactly one filter
// which has to be LZMA1.
if (opt_format == FORMAT_LZMA && (filters_count != 1
|| filters[0].id != LZMA_FILTER_LZMA1))
message_fatal(_("The .lzma format supports only "
"the LZMA1 filter"));
// If we are using the .xz format, make sure that there is no LZMA1
// filter to prevent LZMA_PROG_ERROR.
if (opt_format == FORMAT_XZ)
for (size_t i = 0; i < filters_count; ++i)
if (filters[i].id == LZMA_FILTER_LZMA1)
message_fatal(_("LZMA1 cannot be used "
"with the .xz format"));
// Print the selected filter chain.
message_filters_show(V_DEBUG, filters);
// Get the memory usage. Note that if --format=raw was used,
// we can be decompressing.
const uint64_t memory_limit = hardware_memlimit_get(opt_mode);
uint64_t memory_usage;
if (opt_mode == MODE_COMPRESS) {
#ifdef MYTHREAD_ENABLED
if (opt_format == FORMAT_XZ && hardware_threads_get() > 1) {
mt_options.threads = hardware_threads_get();
mt_options.block_size = opt_block_size;
mt_options.check = check;
memory_usage = lzma_stream_encoder_mt_memusage(
&mt_options);
if (memory_usage != UINT64_MAX)
message(V_DEBUG, _("Using up to %" PRIu32
" threads."),
mt_options.threads);
} else
#endif
{
memory_usage = lzma_raw_encoder_memusage(filters);
}
} else {
memory_usage = lzma_raw_decoder_memusage(filters);
}
if (memory_usage == UINT64_MAX)
message_fatal(_("Unsupported filter chain or filter options"));
// Print memory usage info before possible dictionary
// size auto-adjusting.
message_mem_needed(V_DEBUG, memory_usage);
if (opt_mode == MODE_COMPRESS) {
const uint64_t decmem = lzma_raw_decoder_memusage(filters);
if (decmem != UINT64_MAX)
message(V_DEBUG, _("Decompression will need "
"%s MiB of memory."), uint64_to_str(
round_up_to_mib(decmem), 0));
}
if (memory_usage <= memory_limit)
return;
// If --no-auto-adjust was used or we didn't find LZMA1 or
// LZMA2 as the last filter, give an error immediately.
// --format=raw implies --no-auto-adjust.
if (!opt_auto_adjust || opt_format == FORMAT_RAW)
memlimit_too_small(memory_usage);
assert(opt_mode == MODE_COMPRESS);
#ifdef MYTHREAD_ENABLED
if (opt_format == FORMAT_XZ && mt_options.threads > 1) {
// Try to reduce the number of threads before
// adjusting the compression settings down.
do {
// FIXME? The real single-threaded mode has
// lower memory usage, but it's not comparable
// because it doesn't write the size info
// into Block Headers.
if (--mt_options.threads == 0)
memlimit_too_small(memory_usage);
memory_usage = lzma_stream_encoder_mt_memusage(
&mt_options);
if (memory_usage == UINT64_MAX)
message_bug();
} while (memory_usage > memory_limit);
message(V_WARNING, _("Adjusted the number of threads "
"from %s to %s to not exceed "
"the memory usage limit of %s MiB"),
uint64_to_str(hardware_threads_get(), 0),
uint64_to_str(mt_options.threads, 1),
uint64_to_str(round_up_to_mib(
memory_limit), 2));
}
#endif
if (memory_usage <= memory_limit) {
return;
}
// Look for the last filter if it is LZMA2 or LZMA1, so we can make
// it use less RAM. With other filters we don't know what to do.
size_t i = 0;
while (filters[i].id != LZMA_FILTER_LZMA2
&& filters[i].id != LZMA_FILTER_LZMA1) {
if (filters[i].id == LZMA_VLI_UNKNOWN)
memlimit_too_small(memory_usage);
++i;
}
// Decrease the dictionary size until we meet the memory
// usage limit. First round down to full mebibytes.
lzma_options_lzma *opt = filters[i].options;
const uint32_t orig_dict_size = opt->dict_size;
opt->dict_size &= ~((UINT32_C(1) << 20) - 1);
while (true) {
// If it is below 1 MiB, auto-adjusting failed. We could be
// more sophisticated and scale it down even more, but let's
// see if many complain about this version.
//
// FIXME: Displays the scaled memory usage instead
// of the original.
if (opt->dict_size < (UINT32_C(1) << 20))
memlimit_too_small(memory_usage);
memory_usage = lzma_raw_encoder_memusage(filters);
if (memory_usage == UINT64_MAX)
message_bug();
// Accept it if it is low enough.
if (memory_usage <= memory_limit)
break;
// Otherwise 1 MiB down and try again. I hope this
// isn't too slow method for cases where the original
// dict_size is very big.
opt->dict_size -= UINT32_C(1) << 20;
}
// Tell the user that we decreased the dictionary size.
message(V_WARNING, _("Adjusted LZMA%c dictionary size "
"from %s MiB to %s MiB to not exceed "
"the memory usage limit of %s MiB"),
filters[i].id == LZMA_FILTER_LZMA2
? '2' : '1',
uint64_to_str(orig_dict_size >> 20, 0),
uint64_to_str(opt->dict_size >> 20, 1),
uint64_to_str(round_up_to_mib(memory_limit), 2));
return;
}
static PyObject *
_lzma_is_check_supported_impl(PyModuleDef *module, int check_id)
/*[clinic end generated code: output=bb828e90e00ad96e input=5518297b97b2318f]*/
{
return PyBool_FromLong(lzma_check_is_supported(check_id));
}
Handle<Value> lzmaCheckIsSupported(const Arguments& args) {
HandleScope scope;
Local<Integer> arg = Local<Integer>::Cast(args[0]);
return scope.Close(Boolean::New(lzma_check_is_supported((lzma_check) arg->Value())));
}
lzma_block_buffer_encode(lzma_block *block, lzma_allocator *allocator,
const uint8_t *in, size_t in_size,
uint8_t *out, size_t *out_pos, size_t out_size)
{
size_t check_size;
lzma_ret ret;
size_t i;
// Validate the arguments.
if (block == NULL || (in == NULL && in_size != 0) || out == NULL
|| out_pos == NULL || *out_pos > out_size)
return LZMA_PROG_ERROR;
// The contents of the structure may depend on the version so
// check the version before validating the contents of *block.
if (block->version != 0)
return LZMA_OPTIONS_ERROR;
if ((unsigned int)(block->check) > LZMA_CHECK_ID_MAX
|| block->filters == NULL)
return LZMA_PROG_ERROR;
if (!lzma_check_is_supported(block->check))
return LZMA_UNSUPPORTED_CHECK;
// Size of a Block has to be a multiple of four, so limit the size
// here already. This way we don't need to check it again when adding
// Block Padding.
out_size -= (out_size - *out_pos) & 3;
// Get the size of the Check field.
check_size = lzma_check_size(block->check);
assert(check_size != UINT32_MAX);
// Reserve space for the Check field.
if (out_size - *out_pos <= check_size)
return LZMA_BUF_ERROR;
out_size -= check_size;
// Do the actual compression.
ret = block_encode_normal(block, allocator,
in, in_size, out, out_pos, out_size);
if (ret != LZMA_OK) {
// If the error was something else than output buffer
// becoming full, return the error now.
if (ret != LZMA_BUF_ERROR)
return ret;
// The data was uncompressible (at least with the options
// given to us) or the output buffer was too small. Use the
// uncompressed chunks of LZMA2 to wrap the data into a valid
// Block. If we haven't been given enough output space, even
// this may fail.
return_if_error(block_encode_uncompressed(block, in, in_size,
out, out_pos, out_size));
}
assert(*out_pos <= out_size);
// Block Padding. No buffer overflow here, because we already adjusted
// out_size so that (out_size - out_start) is a multiple of four.
// Thus, if the buffer is full, the loop body can never run.
for (i = (size_t)(block->compressed_size); i & 3; ++i) {
assert(*out_pos < out_size);
out[(*out_pos)++] = 0x00;
}
// If there's no Check field, we are done now.
if (check_size > 0) {
// Calculate the integrity check. We reserved space for
// the Check field earlier so we don't need to check for
// available output space here.
lzma_check_state check;
lzma_check_init(&check, block->check);
lzma_check_update(&check, block->check, in, in_size);
lzma_check_finish(&check, block->check);
memcpy(block->raw_check, check.buffer.u8, check_size);
memcpy(out + *out_pos, check.buffer.u8, check_size);
*out_pos += check_size;
}
return LZMA_OK;
}
lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check,
lzma_allocator *allocator, const uint8_t *in, size_t in_size,
uint8_t *out, size_t *out_pos_ptr, size_t out_size)
{
// Sanity checks
if (filters == NULL || (unsigned int)(check) > LZMA_CHECK_ID_MAX
|| (in == NULL && in_size != 0) || out == NULL
|| out_pos_ptr == NULL || *out_pos_ptr > out_size)
return LZMA_PROG_ERROR;
if (!lzma_check_is_supported(check))
return LZMA_UNSUPPORTED_CHECK;
// Note for the paranoids: Index encoder prevents the Stream from
// getting too big and still being accepted with LZMA_OK, and Block
// encoder catches if the input is too big. So we don't need to
// separately check if the buffers are too big.
// Use a local copy. We update *out_pos_ptr only if everything
// succeeds.
size_t out_pos = *out_pos_ptr;
// Check that there's enough space for both Stream Header and
// Stream Footer.
if (out_size - out_pos <= 2 * LZMA_STREAM_HEADER_SIZE)
return LZMA_BUF_ERROR;
// Reserve space for Stream Footer so we don't need to check for
// available space again before encoding Stream Footer.
out_size -= LZMA_STREAM_HEADER_SIZE;
// Encode the Stream Header.
lzma_stream_flags stream_flags = {
.version = 0,
.check = check,
};
if (lzma_stream_header_encode(&stream_flags, out + out_pos)
!= LZMA_OK)
return LZMA_PROG_ERROR;
out_pos += LZMA_STREAM_HEADER_SIZE;
// Encode a Block but only if there is at least one byte of input.
lzma_block block = {
.version = 0,
.check = check,
.filters = filters,
};
if (in_size > 0)
return_if_error(lzma_block_buffer_encode(&block, allocator,
in, in_size, out, &out_pos, out_size));
// Index
{
// Create an Index. It will have one Record if there was
// at least one byte of input to encode. Otherwise the
// Index will be empty.
lzma_index *i = lzma_index_init(allocator);
if (i == NULL)
return LZMA_MEM_ERROR;
lzma_ret ret = LZMA_OK;
if (in_size > 0)
ret = lzma_index_append(i, allocator,
lzma_block_unpadded_size(&block),
block.uncompressed_size);
// If adding the Record was successful, encode the Index
// and get its size which will be stored into Stream Footer.
if (ret == LZMA_OK) {
ret = lzma_index_buffer_encode(
i, out, &out_pos, out_size);
stream_flags.backward_size = lzma_index_size(i);
}
lzma_index_end(i, allocator);
if (ret != LZMA_OK)
return ret;
}
// Stream Footer. We have already reserved space for this.
if (lzma_stream_footer_encode(&stream_flags, out + out_pos)
!= LZMA_OK)
return LZMA_PROG_ERROR;
out_pos += LZMA_STREAM_HEADER_SIZE;
// Everything went fine, make the new output position available
// to the application.
*out_pos_ptr = out_pos;
return LZMA_OK;
}
static void
parse_real(args_info *args, int argc, char **argv)
{
enum {
OPT_X86 = INT_MIN,
OPT_POWERPC,
OPT_IA64,
OPT_ARM,
OPT_ARMTHUMB,
OPT_SPARC,
OPT_DELTA,
OPT_LZMA1,
OPT_LZMA2,
OPT_SINGLE_STREAM,
OPT_NO_SPARSE,
OPT_FILES,
OPT_FILES0,
OPT_BLOCK_SIZE,
OPT_MEM_COMPRESS,
OPT_MEM_DECOMPRESS,
OPT_NO_ADJUST,
OPT_INFO_MEMORY,
OPT_ROBOT,
};
static const char short_opts[]
= "cC:defF:hHlkM:qQrS:tT:vVz0123456789";
static const struct option long_opts[] = {
// Operation mode
{ "compress", no_argument, NULL, 'z' },
{ "decompress", no_argument, NULL, 'd' },
{ "uncompress", no_argument, NULL, 'd' },
{ "test", no_argument, NULL, 't' },
{ "list", no_argument, NULL, 'l' },
// Operation modifiers
{ "keep", no_argument, NULL, 'k' },
{ "force", no_argument, NULL, 'f' },
{ "stdout", no_argument, NULL, 'c' },
{ "to-stdout", no_argument, NULL, 'c' },
{ "single-stream", no_argument, NULL, OPT_SINGLE_STREAM },
{ "no-sparse", no_argument, NULL, OPT_NO_SPARSE },
{ "suffix", required_argument, NULL, 'S' },
// { "recursive", no_argument, NULL, 'r' }, // TODO
{ "files", optional_argument, NULL, OPT_FILES },
{ "files0", optional_argument, NULL, OPT_FILES0 },
// Basic compression settings
{ "format", required_argument, NULL, 'F' },
{ "check", required_argument, NULL, 'C' },
{ "block-size", required_argument, NULL, OPT_BLOCK_SIZE },
{ "memlimit-compress", required_argument, NULL, OPT_MEM_COMPRESS },
{ "memlimit-decompress", required_argument, NULL, OPT_MEM_DECOMPRESS },
{ "memlimit", required_argument, NULL, 'M' },
{ "memory", required_argument, NULL, 'M' }, // Old alias
{ "no-adjust", no_argument, NULL, OPT_NO_ADJUST },
{ "threads", required_argument, NULL, 'T' },
{ "extreme", no_argument, NULL, 'e' },
{ "fast", no_argument, NULL, '0' },
{ "best", no_argument, NULL, '9' },
// Filters
{ "lzma1", optional_argument, NULL, OPT_LZMA1 },
{ "lzma2", optional_argument, NULL, OPT_LZMA2 },
{ "x86", optional_argument, NULL, OPT_X86 },
{ "powerpc", optional_argument, NULL, OPT_POWERPC },
{ "ia64", optional_argument, NULL, OPT_IA64 },
{ "arm", optional_argument, NULL, OPT_ARM },
{ "armthumb", optional_argument, NULL, OPT_ARMTHUMB },
{ "sparc", optional_argument, NULL, OPT_SPARC },
{ "delta", optional_argument, NULL, OPT_DELTA },
// Other options
{ "quiet", no_argument, NULL, 'q' },
{ "verbose", no_argument, NULL, 'v' },
{ "no-warn", no_argument, NULL, 'Q' },
{ "robot", no_argument, NULL, OPT_ROBOT },
{ "info-memory", no_argument, NULL, OPT_INFO_MEMORY },
{ "help", no_argument, NULL, 'h' },
{ "long-help", no_argument, NULL, 'H' },
{ "version", no_argument, NULL, 'V' },
{ NULL, 0, NULL, 0 }
};
int c;
while ((c = getopt_long(argc, argv, short_opts, long_opts, NULL))
!= -1) {
switch (c) {
// Compression preset (also for decompression if --format=raw)
case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9':
coder_set_preset(c - '0');
break;
// --memlimit-compress
case OPT_MEM_COMPRESS:
parse_memlimit("memlimit-compress",
"memlimit-compress%", optarg,
true, false);
break;
// --memlimit-decompress
case OPT_MEM_DECOMPRESS:
parse_memlimit("memlimit-decompress",
"memlimit-decompress%", optarg,
false, true);
break;
// --memlimit
case 'M':
parse_memlimit("memlimit", "memlimit%", optarg,
true, true);
break;
// --suffix
case 'S':
suffix_set(optarg);
break;
case 'T':
// The max is from src/liblzma/common/common.h.
hardware_threads_set(str_to_uint64("threads",
optarg, 0, 16384));
break;
// --version
case 'V':
// This doesn't return.
message_version();
// --stdout
case 'c':
opt_stdout = true;
break;
// --decompress
case 'd':
opt_mode = MODE_DECOMPRESS;
break;
// --extreme
case 'e':
coder_set_extreme();
break;
// --force
case 'f':
opt_force = true;
break;
// --info-memory
case OPT_INFO_MEMORY:
// This doesn't return.
hardware_memlimit_show();
// --help
case 'h':
// This doesn't return.
message_help(false);
// --long-help
case 'H':
// This doesn't return.
message_help(true);
// --list
case 'l':
opt_mode = MODE_LIST;
break;
// --keep
case 'k':
opt_keep_original = true;
break;
// --quiet
case 'q':
message_verbosity_decrease();
break;
case 'Q':
set_exit_no_warn();
break;
case 't':
opt_mode = MODE_TEST;
break;
// --verbose
case 'v':
message_verbosity_increase();
break;
// --robot
case OPT_ROBOT:
opt_robot = true;
// This is to make sure that floating point numbers
// always have a dot as decimal separator.
setlocale(LC_NUMERIC, "C");
break;
case 'z':
opt_mode = MODE_COMPRESS;
break;
// Filter setup
case OPT_X86:
coder_add_filter(LZMA_FILTER_X86,
options_bcj(optarg));
break;
case OPT_POWERPC:
coder_add_filter(LZMA_FILTER_POWERPC,
options_bcj(optarg));
break;
case OPT_IA64:
coder_add_filter(LZMA_FILTER_IA64,
options_bcj(optarg));
break;
case OPT_ARM:
coder_add_filter(LZMA_FILTER_ARM,
options_bcj(optarg));
break;
case OPT_ARMTHUMB:
coder_add_filter(LZMA_FILTER_ARMTHUMB,
options_bcj(optarg));
break;
case OPT_SPARC:
coder_add_filter(LZMA_FILTER_SPARC,
options_bcj(optarg));
break;
case OPT_DELTA:
coder_add_filter(LZMA_FILTER_DELTA,
options_delta(optarg));
break;
case OPT_LZMA1:
coder_add_filter(LZMA_FILTER_LZMA1,
options_lzma(optarg));
break;
case OPT_LZMA2:
coder_add_filter(LZMA_FILTER_LZMA2,
options_lzma(optarg));
break;
// Other
// --format
case 'F': {
// Just in case, support both "lzma" and "alone" since
// the latter was used for forward compatibility in
// LZMA Utils 4.32.x.
static const struct {
char str[8];
enum format_type format;
} types[] = {
{ "auto", FORMAT_AUTO },
{ "xz", FORMAT_XZ },
{ "lzma", FORMAT_LZMA },
{ "alone", FORMAT_LZMA },
// { "gzip", FORMAT_GZIP },
// { "gz", FORMAT_GZIP },
{ "raw", FORMAT_RAW },
};
size_t i = 0;
while (strcmp(types[i].str, optarg) != 0)
if (++i == ARRAY_SIZE(types))
message_fatal(_("%s: Unknown file "
"format type"),
optarg);
opt_format = types[i].format;
break;
}
// --check
case 'C': {
static const struct {
char str[8];
lzma_check check;
} types[] = {
{ "none", LZMA_CHECK_NONE },
{ "crc32", LZMA_CHECK_CRC32 },
{ "crc64", LZMA_CHECK_CRC64 },
{ "sha256", LZMA_CHECK_SHA256 },
};
size_t i = 0;
while (strcmp(types[i].str, optarg) != 0) {
if (++i == ARRAY_SIZE(types))
message_fatal(_("%s: Unsupported "
"integrity "
"check type"), optarg);
}
// Use a separate check in case we are using different
// liblzma than what was used to compile us.
if (!lzma_check_is_supported(types[i].check))
message_fatal(_("%s: Unsupported integrity "
"check type"), optarg);
coder_set_check(types[i].check);
break;
}
case OPT_BLOCK_SIZE:
opt_block_size = str_to_uint64("block-size", optarg,
0, LZMA_VLI_MAX);
break;
case OPT_SINGLE_STREAM:
opt_single_stream = true;
break;
case OPT_NO_SPARSE:
io_no_sparse();
break;
case OPT_FILES:
args->files_delim = '\n';
// Fall through
case OPT_FILES0:
if (args->files_name != NULL)
message_fatal(_("Only one file can be "
"specified with `--files' "
"or `--files0'."));
if (optarg == NULL) {
args->files_name = (char *)stdin_filename;
args->files_file = stdin;
} else {
args->files_name = optarg;
args->files_file = fopen(optarg,
c == OPT_FILES ? "r" : "rb");
if (args->files_file == NULL)
message_fatal("%s: %s", optarg,
strerror(errno));
}
break;
case OPT_NO_ADJUST:
opt_auto_adjust = false;
break;
default:
message_try_help();
tuklib_exit(E_ERROR, E_ERROR, false);
}
}
return;
}
