extern void coder_set_compression_settings(void) { // The default check type is CRC64, but fallback to CRC32 // if CRC64 isn't supported by the copy of liblzma we are // using. CRC32 is always supported. if (check_default) { check = LZMA_CHECK_CRC64; if (!lzma_check_is_supported(check)) check = LZMA_CHECK_CRC32; } // Options for LZMA1 or LZMA2 in case we are using a preset. static lzma_options_lzma opt_lzma; if (filters_count == 0) { // We are using a preset. This is not a good idea in raw mode // except when playing around with things. Different versions // of this software may use different options in presets, and // thus make uncompressing the raw data difficult. if (opt_format == FORMAT_RAW) { // The message is shown only if warnings are allowed // but the exit status isn't changed. message(V_WARNING, _("Using a preset in raw mode " "is discouraged.")); message(V_WARNING, _("The exact options of the " "presets may vary between software " "versions.")); } // Get the preset for LZMA1 or LZMA2. if (lzma_lzma_preset(&opt_lzma, preset_number)) message_bug(); // Use LZMA2 except with --format=lzma we use LZMA1. filters[0].id = opt_format == FORMAT_LZMA ? LZMA_FILTER_LZMA1 : LZMA_FILTER_LZMA2; filters[0].options = &opt_lzma; filters_count = 1; } // Terminate the filter options array. filters[filters_count].id = LZMA_VLI_UNKNOWN; // If we are using the .lzma format, allow exactly one filter // which has to be LZMA1. if (opt_format == FORMAT_LZMA && (filters_count != 1 || filters[0].id != LZMA_FILTER_LZMA1)) message_fatal(_("The .lzma format supports only " "the LZMA1 filter")); // If we are using the .xz format, make sure that there is no LZMA1 // filter to prevent LZMA_PROG_ERROR. if (opt_format == FORMAT_XZ) for (size_t i = 0; i < filters_count; ++i) if (filters[i].id == LZMA_FILTER_LZMA1) message_fatal(_("LZMA1 cannot be used " "with the .xz format")); // Print the selected filter chain. message_filters_show(V_DEBUG, filters); // Get the memory usage. Note that if --format=raw was used, // we can be decompressing. const uint64_t memory_limit = hardware_memlimit_get(opt_mode); uint64_t memory_usage; if (opt_mode == MODE_COMPRESS) { #ifdef MYTHREAD_ENABLED if (opt_format == FORMAT_XZ && hardware_threads_get() > 1) { mt_options.threads = hardware_threads_get(); mt_options.block_size = opt_block_size; mt_options.check = check; memory_usage = lzma_stream_encoder_mt_memusage( &mt_options); if (memory_usage != UINT64_MAX) message(V_DEBUG, _("Using up to %" PRIu32 " threads."), mt_options.threads); } else #endif { memory_usage = lzma_raw_encoder_memusage(filters); } } else { memory_usage = lzma_raw_decoder_memusage(filters); } if (memory_usage == UINT64_MAX) message_fatal(_("Unsupported filter chain or filter options")); // Print memory usage info before possible dictionary // size auto-adjusting. message_mem_needed(V_DEBUG, memory_usage); if (opt_mode == MODE_COMPRESS) { const uint64_t decmem = lzma_raw_decoder_memusage(filters); if (decmem != UINT64_MAX) message(V_DEBUG, _("Decompression will need " "%s MiB of memory."), uint64_to_str( round_up_to_mib(decmem), 0)); } if (memory_usage <= memory_limit) return; // If --no-auto-adjust was used or we didn't find LZMA1 or // LZMA2 as the last filter, give an error immediately. // --format=raw implies --no-auto-adjust. if (!opt_auto_adjust || opt_format == FORMAT_RAW) memlimit_too_small(memory_usage); assert(opt_mode == MODE_COMPRESS); #ifdef MYTHREAD_ENABLED if (opt_format == FORMAT_XZ && mt_options.threads > 1) { // Try to reduce the number of threads before // adjusting the compression settings down. do { // FIXME? The real single-threaded mode has // lower memory usage, but it's not comparable // because it doesn't write the size info // into Block Headers. if (--mt_options.threads == 0) memlimit_too_small(memory_usage); memory_usage = lzma_stream_encoder_mt_memusage( &mt_options); if (memory_usage == UINT64_MAX) message_bug(); } while (memory_usage > memory_limit); message(V_WARNING, _("Adjusted the number of threads " "from %s to %s to not exceed " "the memory usage limit of %s MiB"), uint64_to_str(hardware_threads_get(), 0), uint64_to_str(mt_options.threads, 1), uint64_to_str(round_up_to_mib( memory_limit), 2)); } #endif if (memory_usage <= memory_limit) { return; } // Look for the last filter if it is LZMA2 or LZMA1, so we can make // it use less RAM. With other filters we don't know what to do. size_t i = 0; while (filters[i].id != LZMA_FILTER_LZMA2 && filters[i].id != LZMA_FILTER_LZMA1) { if (filters[i].id == LZMA_VLI_UNKNOWN) memlimit_too_small(memory_usage); ++i; } // Decrease the dictionary size until we meet the memory // usage limit. First round down to full mebibytes. lzma_options_lzma *opt = filters[i].options; const uint32_t orig_dict_size = opt->dict_size; opt->dict_size &= ~((UINT32_C(1) << 20) - 1); while (true) { // If it is below 1 MiB, auto-adjusting failed. We could be // more sophisticated and scale it down even more, but let's // see if many complain about this version. // // FIXME: Displays the scaled memory usage instead // of the original. if (opt->dict_size < (UINT32_C(1) << 20)) memlimit_too_small(memory_usage); memory_usage = lzma_raw_encoder_memusage(filters); if (memory_usage == UINT64_MAX) message_bug(); // Accept it if it is low enough. if (memory_usage <= memory_limit) break; // Otherwise 1 MiB down and try again. I hope this // isn't too slow method for cases where the original // dict_size is very big. opt->dict_size -= UINT32_C(1) << 20; } // Tell the user that we decreased the dictionary size. message(V_WARNING, _("Adjusted LZMA%c dictionary size " "from %s MiB to %s MiB to not exceed " "the memory usage limit of %s MiB"), filters[i].id == LZMA_FILTER_LZMA2 ? '2' : '1', uint64_to_str(orig_dict_size >> 20, 0), uint64_to_str(opt->dict_size >> 20, 1), uint64_to_str(round_up_to_mib(memory_limit), 2)); return; }
static PyObject * _lzma_is_check_supported_impl(PyModuleDef *module, int check_id) /*[clinic end generated code: output=bb828e90e00ad96e input=5518297b97b2318f]*/ { return PyBool_FromLong(lzma_check_is_supported(check_id)); }
Handle<Value> lzmaCheckIsSupported(const Arguments& args) { HandleScope scope; Local<Integer> arg = Local<Integer>::Cast(args[0]); return scope.Close(Boolean::New(lzma_check_is_supported((lzma_check) arg->Value()))); }
lzma_block_buffer_encode(lzma_block *block, lzma_allocator *allocator, const uint8_t *in, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) { size_t check_size; lzma_ret ret; size_t i; // Validate the arguments. if (block == NULL || (in == NULL && in_size != 0) || out == NULL || out_pos == NULL || *out_pos > out_size) return LZMA_PROG_ERROR; // The contents of the structure may depend on the version so // check the version before validating the contents of *block. if (block->version != 0) return LZMA_OPTIONS_ERROR; if ((unsigned int)(block->check) > LZMA_CHECK_ID_MAX || block->filters == NULL) return LZMA_PROG_ERROR; if (!lzma_check_is_supported(block->check)) return LZMA_UNSUPPORTED_CHECK; // Size of a Block has to be a multiple of four, so limit the size // here already. This way we don't need to check it again when adding // Block Padding. out_size -= (out_size - *out_pos) & 3; // Get the size of the Check field. check_size = lzma_check_size(block->check); assert(check_size != UINT32_MAX); // Reserve space for the Check field. if (out_size - *out_pos <= check_size) return LZMA_BUF_ERROR; out_size -= check_size; // Do the actual compression. ret = block_encode_normal(block, allocator, in, in_size, out, out_pos, out_size); if (ret != LZMA_OK) { // If the error was something else than output buffer // becoming full, return the error now. if (ret != LZMA_BUF_ERROR) return ret; // The data was uncompressible (at least with the options // given to us) or the output buffer was too small. Use the // uncompressed chunks of LZMA2 to wrap the data into a valid // Block. If we haven't been given enough output space, even // this may fail. return_if_error(block_encode_uncompressed(block, in, in_size, out, out_pos, out_size)); } assert(*out_pos <= out_size); // Block Padding. No buffer overflow here, because we already adjusted // out_size so that (out_size - out_start) is a multiple of four. // Thus, if the buffer is full, the loop body can never run. for (i = (size_t)(block->compressed_size); i & 3; ++i) { assert(*out_pos < out_size); out[(*out_pos)++] = 0x00; } // If there's no Check field, we are done now. if (check_size > 0) { // Calculate the integrity check. We reserved space for // the Check field earlier so we don't need to check for // available output space here. lzma_check_state check; lzma_check_init(&check, block->check); lzma_check_update(&check, block->check, in, in_size); lzma_check_finish(&check, block->check); memcpy(block->raw_check, check.buffer.u8, check_size); memcpy(out + *out_pos, check.buffer.u8, check_size); *out_pos += check_size; } return LZMA_OK; }
lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check, lzma_allocator *allocator, const uint8_t *in, size_t in_size, uint8_t *out, size_t *out_pos_ptr, size_t out_size) { // Sanity checks if (filters == NULL || (unsigned int)(check) > LZMA_CHECK_ID_MAX || (in == NULL && in_size != 0) || out == NULL || out_pos_ptr == NULL || *out_pos_ptr > out_size) return LZMA_PROG_ERROR; if (!lzma_check_is_supported(check)) return LZMA_UNSUPPORTED_CHECK; // Note for the paranoids: Index encoder prevents the Stream from // getting too big and still being accepted with LZMA_OK, and Block // encoder catches if the input is too big. So we don't need to // separately check if the buffers are too big. // Use a local copy. We update *out_pos_ptr only if everything // succeeds. size_t out_pos = *out_pos_ptr; // Check that there's enough space for both Stream Header and // Stream Footer. if (out_size - out_pos <= 2 * LZMA_STREAM_HEADER_SIZE) return LZMA_BUF_ERROR; // Reserve space for Stream Footer so we don't need to check for // available space again before encoding Stream Footer. out_size -= LZMA_STREAM_HEADER_SIZE; // Encode the Stream Header. lzma_stream_flags stream_flags = { .version = 0, .check = check, }; if (lzma_stream_header_encode(&stream_flags, out + out_pos) != LZMA_OK) return LZMA_PROG_ERROR; out_pos += LZMA_STREAM_HEADER_SIZE; // Encode a Block but only if there is at least one byte of input. lzma_block block = { .version = 0, .check = check, .filters = filters, }; if (in_size > 0) return_if_error(lzma_block_buffer_encode(&block, allocator, in, in_size, out, &out_pos, out_size)); // Index { // Create an Index. It will have one Record if there was // at least one byte of input to encode. Otherwise the // Index will be empty. lzma_index *i = lzma_index_init(allocator); if (i == NULL) return LZMA_MEM_ERROR; lzma_ret ret = LZMA_OK; if (in_size > 0) ret = lzma_index_append(i, allocator, lzma_block_unpadded_size(&block), block.uncompressed_size); // If adding the Record was successful, encode the Index // and get its size which will be stored into Stream Footer. if (ret == LZMA_OK) { ret = lzma_index_buffer_encode( i, out, &out_pos, out_size); stream_flags.backward_size = lzma_index_size(i); } lzma_index_end(i, allocator); if (ret != LZMA_OK) return ret; } // Stream Footer. We have already reserved space for this. if (lzma_stream_footer_encode(&stream_flags, out + out_pos) != LZMA_OK) return LZMA_PROG_ERROR; out_pos += LZMA_STREAM_HEADER_SIZE; // Everything went fine, make the new output position available // to the application. *out_pos_ptr = out_pos; return LZMA_OK; }
static void parse_real(args_info *args, int argc, char **argv) { enum { OPT_X86 = INT_MIN, OPT_POWERPC, OPT_IA64, OPT_ARM, OPT_ARMTHUMB, OPT_SPARC, OPT_DELTA, OPT_LZMA1, OPT_LZMA2, OPT_SINGLE_STREAM, OPT_NO_SPARSE, OPT_FILES, OPT_FILES0, OPT_BLOCK_SIZE, OPT_MEM_COMPRESS, OPT_MEM_DECOMPRESS, OPT_NO_ADJUST, OPT_INFO_MEMORY, OPT_ROBOT, }; static const char short_opts[] = "cC:defF:hHlkM:qQrS:tT:vVz0123456789"; static const struct option long_opts[] = { // Operation mode { "compress", no_argument, NULL, 'z' }, { "decompress", no_argument, NULL, 'd' }, { "uncompress", no_argument, NULL, 'd' }, { "test", no_argument, NULL, 't' }, { "list", no_argument, NULL, 'l' }, // Operation modifiers { "keep", no_argument, NULL, 'k' }, { "force", no_argument, NULL, 'f' }, { "stdout", no_argument, NULL, 'c' }, { "to-stdout", no_argument, NULL, 'c' }, { "single-stream", no_argument, NULL, OPT_SINGLE_STREAM }, { "no-sparse", no_argument, NULL, OPT_NO_SPARSE }, { "suffix", required_argument, NULL, 'S' }, // { "recursive", no_argument, NULL, 'r' }, // TODO { "files", optional_argument, NULL, OPT_FILES }, { "files0", optional_argument, NULL, OPT_FILES0 }, // Basic compression settings { "format", required_argument, NULL, 'F' }, { "check", required_argument, NULL, 'C' }, { "block-size", required_argument, NULL, OPT_BLOCK_SIZE }, { "memlimit-compress", required_argument, NULL, OPT_MEM_COMPRESS }, { "memlimit-decompress", required_argument, NULL, OPT_MEM_DECOMPRESS }, { "memlimit", required_argument, NULL, 'M' }, { "memory", required_argument, NULL, 'M' }, // Old alias { "no-adjust", no_argument, NULL, OPT_NO_ADJUST }, { "threads", required_argument, NULL, 'T' }, { "extreme", no_argument, NULL, 'e' }, { "fast", no_argument, NULL, '0' }, { "best", no_argument, NULL, '9' }, // Filters { "lzma1", optional_argument, NULL, OPT_LZMA1 }, { "lzma2", optional_argument, NULL, OPT_LZMA2 }, { "x86", optional_argument, NULL, OPT_X86 }, { "powerpc", optional_argument, NULL, OPT_POWERPC }, { "ia64", optional_argument, NULL, OPT_IA64 }, { "arm", optional_argument, NULL, OPT_ARM }, { "armthumb", optional_argument, NULL, OPT_ARMTHUMB }, { "sparc", optional_argument, NULL, OPT_SPARC }, { "delta", optional_argument, NULL, OPT_DELTA }, // Other options { "quiet", no_argument, NULL, 'q' }, { "verbose", no_argument, NULL, 'v' }, { "no-warn", no_argument, NULL, 'Q' }, { "robot", no_argument, NULL, OPT_ROBOT }, { "info-memory", no_argument, NULL, OPT_INFO_MEMORY }, { "help", no_argument, NULL, 'h' }, { "long-help", no_argument, NULL, 'H' }, { "version", no_argument, NULL, 'V' }, { NULL, 0, NULL, 0 } }; int c; while ((c = getopt_long(argc, argv, short_opts, long_opts, NULL)) != -1) { switch (c) { // Compression preset (also for decompression if --format=raw) case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': coder_set_preset(c - '0'); break; // --memlimit-compress case OPT_MEM_COMPRESS: parse_memlimit("memlimit-compress", "memlimit-compress%", optarg, true, false); break; // --memlimit-decompress case OPT_MEM_DECOMPRESS: parse_memlimit("memlimit-decompress", "memlimit-decompress%", optarg, false, true); break; // --memlimit case 'M': parse_memlimit("memlimit", "memlimit%", optarg, true, true); break; // --suffix case 'S': suffix_set(optarg); break; case 'T': // The max is from src/liblzma/common/common.h. hardware_threads_set(str_to_uint64("threads", optarg, 0, 16384)); break; // --version case 'V': // This doesn't return. message_version(); // --stdout case 'c': opt_stdout = true; break; // --decompress case 'd': opt_mode = MODE_DECOMPRESS; break; // --extreme case 'e': coder_set_extreme(); break; // --force case 'f': opt_force = true; break; // --info-memory case OPT_INFO_MEMORY: // This doesn't return. hardware_memlimit_show(); // --help case 'h': // This doesn't return. message_help(false); // --long-help case 'H': // This doesn't return. message_help(true); // --list case 'l': opt_mode = MODE_LIST; break; // --keep case 'k': opt_keep_original = true; break; // --quiet case 'q': message_verbosity_decrease(); break; case 'Q': set_exit_no_warn(); break; case 't': opt_mode = MODE_TEST; break; // --verbose case 'v': message_verbosity_increase(); break; // --robot case OPT_ROBOT: opt_robot = true; // This is to make sure that floating point numbers // always have a dot as decimal separator. setlocale(LC_NUMERIC, "C"); break; case 'z': opt_mode = MODE_COMPRESS; break; // Filter setup case OPT_X86: coder_add_filter(LZMA_FILTER_X86, options_bcj(optarg)); break; case OPT_POWERPC: coder_add_filter(LZMA_FILTER_POWERPC, options_bcj(optarg)); break; case OPT_IA64: coder_add_filter(LZMA_FILTER_IA64, options_bcj(optarg)); break; case OPT_ARM: coder_add_filter(LZMA_FILTER_ARM, options_bcj(optarg)); break; case OPT_ARMTHUMB: coder_add_filter(LZMA_FILTER_ARMTHUMB, options_bcj(optarg)); break; case OPT_SPARC: coder_add_filter(LZMA_FILTER_SPARC, options_bcj(optarg)); break; case OPT_DELTA: coder_add_filter(LZMA_FILTER_DELTA, options_delta(optarg)); break; case OPT_LZMA1: coder_add_filter(LZMA_FILTER_LZMA1, options_lzma(optarg)); break; case OPT_LZMA2: coder_add_filter(LZMA_FILTER_LZMA2, options_lzma(optarg)); break; // Other // --format case 'F': { // Just in case, support both "lzma" and "alone" since // the latter was used for forward compatibility in // LZMA Utils 4.32.x. static const struct { char str[8]; enum format_type format; } types[] = { { "auto", FORMAT_AUTO }, { "xz", FORMAT_XZ }, { "lzma", FORMAT_LZMA }, { "alone", FORMAT_LZMA }, // { "gzip", FORMAT_GZIP }, // { "gz", FORMAT_GZIP }, { "raw", FORMAT_RAW }, }; size_t i = 0; while (strcmp(types[i].str, optarg) != 0) if (++i == ARRAY_SIZE(types)) message_fatal(_("%s: Unknown file " "format type"), optarg); opt_format = types[i].format; break; } // --check case 'C': { static const struct { char str[8]; lzma_check check; } types[] = { { "none", LZMA_CHECK_NONE }, { "crc32", LZMA_CHECK_CRC32 }, { "crc64", LZMA_CHECK_CRC64 }, { "sha256", LZMA_CHECK_SHA256 }, }; size_t i = 0; while (strcmp(types[i].str, optarg) != 0) { if (++i == ARRAY_SIZE(types)) message_fatal(_("%s: Unsupported " "integrity " "check type"), optarg); } // Use a separate check in case we are using different // liblzma than what was used to compile us. if (!lzma_check_is_supported(types[i].check)) message_fatal(_("%s: Unsupported integrity " "check type"), optarg); coder_set_check(types[i].check); break; } case OPT_BLOCK_SIZE: opt_block_size = str_to_uint64("block-size", optarg, 0, LZMA_VLI_MAX); break; case OPT_SINGLE_STREAM: opt_single_stream = true; break; case OPT_NO_SPARSE: io_no_sparse(); break; case OPT_FILES: args->files_delim = '\n'; // Fall through case OPT_FILES0: if (args->files_name != NULL) message_fatal(_("Only one file can be " "specified with `--files' " "or `--files0'.")); if (optarg == NULL) { args->files_name = (char *)stdin_filename; args->files_file = stdin; } else { args->files_name = optarg; args->files_file = fopen(optarg, c == OPT_FILES ? "r" : "rb"); if (args->files_file == NULL) message_fatal("%s: %s", optarg, strerror(errno)); } break; case OPT_NO_ADJUST: opt_auto_adjust = false; break; default: message_try_help(); tuklib_exit(E_ERROR, E_ERROR, false); } } return; }