static int
LZMAVSetField(TIFF* tif, uint32 tag, va_list ap)
{
static const char module[] = "LZMAVSetField";
LZMAState* sp = LState(tif);
switch (tag) {
case TIFFTAG_LZMAPRESET:
sp->preset = (int) va_arg(ap, int);
lzma_lzma_preset(&sp->opt_lzma, sp->preset);
if (sp->state & LSTATE_INIT_ENCODE) {
lzma_ret ret = lzma_stream_encoder(&sp->stream,
sp->filters,
sp->check);
if (ret != LZMA_OK) {
TIFFErrorExt(tif->tif_clientdata, module,
"Liblzma error: %s",
LZMAStrerror(ret));
}
}
return 1;
default:
return (*sp->vsetparent)(tif, tag, ap);
}
/*NOTREACHED*/
}
void *
mkuz_lzma_init(uint32_t blksz)
{
struct mkuz_lzma *ulp;
if (blksz % USED_BLOCKSIZE != 0) {
errx(1, "cluster size should be multiple of %d",
USED_BLOCKSIZE);
/* Not reached */
}
if (blksz > MAXPHYS) {
errx(1, "cluster size is too large");
/* Not reached */
}
ulp = mkuz_safe_zmalloc(sizeof(struct mkuz_lzma));
/* Init lzma encoder */
ulp->strm = lzma_stream_init;
if (lzma_lzma_preset(&ulp->opt_lzma, LZMA_PRESET_DEFAULT))
errx(1, "Error loading LZMA preset");
ulp->filters[0].id = LZMA_FILTER_LZMA2;
ulp->filters[0].options = &ulp->opt_lzma;
ulp->filters[1].id = LZMA_VLI_UNKNOWN;
ulp->blksz = blksz;
return (void *)ulp;
}
static int
Compressor_init_alone(lzma_stream *lzs, uint32_t preset, PyObject *filterspecs)
{
lzma_ret lzret;
if (filterspecs == Py_None) {
lzma_options_lzma options;
if (lzma_lzma_preset(&options, preset)) {
PyErr_Format(Error, "Invalid compression preset: %d", preset);
return -1;
}
lzret = lzma_alone_encoder(lzs, &options);
} else {
lzma_filter filters[LZMA_FILTERS_MAX + 1];
if (parse_filter_chain_spec(filters, filterspecs) == -1)
return -1;
if (filters[0].id == LZMA_FILTER_LZMA1 &&
filters[1].id == LZMA_VLI_UNKNOWN) {
lzret = lzma_alone_encoder(lzs, filters[0].options);
} else {
PyErr_SetString(PyExc_ValueError,
"Invalid filter chain for FORMAT_ALONE - "
"must be a single LZMA1 filter");
lzret = LZMA_PROG_ERROR;
}
free_filter_chain(filters);
}
if (PyErr_Occurred() || catch_lzma_error(lzret))
return -1;
else
return 0;
}
/*@null@*/
static XZFILE *xzopen_internal(const char *path, const char *mode, int fdno, int xz)
/*@globals fileSystem @*/
/*@modifies fileSystem @*/
{
int level = LZMA_PRESET_DEFAULT;
int encoding = 0;
FILE *fp;
XZFILE *xzfile;
lzma_stream tmp;
lzma_ret ret;
for (; *mode != '\0'; mode++) {
if (*mode == 'w')
encoding = 1;
else if (*mode == 'r')
encoding = 0;
else if (*mode >= '0' && *mode <= '9')
level = (int)(*mode - '0');
}
if (fdno != -1)
fp = fdopen(fdno, encoding ? "w" : "r");
else
fp = fopen(path, encoding ? "w" : "r");
if (!fp)
return NULL;
xzfile = calloc(1, sizeof(*xzfile));
if (!xzfile) {
(void) fclose(fp);
return NULL;
}
xzfile->fp = fp;
xzfile->encoding = encoding;
xzfile->eof = 0;
tmp = (lzma_stream)LZMA_STREAM_INIT;
xzfile->strm = tmp;
if (encoding) {
if (xz) {
ret = lzma_easy_encoder(&xzfile->strm, level, LZMA_CHECK_CRC32);
} else {
lzma_options_lzma options;
(void) lzma_lzma_preset(&options, level);
ret = lzma_alone_encoder(&xzfile->strm, &options);
}
} else {
/* We set the memlimit for decompression to 100MiB which should be
* more than enough to be sufficient for level 9 which requires 65 MiB.
*/
ret = lzma_auto_decoder(&xzfile->strm, 100<<20, 0);
}
if (ret != LZMA_OK) {
(void) fclose(fp);
memset(xzfile, 0, sizeof(*xzfile));
free(xzfile);
return NULL;
}
return xzfile;
}
/*
* Setup callback.
*/
static int
archive_compressor_xz_init(struct archive_write *a)
{
int ret;
struct private_data *state;
struct private_config *config;
if (a->client_opener != NULL) {
ret = (a->client_opener)(&a->archive, a->client_data);
if (ret != ARCHIVE_OK)
return (ret);
}
state = (struct private_data *)malloc(sizeof(*state));
if (state == NULL) {
archive_set_error(&a->archive, ENOMEM,
"Can't allocate data for compression");
return (ARCHIVE_FATAL);
}
memset(state, 0, sizeof(*state));
config = a->compressor.config;
/*
* See comment above. We should set compressed_buffer_size to
* max(bytes_per_block, 65536), but the code can't handle that yet.
*/
state->compressed_buffer_size = a->bytes_per_block;
state->compressed = (unsigned char *)malloc(state->compressed_buffer_size);
if (state->compressed == NULL) {
archive_set_error(&a->archive, ENOMEM,
"Can't allocate data for compression buffer");
free(state);
return (ARCHIVE_FATAL);
}
a->compressor.write = archive_compressor_xz_write;
/* Initialize compression library. */
if (lzma_lzma_preset(&state->lzma_opt, config->compression_level)) {
archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,
"Internal error initializing compression library");
free(state->compressed);
free(state);
}
state->lzmafilters[0].id = LZMA_FILTER_LZMA2;
state->lzmafilters[0].options = &state->lzma_opt;
state->lzmafilters[1].id = LZMA_VLI_UNKNOWN;/* Terminate */
ret = archive_compressor_xz_init_stream(a, state);
if (ret == LZMA_OK) {
a->compressor.data = state;
return (0);
}
/* Library setup failed: clean up. */
free(state->compressed);
free(state);
return (ARCHIVE_FATAL);
}
static void *
parse_filter_spec_lzma(PyObject *spec)
{
static char *optnames[] = {"id", "preset", "dict_size", "lc", "lp",
"pb", "mode", "nice_len", "mf", "depth", NULL};
PyObject *id;
PyObject *preset_obj;
uint32_t preset = LZMA_PRESET_DEFAULT;
lzma_options_lzma *options;
/* First, fill in default values for all the options using a preset.
Then, override the defaults with any values given by the caller. */
preset_obj = PyMapping_GetItemString(spec, "preset");
if (preset_obj == NULL) {
if (PyErr_ExceptionMatches(PyExc_KeyError))
PyErr_Clear();
else
return NULL;
} else {
int ok = uint32_converter(preset_obj, &preset);
Py_DECREF(preset_obj);
if (!ok)
return NULL;
}
options = (lzma_options_lzma *)PyMem_Malloc(sizeof *options);
if (options == NULL)
return PyErr_NoMemory();
memset(options, 0, sizeof *options);
if (lzma_lzma_preset(options, preset)) {
PyMem_Free(options);
PyErr_Format(Error, "Invalid compression preset: %d", preset);
return NULL;
}
if (!PyArg_ParseTupleAndKeywords(empty_tuple, spec,
"|OOO&O&O&O&O&O&O&O&", optnames,
&id, &preset_obj,
uint32_converter, &options->dict_size,
uint32_converter, &options->lc,
uint32_converter, &options->lp,
uint32_converter, &options->pb,
lzma_mode_converter, &options->mode,
uint32_converter, &options->nice_len,
lzma_mf_converter, &options->mf,
uint32_converter, &options->depth)) {
PyErr_SetString(PyExc_ValueError,
"Invalid filter specifier for LZMA filter");
PyMem_Free(options);
options = NULL;
}
return options;
}
size_t fa_decompress_block(fa_compression_t compression, void* out, size_t outSize, const void* in, size_t inSize)
{
switch (compression)
{
default: return 0;
case FA_COMPRESSION_FASTLZ:
{
return fastlz_decompress(in, inSize, out, outSize);
}
break;
#if defined(FA_ZLIB_ENABLE)
case FA_COMPRESSION_DEFLATE:
{
uLongf destLen = outSize;
if (uncompress(out, &destLen, in, inSize) != Z_OK)
{
return 0;
}
return destLen;
}
break;
#endif
#if defined(FA_LZMA_ENABLE)
case FA_COMPRESSION_LZMA2:
{
size_t inPos = 0, outPos = 0;
lzma_filter filters[2];
lzma_options_lzma options;
if (lzma_lzma_preset(&options, 6))
{
return 0;
}
filters[0].id = LZMA_FILTER_LZMA2;
filters[0].options = &options;
filters[1].id = LZMA_VLI_UNKNOWN;
lzma_ret ret = lzma_raw_buffer_decode(filters, NULL, in, &inPos, inSize, out, &outPos, outSize);
if (ret != LZMA_OK)
{
return 0;
}
return outPos;
}
break;
#endif
}
}
static int php_xz_init_encoder(struct php_xz_stream_data_t *self)
{
lzma_stream *strm = &self->strm;
lzma_options_lzma opt_lzma2;
if (lzma_lzma_preset(&opt_lzma2, LZMA_PRESET_DEFAULT)) {
return 0;
}
lzma_filter filters[] = {
{ .id = LZMA_FILTER_LZMA2, .options = &opt_lzma2 },
{ .id = LZMA_VLI_UNKNOWN, .options = NULL },
extern bool
lzma_easy_preset(lzma_options_easy *opt_easy, uint32_t preset)
{
if (lzma_lzma_preset(&opt_easy->opt_lzma, preset))
return true;
opt_easy->filters[0].id = LZMA_FILTER_LZMA2;
opt_easy->filters[0].options = &opt_easy->opt_lzma;
opt_easy->filters[1].id = LZMA_VLI_UNKNOWN;
return false;
}
static LZFILE *lzopen_internal(const char *path, const char *mode, int fd, int xz)
{
int level = 7; /* Use XZ's default compression level if unspecified */
int encoding = 0;
FILE *fp;
LZFILE *lzfile;
lzma_ret ret;
lzma_stream init_strm = LZMA_STREAM_INIT;
for (; *mode; mode++) {
if (*mode == 'w')
encoding = 1;
else if (*mode == 'r')
encoding = 0;
else if (*mode >= '1' && *mode <= '9')
level = *mode - '0';
}
if (fd != -1)
fp = fdopen(fd, encoding ? "w" : "r");
else
fp = fopen(path, encoding ? "w" : "r");
if (!fp)
return 0;
lzfile = calloc(1, sizeof(*lzfile));
if (!lzfile) {
fclose(fp);
return 0;
}
lzfile->file = fp;
lzfile->encoding = encoding;
lzfile->eof = 0;
lzfile->strm = init_strm;
if (encoding) {
if (xz) {
ret = lzma_easy_encoder(&lzfile->strm, level, LZMA_CHECK_SHA256);
} else {
lzma_options_lzma options;
lzma_lzma_preset(&options, level);
ret = lzma_alone_encoder(&lzfile->strm, &options);
}
} else { /* lzma_easy_decoder_memusage(level) is not ready yet, use hardcoded limit for now */
ret = lzma_auto_decoder(&lzfile->strm, 100<<20, 0);
}
if (ret != LZMA_OK) {
fclose(fp);
free(lzfile);
return 0;
}
return lzfile;
}
static void io_do_compress(io_private_t *io_ptr)
{
lzma_stream l = LZMA_STREAM_INIT;
io_ptr->lzma_handle = l;
lzma_filter lzf[2];
lzma_options_lzma lzo;
lzma_lzma_preset(&lzo, LZMA_PRESET_DEFAULT);
lzf[0].id = LZMA_FILTER_LZMA2;
lzf[0].options = &lzo;
lzf[1].id = LZMA_VLI_UNKNOWN;
if (lzma_stream_encoder(&io_ptr->lzma_handle, lzf, LZMA_CHECK_NONE) != LZMA_OK)
return;
io_ptr->lzma_init = true;
return;
}
static void
filter_lzma_init(struct io_lzma *io, lzma_stream *s)
{
uint32_t preset;
lzma_options_lzma options;
lzma_ret ret;
io->status |= DPKG_STREAM_COMPRESS;
preset = io->params->level;
if (io->params->strategy == COMPRESSOR_STRATEGY_EXTREME)
preset |= LZMA_PRESET_EXTREME;
if (lzma_lzma_preset(&options, preset))
filter_lzma_error(io, LZMA_OPTIONS_ERROR);
ret = lzma_alone_encoder(s, &options);
if (ret != LZMA_OK)
filter_lzma_error(io, ret);
}
int
TIFFInitLZMA(TIFF* tif, int scheme)
{
static const char module[] = "TIFFInitLZMA";
LZMAState* sp;
lzma_stream tmp_stream = LZMA_STREAM_INIT;
assert( scheme == COMPRESSION_LZMA );
/*
* Merge codec-specific tag information.
*/
if (!_TIFFMergeFields(tif, lzmaFields, TIFFArrayCount(lzmaFields))) {
TIFFErrorExt(tif->tif_clientdata, module,
"Merging LZMA2 codec-specific tags failed");
return 0;
}
/*
* Allocate state block so tag methods have storage to record values.
*/
tif->tif_data = (uint8*) _TIFFmalloc(sizeof(LZMAState));
if (tif->tif_data == NULL)
goto bad;
sp = LState(tif);
memcpy(&sp->stream, &tmp_stream, sizeof(lzma_stream));
/*
* Override parent get/set field methods.
*/
sp->vgetparent = tif->tif_tagmethods.vgetfield;
tif->tif_tagmethods.vgetfield = LZMAVGetField; /* hook for codec tags */
sp->vsetparent = tif->tif_tagmethods.vsetfield;
tif->tif_tagmethods.vsetfield = LZMAVSetField; /* hook for codec tags */
/* Default values for codec-specific fields */
sp->preset = LZMA_PRESET_DEFAULT; /* default comp. level */
sp->check = LZMA_CHECK_NONE;
sp->state = 0;
/* Data filters. So far we are using delta and LZMA2 filters only. */
sp->opt_delta.type = LZMA_DELTA_TYPE_BYTE;
/*
* The sample size in bytes seems to be reasonable distance for delta
* filter.
*/
sp->opt_delta.dist = (tif->tif_dir.td_bitspersample % 8) ?
1 : tif->tif_dir.td_bitspersample / 8;
sp->filters[0].id = LZMA_FILTER_DELTA;
sp->filters[0].options = &sp->opt_delta;
lzma_lzma_preset(&sp->opt_lzma, sp->preset);
sp->filters[1].id = LZMA_FILTER_LZMA2;
sp->filters[1].options = &sp->opt_lzma;
sp->filters[2].id = LZMA_VLI_UNKNOWN;
sp->filters[2].options = NULL;
/*
* Install codec methods.
*/
tif->tif_fixuptags = LZMAFixupTags;
tif->tif_setupdecode = LZMASetupDecode;
tif->tif_predecode = LZMAPreDecode;
tif->tif_decoderow = LZMADecode;
tif->tif_decodestrip = LZMADecode;
tif->tif_decodetile = LZMADecode;
tif->tif_setupencode = LZMASetupEncode;
tif->tif_preencode = LZMAPreEncode;
tif->tif_postencode = LZMAPostEncode;
tif->tif_encoderow = LZMAEncode;
tif->tif_encodestrip = LZMAEncode;
tif->tif_encodetile = LZMAEncode;
tif->tif_cleanup = LZMACleanup;
/*
* Setup predictor setup.
*/
(void) TIFFPredictorInit(tif);
return 1;
bad:
TIFFErrorExt(tif->tif_clientdata, module,
"No space for LZMA2 state block");
return 0;
}
static inline lzma_ret setup_alone_encoder(lzma_stream *strm, int level)
{
lzma_options_lzma options;
lzma_lzma_preset(&options, level);
return lzma_alone_encoder(strm, &options);
}
/*
* Setup callback.
*/
static int
archive_compressor_xz_open(struct archive_write_filter *f)
{
struct private_data *data = f->data;
int ret;
ret = __archive_write_open_filter(f->next_filter);
if (ret != ARCHIVE_OK)
return (ret);
if (data->compressed == NULL) {
size_t bs = 65536, bpb;
if (f->archive->magic == ARCHIVE_WRITE_MAGIC) {
/* Buffer size should be a multiple number of the of bytes
* per block for performance. */
bpb = archive_write_get_bytes_per_block(f->archive);
if (bpb > bs)
bs = bpb;
else if (bpb != 0)
bs -= bs % bpb;
}
data->compressed_buffer_size = bs;
data->compressed
= (unsigned char *)malloc(data->compressed_buffer_size);
if (data->compressed == NULL) {
archive_set_error(f->archive, ENOMEM,
"Can't allocate data for compression buffer");
return (ARCHIVE_FATAL);
}
}
f->write = archive_compressor_xz_write;
/* Initialize compression library. */
if (f->code == ARCHIVE_FILTER_LZIP) {
const struct option_value *val =
&option_values[data->compression_level];
data->lzma_opt.dict_size = val->dict_size;
data->lzma_opt.preset_dict = NULL;
data->lzma_opt.preset_dict_size = 0;
data->lzma_opt.lc = LZMA_LC_DEFAULT;
data->lzma_opt.lp = LZMA_LP_DEFAULT;
data->lzma_opt.pb = LZMA_PB_DEFAULT;
data->lzma_opt.mode =
data->compression_level<= 2? LZMA_MODE_FAST:LZMA_MODE_NORMAL;
data->lzma_opt.nice_len = val->nice_len;
data->lzma_opt.mf = val->mf;
data->lzma_opt.depth = 0;
data->lzmafilters[0].id = LZMA_FILTER_LZMA1;
data->lzmafilters[0].options = &data->lzma_opt;
data->lzmafilters[1].id = LZMA_VLI_UNKNOWN;/* Terminate */
} else {
if (lzma_lzma_preset(&data->lzma_opt, data->compression_level)) {
archive_set_error(f->archive, ARCHIVE_ERRNO_MISC,
"Internal error initializing compression library");
}
data->lzmafilters[0].id = LZMA_FILTER_LZMA2;
data->lzmafilters[0].options = &data->lzma_opt;
data->lzmafilters[1].id = LZMA_VLI_UNKNOWN;/* Terminate */
}
ret = archive_compressor_xz_init_stream(f, data);
if (ret == LZMA_OK) {
f->data = data;
return (0);
}
return (ARCHIVE_FATAL);
}
extern void
coder_set_compression_settings(void)
{
// The default check type is CRC64, but fallback to CRC32
// if CRC64 isn't supported by the copy of liblzma we are
// using. CRC32 is always supported.
if (check_default) {
check = LZMA_CHECK_CRC64;
if (!lzma_check_is_supported(check))
check = LZMA_CHECK_CRC32;
}
// Options for LZMA1 or LZMA2 in case we are using a preset.
static lzma_options_lzma opt_lzma;
if (filters_count == 0) {
// We are using a preset. This is not a good idea in raw mode
// except when playing around with things. Different versions
// of this software may use different options in presets, and
// thus make uncompressing the raw data difficult.
if (opt_format == FORMAT_RAW) {
// The message is shown only if warnings are allowed
// but the exit status isn't changed.
message(V_WARNING, _("Using a preset in raw mode "
"is discouraged."));
message(V_WARNING, _("The exact options of the "
"presets may vary between software "
"versions."));
}
// Get the preset for LZMA1 or LZMA2.
if (lzma_lzma_preset(&opt_lzma, preset_number))
message_bug();
// Use LZMA2 except with --format=lzma we use LZMA1.
filters[0].id = opt_format == FORMAT_LZMA
? LZMA_FILTER_LZMA1 : LZMA_FILTER_LZMA2;
filters[0].options = &opt_lzma;
filters_count = 1;
}
// Terminate the filter options array.
filters[filters_count].id = LZMA_VLI_UNKNOWN;
// If we are using the .lzma format, allow exactly one filter
// which has to be LZMA1.
if (opt_format == FORMAT_LZMA && (filters_count != 1
|| filters[0].id != LZMA_FILTER_LZMA1))
message_fatal(_("The .lzma format supports only "
"the LZMA1 filter"));
// If we are using the .xz format, make sure that there is no LZMA1
// filter to prevent LZMA_PROG_ERROR.
if (opt_format == FORMAT_XZ)
for (size_t i = 0; i < filters_count; ++i)
if (filters[i].id == LZMA_FILTER_LZMA1)
message_fatal(_("LZMA1 cannot be used "
"with the .xz format"));
// Print the selected filter chain.
message_filters_show(V_DEBUG, filters);
// Get the memory usage. Note that if --format=raw was used,
// we can be decompressing.
const uint64_t memory_limit = hardware_memlimit_get(opt_mode);
uint64_t memory_usage;
if (opt_mode == MODE_COMPRESS) {
#ifdef MYTHREAD_ENABLED
if (opt_format == FORMAT_XZ && hardware_threads_get() > 1) {
mt_options.threads = hardware_threads_get();
mt_options.block_size = opt_block_size;
mt_options.check = check;
memory_usage = lzma_stream_encoder_mt_memusage(
&mt_options);
if (memory_usage != UINT64_MAX)
message(V_DEBUG, _("Using up to %" PRIu32
" threads."),
mt_options.threads);
} else
#endif
{
memory_usage = lzma_raw_encoder_memusage(filters);
}
} else {
memory_usage = lzma_raw_decoder_memusage(filters);
}
if (memory_usage == UINT64_MAX)
message_fatal(_("Unsupported filter chain or filter options"));
// Print memory usage info before possible dictionary
// size auto-adjusting.
message_mem_needed(V_DEBUG, memory_usage);
if (opt_mode == MODE_COMPRESS) {
const uint64_t decmem = lzma_raw_decoder_memusage(filters);
if (decmem != UINT64_MAX)
message(V_DEBUG, _("Decompression will need "
"%s MiB of memory."), uint64_to_str(
round_up_to_mib(decmem), 0));
}
if (memory_usage <= memory_limit)
return;
// If --no-auto-adjust was used or we didn't find LZMA1 or
// LZMA2 as the last filter, give an error immediately.
// --format=raw implies --no-auto-adjust.
if (!opt_auto_adjust || opt_format == FORMAT_RAW)
memlimit_too_small(memory_usage);
assert(opt_mode == MODE_COMPRESS);
#ifdef MYTHREAD_ENABLED
if (opt_format == FORMAT_XZ && mt_options.threads > 1) {
// Try to reduce the number of threads before
// adjusting the compression settings down.
do {
// FIXME? The real single-threaded mode has
// lower memory usage, but it's not comparable
// because it doesn't write the size info
// into Block Headers.
if (--mt_options.threads == 0)
memlimit_too_small(memory_usage);
memory_usage = lzma_stream_encoder_mt_memusage(
&mt_options);
if (memory_usage == UINT64_MAX)
message_bug();
} while (memory_usage > memory_limit);
message(V_WARNING, _("Adjusted the number of threads "
"from %s to %s to not exceed "
"the memory usage limit of %s MiB"),
uint64_to_str(hardware_threads_get(), 0),
uint64_to_str(mt_options.threads, 1),
uint64_to_str(round_up_to_mib(
memory_limit), 2));
}
#endif
if (memory_usage <= memory_limit) {
return;
}
// Look for the last filter if it is LZMA2 or LZMA1, so we can make
// it use less RAM. With other filters we don't know what to do.
size_t i = 0;
while (filters[i].id != LZMA_FILTER_LZMA2
&& filters[i].id != LZMA_FILTER_LZMA1) {
if (filters[i].id == LZMA_VLI_UNKNOWN)
memlimit_too_small(memory_usage);
++i;
}
// Decrease the dictionary size until we meet the memory
// usage limit. First round down to full mebibytes.
lzma_options_lzma *opt = filters[i].options;
const uint32_t orig_dict_size = opt->dict_size;
opt->dict_size &= ~((UINT32_C(1) << 20) - 1);
while (true) {
// If it is below 1 MiB, auto-adjusting failed. We could be
// more sophisticated and scale it down even more, but let's
// see if many complain about this version.
//
// FIXME: Displays the scaled memory usage instead
// of the original.
if (opt->dict_size < (UINT32_C(1) << 20))
memlimit_too_small(memory_usage);
memory_usage = lzma_raw_encoder_memusage(filters);
if (memory_usage == UINT64_MAX)
message_bug();
// Accept it if it is low enough.
if (memory_usage <= memory_limit)
break;
// Otherwise 1 MiB down and try again. I hope this
// isn't too slow method for cases where the original
// dict_size is very big.
opt->dict_size -= UINT32_C(1) << 20;
}
// Tell the user that we decreased the dictionary size.
message(V_WARNING, _("Adjusted LZMA%c dictionary size "
"from %s MiB to %s MiB to not exceed "
"the memory usage limit of %s MiB"),
filters[i].id == LZMA_FILTER_LZMA2
? '2' : '1',
uint64_to_str(orig_dict_size >> 20, 0),
uint64_to_str(opt->dict_size >> 20, 1),
uint64_to_str(round_up_to_mib(memory_limit), 2));
return;
}
int main( int argc, char **argv ) {
int i;
uint64_t p, threads, chunk_size;
uint8_t *m;
struct stat s;
ssize_t rd, ts = 0;
size_t page_size;
struct sigaction new_action, old_action;
struct utimbuf u;
lzma_filter filters[LZMA_FILTERS_MAX + 1];
lzma_options_lzma lzma_options;
xzcmd_max = sysconf(_SC_ARG_MAX);
page_size = sysconf(_SC_PAGE_SIZE);
xzcmd = malloc(xzcmd_max);
snprintf(xzcmd, xzcmd_max, XZ_BINARY);
parse_args(argc, argv);
lzma_lzma_preset(&lzma_options, opt_complevel);
filters[0].id = LZMA_FILTER_LZMA2;
filters[0].options = &lzma_options;
filters[1].id = LZMA_VLI_UNKNOWN;
for (i=0; i<files; i++) {
int std_in = file[i][0] == '-' && file[i][1] == '\0';
#ifdef _OPENMP
threads = omp_get_max_threads();
#else
threads = 1;
#endif
if ( (rd=strlen(file[i])) >= 3 && !strncmp(&file[i][rd-3], ".xz", 3) ) {
if (opt_verbose) {
error(EXIT_FAILURE, 0, "ignoring '%s', it seems to be already compressed", file[i]);
}
continue;
}
if ( !std_in ) {
if ( stat(file[i], &s)) {
error(EXIT_FAILURE, errno, "can't stat '%s'", file[i]);
}
}
chunk_size = opt_context_size * lzma_options.dict_size;
chunk_size = (chunk_size + page_size)&~(page_size-1);
if ( opt_verbose ) {
fprintf(stderr, "context size per thread: %"PRIu64" B\n", chunk_size);
}
if ( opt_threads && (threads > opt_threads || opt_force) ) {
threads = opt_threads;
}
fo = stdout;
if ( std_in ) {
fi = stdin;
} else {
if ( !(fi=fopen(file[i], "rb")) ) {
error(EXIT_FAILURE, errno, "can't open '%s' for reading", file[i]);
}
if ( !opt_stdout ) {
snprintf(str, sizeof(str), "%s.xz", file[i]);
if ( !(fo=fopen(str, "wb")) ) {
error(EXIT_FAILURE, errno, "error creating target archive '%s'", str);
}
}
}
if ( opt_verbose ) {
if ( fo != stdout ) {
fprintf(stderr, "%s -> %"PRIu64"/%"PRIu64" thread%c: [", file[i], threads, (s.st_size+chunk_size-1)/chunk_size, threads != 1 ? 's' : ' ');
} else {
fprintf(stderr, "%"PRIu64" thread%c: [", threads, threads != 1 ? 's' : ' ');
}
fflush(stderr);
}
m = malloc(threads*chunk_size);
new_action.sa_handler = term_handler;
sigemptyset (&new_action.sa_mask);
new_action.sa_flags = 0;
sigaction(SIGINT, NULL, &old_action);
if (old_action.sa_handler != SIG_IGN) sigaction(SIGINT, &new_action, NULL);
sigaction(SIGHUP, NULL, &old_action);
if (old_action.sa_handler != SIG_IGN) sigaction(SIGHUP, &new_action, NULL);
sigaction(SIGTERM, NULL, &old_action);
if (old_action.sa_handler != SIG_IGN) sigaction(SIGTERM, &new_action, NULL);
ftemp = malloc(threads*sizeof(ftemp[0]));
while ( !feof(fi) ) {
size_t actrd;
for (p=0; p<threads; p++) {
ftemp[p] = tmpfile();
}
for ( actrd=rd=0; !feof(fi) && !ferror(fi) && (uint64_t)rd < threads*chunk_size; rd += actrd) {
actrd = fread(&m[rd], 1, threads*chunk_size-actrd, fi);
}
if (ferror(fi)) {
error(EXIT_FAILURE, errno, "error in reading input");
}
#pragma omp parallel for private(p) num_threads(threads)
for ( p=0; p<(rd+chunk_size-1)/chunk_size; p++ ) {
off_t pt, len = rd-p*chunk_size >= chunk_size ? chunk_size : rd-p*chunk_size;
uint8_t *mo;
lzma_stream strm = LZMA_STREAM_INIT;
lzma_ret ret;
mo = malloc(BUFFSIZE);
if ( lzma_stream_encoder(&strm, filters, LZMA_CHECK_CRC64) != LZMA_OK ) {
error(EXIT_FAILURE, errno, "unable to initialize LZMA encoder");
}
for (pt=0; pt<len; pt+=BUFFSIZE) {
strm.next_in = &m[p*chunk_size+pt];
strm.avail_in = len-pt >= BUFFSIZE ? BUFFSIZE : len-pt;
strm.next_out = mo;
strm.avail_out = BUFFSIZE;
do {
ret = lzma_code(&strm, LZMA_RUN);
if ( ret != LZMA_OK ) {
error(EXIT_FAILURE, 0, "error in LZMA_RUN");
}
if ( BUFFSIZE - strm.avail_out > 0 ) {
if ( !fwrite(mo, 1, BUFFSIZE - strm.avail_out, ftemp[p]) ) {
error(EXIT_FAILURE, errno, "writing to temp file failed");
}
strm.next_out = mo;
strm.avail_out = BUFFSIZE;
}
} while ( strm.avail_in );
}
strm.next_out = mo;
strm.avail_out = BUFFSIZE;
do {
ret = lzma_code(&strm, LZMA_FINISH);
if ( ret != LZMA_OK && ret != LZMA_STREAM_END ) {
error(EXIT_FAILURE, 0, "error in LZMA_FINISH");
}
if ( BUFFSIZE - strm.avail_out > 0 ) {
if ( !fwrite(mo, 1, BUFFSIZE - strm.avail_out, ftemp[p]) ) {
error(EXIT_FAILURE, errno, "writing to temp file failed");
}
strm.next_out = mo;
strm.avail_out = BUFFSIZE;
}
} while ( ret == LZMA_OK );
lzma_end(&strm);
free(mo);
if ( opt_verbose ) {
fprintf(stderr, "%"PRIu64" ", p);
fflush(stderr);
}
}
for ( p=0; p<threads; p++ ) {
rewind(ftemp[p]);
while ( (rd=fread(buf, 1, sizeof(buf), ftemp[p])) > 0 ) {
if ( fwrite(buf, 1, rd, fo) != (size_t)rd ) {
error(0, errno, "writing to archive failed");
if ( fo != stdout && unlink(str) ) {
error(0, errno, "error deleting corrupted target archive %s", str);
}
exit(EXIT_FAILURE);
} else ts += rd;
}
if (rd < 0) {
error(0, errno, "reading from temporary file failed");
if ( fo != stdout && unlink(str) ) {
error(0, errno, "error deleting corrupted target archive %s", str);
}
exit(EXIT_FAILURE);
}
if ( close_stream(ftemp[p]) ) {
error(0, errno, "I/O error in temp file");
}
}
}
if ( fi != stdin && close_stream(fi) ) {
error(0, errno, "I/O error in input file");
}
if ( opt_verbose ) {
fprintf(stderr, "] ");
}
free(ftemp);
if ( fo != stdout ) {
if ( close_stream(fo) ) {
error(0, errno, "I/O error in target archive");
}
} else return 0;
if ( chmod(str, s.st_mode) ) {
error(0, errno, "warning: unable to change archive permissions");
}
u.actime = s.st_atime;
u.modtime = s.st_mtime;
if ( utime(str, &u) ) {
error(0, errno, "warning: unable to change archive timestamp");
}
sigaction(SIGINT, &old_action, NULL);
sigaction(SIGHUP, &old_action, NULL);
sigaction(SIGTERM, &old_action, NULL);
if ( opt_verbose ) {
fprintf(stderr, "%"PRIu64" -> %zd %3.3f%%\n", s.st_size, ts, ts*100./s.st_size);
}
if ( !opt_keep && unlink(file[i]) ) {
error(0, errno, "error deleting input file %s", file[i]);
}
}
return 0;
}
