/* lzma_open opens the file whose name is the string pointed to
by 'path' and associates a stream with it. The 'mode' argument
is expected to be 'r' or 'w'. Upon successful completion, a
lzmafile pointer will be returned. Upon error, NULL will be returned.*/
void* lzma_open(const char *path, const char* mode)
{
int ret;
/* initialize LZMA stream */
struct lzmafile* lf = malloc(sizeof(struct lzmafile));
lf->fp = fopen(path, mode);
lf->str = lzma_stream_init;
lf->mode = mode[0];
if (mode[0] == 'r') {
#if LZMA_VERSION <= UINT32_C(49990030)
ret = lzma_auto_decoder(&lf->str, NULL, NULL);
#else
ret = lzma_auto_decoder(&lf->str, -1, 0);
#endif
lf->str.avail_in = 0;
} else {
/* I decided to use level 2 encoding */
/* Perhaps this should be user configurable in an environment variable */
ret = LZMA_EASY_ENCODER(&lf->str, 2);
}
if (ret != LZMA_OK) {
fprintf(stderr, "lzma_open error: %d\n", ret);
return NULL;
}
return (void*)lf;
}
static int
lzma_open(struct stream_encoded *stream, int fd)
{
struct lzma_enc_data *data = mem_alloc(sizeof(*data));
int err;
stream->data = NULL;
if (!data) {
return -1;
}
memset(&data->flzma_stream, 0, sizeof(data->flzma_stream));
data->fdread = fd;
data->last_read = 0;
err = lzma_auto_decoder(&data->flzma_stream, ELINKS_LZMA_MEMORY_LIMIT, 0);
if (err != LZMA_OK) {
mem_free(data);
return -1;
}
stream->data = data;
return 0;
}
/*@null@*/
static XZFILE *xzopen_internal(const char *path, const char *mode, int fdno, int xz)
/*@globals fileSystem @*/
/*@modifies fileSystem @*/
{
int level = LZMA_PRESET_DEFAULT;
int encoding = 0;
FILE *fp;
XZFILE *xzfile;
lzma_stream tmp;
lzma_ret ret;
for (; *mode != '\0'; mode++) {
if (*mode == 'w')
encoding = 1;
else if (*mode == 'r')
encoding = 0;
else if (*mode >= '0' && *mode <= '9')
level = (int)(*mode - '0');
}
if (fdno != -1)
fp = fdopen(fdno, encoding ? "w" : "r");
else
fp = fopen(path, encoding ? "w" : "r");
if (!fp)
return NULL;
xzfile = calloc(1, sizeof(*xzfile));
if (!xzfile) {
(void) fclose(fp);
return NULL;
}
xzfile->fp = fp;
xzfile->encoding = encoding;
xzfile->eof = 0;
tmp = (lzma_stream)LZMA_STREAM_INIT;
xzfile->strm = tmp;
if (encoding) {
if (xz) {
ret = lzma_easy_encoder(&xzfile->strm, level, LZMA_CHECK_CRC32);
} else {
lzma_options_lzma options;
(void) lzma_lzma_preset(&options, level);
ret = lzma_alone_encoder(&xzfile->strm, &options);
}
} else {
/* We set the memlimit for decompression to 100MiB which should be
* more than enough to be sufficient for level 9 which requires 65 MiB.
*/
ret = lzma_auto_decoder(&xzfile->strm, 100<<20, 0);
}
if (ret != LZMA_OK) {
(void) fclose(fp);
memset(xzfile, 0, sizeof(*xzfile));
free(xzfile);
return NULL;
}
return xzfile;
}
static LZFILE *lzopen(const char *path, const char *mode, int fd, int isxz)
{
int level = 7;
int encoding = 0;
FILE *fp;
LZFILE *lzfile;
lzma_ret ret;
if (!path && fd < 0)
return 0;
for (; *mode; mode++)
{
if (*mode == 'w')
encoding = 1;
else if (*mode == 'r')
encoding = 0;
else if (*mode >= '1' && *mode <= '9')
level = *mode - '0';
}
if (fd != -1)
fp = fdopen(fd, encoding ? "w" : "r");
else
fp = fopen(path, encoding ? "w" : "r");
if (!fp)
return 0;
lzfile = calloc(1, sizeof(*lzfile));
if (!lzfile)
{
fclose(fp);
return 0;
}
lzfile->file = fp;
lzfile->encoding = encoding;
lzfile->eof = 0;
lzfile->strm = stream_init;
if (encoding)
{
if (isxz)
ret = lzma_easy_encoder(&lzfile->strm, level, LZMA_CHECK_SHA256);
else
ret = setup_alone_encoder(&lzfile->strm, level);
}
else
ret = lzma_auto_decoder(&lzfile->strm, 100 << 20, 0);
if (ret != LZMA_OK)
{
fclose(fp);
free(lzfile);
return 0;
}
return lzfile;
}
static unsigned char *
lzma_decode_buffer(struct stream_encoded *st, unsigned char *data, int len, int *new_len)
{
struct lzma_enc_data *enc_data = (struct lzma_enc_data *) st->data;
lzma_stream *stream = &enc_data->flzma_stream;
unsigned char *buffer = NULL;
int error;
*new_len = 0; /* default, left there if an error occurs */
stream->next_in = data;
stream->avail_in = len;
stream->total_out = 0;
if (lzma_auto_decoder(stream, ELINKS_LZMA_MEMORY_LIMIT, 0) != LZMA_OK)
return NULL;
do {
unsigned char *new_buffer;
size_t size = stream->total_out + MAX_STR_LEN;
new_buffer = mem_realloc(buffer, size);
if (!new_buffer) {
error = LZMA_MEM_ERROR;
break;
}
buffer = new_buffer;
stream->next_out = buffer + stream->total_out;
stream->avail_out = MAX_STR_LEN;
error = lzma_code(stream, LZMA_RUN);
if (error == LZMA_STREAM_END) {
error = LZMA_OK;
break;
}
} while (error == LZMA_OK && stream->avail_in > 0);
if (error == LZMA_STREAM_END) {
lzma_end(stream);
enc_data->after_end = 1;
error = LZMA_OK;
}
if (error == LZMA_OK) {
*new_len = stream->total_out;
return buffer;
} else {
if (buffer) mem_free(buffer);
return NULL;
}
}
static LZFILE *lzopen_internal(const char *path, const char *mode, int fd, int xz)
{
int level = 7; /* Use XZ's default compression level if unspecified */
int encoding = 0;
FILE *fp;
LZFILE *lzfile;
lzma_ret ret;
lzma_stream init_strm = LZMA_STREAM_INIT;
for (; *mode; mode++) {
if (*mode == 'w')
encoding = 1;
else if (*mode == 'r')
encoding = 0;
else if (*mode >= '1' && *mode <= '9')
level = *mode - '0';
}
if (fd != -1)
fp = fdopen(fd, encoding ? "w" : "r");
else
fp = fopen(path, encoding ? "w" : "r");
if (!fp)
return 0;
lzfile = calloc(1, sizeof(*lzfile));
if (!lzfile) {
fclose(fp);
return 0;
}
lzfile->file = fp;
lzfile->encoding = encoding;
lzfile->eof = 0;
lzfile->strm = init_strm;
if (encoding) {
if (xz) {
ret = lzma_easy_encoder(&lzfile->strm, level, LZMA_CHECK_SHA256);
} else {
lzma_options_lzma options;
lzma_lzma_preset(&options, level);
ret = lzma_alone_encoder(&lzfile->strm, &options);
}
} else { /* lzma_easy_decoder_memusage(level) is not ready yet, use hardcoded limit for now */
ret = lzma_auto_decoder(&lzfile->strm, 100<<20, 0);
}
if (ret != LZMA_OK) {
fclose(fp);
free(lzfile);
return 0;
}
return lzfile;
}
void libmaus2::lz::XzDecoder::initDecoder()
{
#if defined(LIBMAUS2_HAVE_LZMA)
#if LZMA_VERSION <= UINT32_C(49990030)
int const ret = lzma_auto_decoder(&lstr, NULL, NULL);
#else
lzma_ret const ret = lzma_auto_decoder(&lstr, UINT64_MAX, 0);
#endif
if ( ret != LZMA_OK )
{
libmaus2::exception::LibMausException lme;
lme.getStream() << "XzDecoder::initDecoder(): lzma_auto_decoder failed: " << ret << std::endl;
lme.finish();
throw lme;
}
#else
libmaus2::exception::LibMausException lme;
lme.getStream() << "XzDecoder::initDecoder(): libmaus2 is compiled without liblzma support" << std::endl;
lme.finish();
throw lme;
#endif
}
int main(int argc, char **argv)
{
int ret;
lzma_stream strm;
size_t write_size;
FILE *file_in = stdin;
FILE *file_out = stdout;
unsigned char *buffer_in = malloc (CHUNKSIZE_IN);
unsigned char *buffer_out = malloc (CHUNKSIZE_OUT);
/* Check the command line arguments. */
if (argc > 1 && 0 == strcmp (argv[1], "--help")) {
printf ("\nLZMAdec - a small LZMA decoder\n\nUsage: %s [--help]\n\nThe compressed data is read from stdin and uncompressed to stdout.\n\n", argv[0]);
return 0;
}
if (buffer_in == NULL || buffer_out == NULL) {
fprintf (stderr, "%s: Not enough memory.\n", argv[0]);
return 5;
}
strm.avail_in = 0;
strm.next_in = NULL;
strm = LZMA_STREAM_INIT_VAR;
lzma_auto_decoder(&strm, 0, 0);
strm.next_out = buffer_out;
strm.avail_out = CHUNKSIZE_OUT;
for (;;) {
if (!strm.avail_in) {
strm.next_in = buffer_in;
strm.avail_in = fread (buffer_in, sizeof (unsigned char),
CHUNKSIZE_IN, file_in);
}
ret = lzma_code(&strm, LZMA_RUN);
if (ret == LZMA_STREAM_END) {
write_size = CHUNKSIZE_OUT - strm.avail_out;
if (write_size != (fwrite (buffer_out, sizeof (unsigned char),
write_size, file_out)))
lzma_end (&strm);
return 0;
}
if (ret != LZMA_OK && ret != LZMA_STREAM_END)
return 1;
}
}
io_t *lzma_open(io_t *parent)
{
io_t *io;
if (!parent)
return NULL;
io = malloc(sizeof(io_t));
io->source = &lzma_source;
io->data = malloc(sizeof(struct lzma_t));
DATA(io)->parent = parent;
memset(&DATA(io)->strm, 0, sizeof(DATA(io)->strm));
DATA(io)->err = ERR_OK;
if (lzma_auto_decoder(&DATA(io)->strm, UINT64_MAX, 0) != LZMA_OK) {
free(io->data);
free(io);
fprintf(stderr, "auto decoder failed\n");
return NULL;
}
return io;
}
static void* decompress_lzma(const void* buf, const int buf_len,
const char* dir_full_path, int* new_buf_len) {
lzma_stream stream = LZMA_STREAM_INIT;
lzma_ret lzrt;
unsigned char* result = NULL;
size_t result_size = 0;
size_t pagesize = 0;
stream.avail_in = buf_len;
stream.next_in = (void*)buf;
lzrt = lzma_auto_decoder(&stream, -1, 0);
if (lzrt != LZMA_OK) {
log_err("Unable to initialize lzma_auto_decoder: %d", lzrt);
goto error_out;
}
pagesize = getpagesize();
result_size = ((buf_len + pagesize - 1) & ~(pagesize - 1));
do {
do {
unsigned char* tmp_result = result;
/* Double the buffer size and realloc */
result_size *= 2;
result = (unsigned char*)realloc(result, result_size * sizeof(unsigned char));
if (result == NULL) {
free(tmp_result);
log_err("Unable to allocate %d bytes to decompress file %s", result_size * sizeof(unsigned char), dir_full_path);
goto error_out;
}
stream.avail_out = result_size / 2;
stream.next_out = &result[stream.total_out];
lzrt = lzma_code(&stream, LZMA_RUN);
log_debug("lzma_code ret = %d", lzrt);
switch(lzrt) {
case LZMA_OK:
case LZMA_STREAM_END:
break;
default:
log_err("Found mem/data error while decompressing xz/lzma stream: %d", lzrt);
goto error_out;
}
} while(stream.avail_out == 0);
} while(lzrt == LZMA_OK);
*new_buf_len = stream.total_out;
if (lzrt == LZMA_STREAM_END) {
lzma_end(&stream);
return result;
}
error_out:
lzma_end(&stream);
*new_buf_len = 0;
if (result) {
free(result);
}
return NULL;
}
std::unique_ptr<IOBuf> LZMA2Codec::doUncompress(const IOBuf* data,
uint64_t uncompressedLength) {
lzma_ret rc;
lzma_stream stream = LZMA_STREAM_INIT;
rc = lzma_auto_decoder(&stream, std::numeric_limits<uint64_t>::max(), 0);
if (rc != LZMA_OK) {
throw std::runtime_error(folly::to<std::string>(
"LZMA2Codec: lzma_auto_decoder error: ", rc));
}
SCOPE_EXIT { lzma_end(&stream); };
// Max 64MiB in one go
constexpr uint32_t maxSingleStepLength = uint32_t(64) << 20; // 64MiB
constexpr uint32_t defaultBufferLength = uint32_t(4) << 20; // 4MiB
folly::io::Cursor cursor(data);
uint64_t actualUncompressedLength;
if (encodeSize()) {
actualUncompressedLength = decodeVarintFromCursor(cursor);
if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH &&
uncompressedLength != actualUncompressedLength) {
throw std::runtime_error("LZMA2Codec: invalid uncompressed length");
}
} else {
actualUncompressedLength = uncompressedLength;
DCHECK_NE(actualUncompressedLength, UNKNOWN_UNCOMPRESSED_LENGTH);
}
auto out = addOutputBuffer(
&stream,
(actualUncompressedLength <= maxSingleStepLength ?
actualUncompressedLength :
defaultBufferLength));
bool streamEnd = false;
auto buf = cursor.peekBytes();
while (!buf.empty()) {
stream.next_in = const_cast<uint8_t*>(buf.data());
stream.avail_in = buf.size();
while (stream.avail_in != 0) {
if (streamEnd) {
throw std::runtime_error(to<std::string>(
"LZMA2Codec: junk after end of data"));
}
streamEnd = doInflate(&stream, out.get(), defaultBufferLength);
}
cursor.skip(buf.size());
buf = cursor.peekBytes();
}
while (!streamEnd) {
streamEnd = doInflate(&stream, out.get(), defaultBufferLength);
}
out->prev()->trimEnd(stream.avail_out);
if (actualUncompressedLength != stream.total_out) {
throw std::runtime_error(to<std::string>(
"LZMA2Codec: invalid uncompressed length"));
}
return out;
}
static int
_lzma_LZMADecompressor___init___impl(Decompressor *self, int format, PyObject *memlimit, PyObject *filters)
/*[clinic end generated code: output=9b119f6f2cc2d7a8 input=458ca6132ef29801]*/
{
const uint32_t decoder_flags = LZMA_TELL_ANY_CHECK | LZMA_TELL_NO_CHECK;
uint64_t memlimit_ = UINT64_MAX;
lzma_ret lzret;
if (memlimit != Py_None) {
if (format == FORMAT_RAW) {
PyErr_SetString(PyExc_ValueError,
"Cannot specify memory limit with FORMAT_RAW");
return -1;
}
memlimit_ = PyLong_AsUnsignedLongLong(memlimit);
if (PyErr_Occurred())
return -1;
}
if (format == FORMAT_RAW && filters == Py_None) {
PyErr_SetString(PyExc_ValueError,
"Must specify filters for FORMAT_RAW");
return -1;
} else if (format != FORMAT_RAW && filters != Py_None) {
PyErr_SetString(PyExc_ValueError,
"Cannot specify filters except with FORMAT_RAW");
return -1;
}
self->alloc.opaque = NULL;
self->alloc.alloc = PyLzma_Malloc;
self->alloc.free = PyLzma_Free;
self->lzs.allocator = &self->alloc;
#ifdef WITH_THREAD
self->lock = PyThread_allocate_lock();
if (self->lock == NULL) {
PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock");
return -1;
}
#endif
self->check = LZMA_CHECK_UNKNOWN;
self->unused_data = PyBytes_FromStringAndSize(NULL, 0);
if (self->unused_data == NULL)
goto error;
switch (format) {
case FORMAT_AUTO:
lzret = lzma_auto_decoder(&self->lzs, memlimit_, decoder_flags);
if (catch_lzma_error(lzret))
break;
return 0;
case FORMAT_XZ:
lzret = lzma_stream_decoder(&self->lzs, memlimit_, decoder_flags);
if (catch_lzma_error(lzret))
break;
return 0;
case FORMAT_ALONE:
self->check = LZMA_CHECK_NONE;
lzret = lzma_alone_decoder(&self->lzs, memlimit_);
if (catch_lzma_error(lzret))
break;
return 0;
case FORMAT_RAW:
self->check = LZMA_CHECK_NONE;
if (Decompressor_init_raw(&self->lzs, filters) == -1)
break;
return 0;
default:
PyErr_Format(PyExc_ValueError,
"Invalid container format: %d", format);
break;
}
error:
Py_CLEAR(self->unused_data);
#ifdef WITH_THREAD
PyThread_free_lock(self->lock);
self->lock = NULL;
#endif
return -1;
}
/**
* zif_file_decompress_lzma:
**/
static gboolean
zif_file_decompress_lzma (const gchar *in, const gchar *out, ZifState *state, GError **error)
{
gboolean ret = FALSE;
gint size;
gint written;
FILE *f_in = NULL;
FILE *f_out = NULL;
guchar in_buf[ZIF_BUFFER_SIZE];
guchar out_buf[ZIF_BUFFER_SIZE];
GCancellable *cancellable;
lzma_ret r;
lzma_stream stream = LZMA_STREAM_INIT;
lzma_stream *strm = &stream;
lzma_action action;
g_return_val_if_fail (in != NULL, FALSE);
g_return_val_if_fail (out != NULL, FALSE);
g_return_val_if_fail (zif_state_valid (state), FALSE);
/* get cancellable */
cancellable = zif_state_get_cancellable (state);
r = lzma_auto_decoder(strm, UINT64_MAX, 0);
if (r == LZMA_MEM_ERROR) {
g_set_error (error,
ZIF_UTILS_ERROR,
ZIF_UTILS_ERROR_FAILED,
"out of memory");
goto out;
}
else if (r != LZMA_OK) {
g_set_error (error,
ZIF_UTILS_ERROR,
ZIF_UTILS_ERROR_FAILED,
"internal error");
goto out;
}
/* open file for reading */
f_in = fopen (in, "rb");
if (f_in == NULL) {
g_set_error (error,
ZIF_UTILS_ERROR,
ZIF_UTILS_ERROR_FAILED_TO_READ,
"cannot open %s for reading", in);
goto out;
}
/* open file for writing */
f_out = fopen (out, "w");
if (f_out == NULL) {
g_set_error (error,
ZIF_UTILS_ERROR,
ZIF_UTILS_ERROR_FAILED_TO_WRITE,
"cannot open %s for writing", out);
goto out;
}
strm->avail_in = 0;
strm->next_out = out_buf;
strm->avail_out = ZIF_BUFFER_SIZE;
action = LZMA_RUN;
/* read in all data in chunks */
while (r == LZMA_OK) {
/* read data */
if (strm->avail_in == 0) {
size = fread (in_buf, 1, ZIF_BUFFER_SIZE, f_in);
/* error */
if (ferror (f_in)) {
g_set_error_literal (error, ZIF_UTILS_ERROR, ZIF_UTILS_ERROR_FAILED_TO_READ,
"failed read");
goto out;
}
if (feof (f_in))
action = LZMA_FINISH;
strm->next_in = in_buf;
strm->avail_in = size;
}
r = lzma_code (strm, action);
/* write data */
if (strm->avail_out == 0 || r != LZMA_OK) {
size = ZIF_BUFFER_SIZE - strm->avail_out;
written = fwrite (out_buf, 1, size, f_out);
if (written != size) {
g_set_error (error, ZIF_UTILS_ERROR, ZIF_UTILS_ERROR_FAILED_TO_WRITE,
"only wrote %i/%i bytes", written, size);
goto out;
}
strm->next_out = out_buf;
strm->avail_out = ZIF_BUFFER_SIZE;
}
/* is cancelled */
ret = !g_cancellable_is_cancelled (cancellable);
if (!ret) {
g_set_error_literal (error, ZIF_UTILS_ERROR, ZIF_UTILS_ERROR_CANCELLED, "cancelled");
goto out;
}
}
/* failed to read */
if (r != LZMA_STREAM_END) {
g_set_error (error, ZIF_UTILS_ERROR, ZIF_UTILS_ERROR_FAILED,
"did not decompress file: %s", in);
goto out;
}
/* success */
ret = TRUE;
out:
lzma_end (strm);
if (f_in != NULL)
fclose (f_in);
if (f_out != NULL)
fclose (f_out);
return ret;
}
