static SRes Encode(ISeqOutStream *outStream, ISeqInStream *inStream, UInt64 fileSize, ICompressProgress *pg, int lvl)
{
CLzmaEncHandle enc;
SRes res;
CLzmaEncProps props;
enc = LzmaEnc_Create(&g_Alloc);
if (enc == 0)
return SZ_ERROR_MEM;
LzmaEncProps_Init(&props);
props.level = lvl;
res = LzmaEnc_SetProps(enc, &props);
if(res == SZ_OK) {
Byte header[LZMA_PROPS_SIZE + 8];
size_t headerSize = LZMA_PROPS_SIZE;
int i;
res = LzmaEnc_WriteProperties(enc, header, &headerSize);
for(i = 0; i < 8; i++)
header[headerSize++] = (Byte)(fileSize >> (8 * i));
if(outStream->Write(outStream, header, headerSize) != headerSize)
res = SZ_ERROR_WRITE;
else {
if(res == SZ_OK)
res = LzmaEnc_Encode(enc, outStream, inStream, pg, &g_Alloc, &g_Alloc);
}
}
void CompressInc(std::vector<unsigned char> &outBuf, const std::vector<unsigned char> &inBuf)
{
CLzmaEncHandle enc = LzmaEnc_Create(&g_Alloc);
assert(enc);
CLzmaEncProps props;
LzmaEncProps_Init(&props);
//props.writeEndMark = 1; // 0 or 1
SRes res = LzmaEnc_SetProps(enc, &props);
assert(res == SZ_OK);
unsigned char tempprops[5];
unsigned propsSize = LZMA_PROPS_SIZE;
//outBuf.resize(propsSize); //fill with zero, which cost me a week to check the mistake.
res = LzmaEnc_WriteProperties(enc, tempprops, &propsSize);
assert(res == SZ_OK && propsSize == LZMA_PROPS_SIZE);
VectorInStream inStream = { &VectorInStream_Read, &inBuf, 0 };
VectorOutStream outStream = { &VectorOutStream_Write, &outBuf };
res = LzmaEnc_Encode(enc,
(ISeqOutStream*)&outStream, (ISeqInStream*)&inStream,
0, &g_Alloc, &g_Alloc);
assert(res == SZ_OK);
UInt64 resLen = inBuf.size();
//printf("uint64 resLen : %" PRIu64 "\n", resLen); //print uint64
Byte header[8];
for (int i = 0; i < 8; i++)
header[i] = (Byte)(resLen >> (8 * i));
FILE *file = fopen("data3.dat", "wb+");
fwrite(tempprops, 1, propsSize, file);
fwrite(&header[0], 1, 8, file);
fwrite(&outBuf[0], 1, outBuf.size(), file);
fclose(file);
LzmaEnc_Destroy(enc, &g_Alloc, &g_Alloc);
}
STDMETHODIMP CEncoder::SetCoderProperties(const PROPID *propIDs,
const PROPVARIANT *coderProps, UInt32 numProps)
{
CLzmaEncProps props;
LzmaEncProps_Init(&props);
for (UInt32 i = 0; i < numProps; i++)
{
const PROPVARIANT &prop = coderProps[i];
switch (propIDs[i])
{
case NCoderPropID::kNumFastBytes:
if (prop.vt != VT_UI4) return E_INVALIDARG; props.fb = prop.ulVal; break;
case NCoderPropID::kMatchFinderCycles:
if (prop.vt != VT_UI4) return E_INVALIDARG; props.mc = prop.ulVal; break;
case NCoderPropID::kAlgorithm:
if (prop.vt != VT_UI4) return E_INVALIDARG; props.algo = prop.ulVal; break;
case NCoderPropID::kDictionarySize:
if (prop.vt != VT_UI4) return E_INVALIDARG; props.dictSize = prop.ulVal; break;
case NCoderPropID::kPosStateBits:
if (prop.vt != VT_UI4) return E_INVALIDARG; props.pb = prop.ulVal; break;
case NCoderPropID::kLitPosBits:
if (prop.vt != VT_UI4) return E_INVALIDARG; props.lp = prop.ulVal; break;
case NCoderPropID::kLitContextBits:
if (prop.vt != VT_UI4) return E_INVALIDARG; props.lc = prop.ulVal; break;
case NCoderPropID::kNumThreads:
if (prop.vt != VT_UI4) return E_INVALIDARG; props.numThreads = prop.ulVal; break;
case NCoderPropID::kMultiThread:
if (prop.vt != VT_BOOL) return E_INVALIDARG; props.numThreads = ((prop.boolVal == VARIANT_TRUE) ? 2 : 1); break;
case NCoderPropID::kEndMarker:
if (prop.vt != VT_BOOL) return E_INVALIDARG; props.writeEndMark = (prop.boolVal == VARIANT_TRUE); break;
case NCoderPropID::kMatchFinder:
if (prop.vt != VT_BSTR) return E_INVALIDARG;
if (!ParseMatchFinder(prop.bstrVal, &props.btMode, &props.numHashBytes /* , &_matchFinderBase.skipModeBits */))
return E_INVALIDARG; break;
default:
return E_INVALIDARG;
}
}
return SResToHRESULT(LzmaEnc_SetProps(_encoder, &props));
}
SRes MyLzmaEncode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen,
const CLzmaEncProps *props, Byte *propsEncoded, SizeT *propsSize, int writeEndMark,
ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig)
{
CLzmaEncHandle *p = (CLzmaEncHandle *)LzmaEnc_Create(alloc);
SRes res;
if (p == 0)
return SZ_ERROR_MEM;
res = LzmaEnc_SetProps(p, props);
if (res == SZ_OK)
{
res = LzmaEnc_WriteProperties(p, propsEncoded, propsSize);
if (res == SZ_OK)
res = LzmaEnc_MemEncode(p, dest, destLen, src, srcLen,
writeEndMark, progress, alloc, allocBig);
}
LzmaEnc_Destroy(p, alloc, allocBig);
return res;
}
STATIC int INIT lzma_alloc_workspace(CLzmaEncProps *props)
{
if ((p = (CLzmaEncHandle *)LzmaEnc_Create(&lzma_alloc)) == NULL)
{
PRINT_ERROR("Failed to allocate lzma deflate workspace\n");
return -ENOMEM;
}
if (LzmaEnc_SetProps(p, props) != SZ_OK)
{
lzma_free_workspace();
return -1;
}
if (LzmaEnc_WriteProperties(p, propsEncoded, &propsSize) != SZ_OK)
{
lzma_free_workspace();
return -1;
}
return 0;
}
STDMETHODIMP CEncoder::SetCoderProperties(const PROPID *propIDs,
const PROPVARIANT *coderProps, UInt32 numProps)
{
CLzmaEncProps props;
LzmaEncProps_Init(&props);
for (UInt32 i = 0; i < numProps; i++)
{
const PROPVARIANT &prop = coderProps[i];
PROPID propID = propIDs[i];
switch (propID)
{
case NCoderPropID::kEndMarker:
if (prop.vt != VT_BOOL) return E_INVALIDARG; props.writeEndMark = (prop.boolVal == VARIANT_TRUE); break;
case NCoderPropID::kNumThreads:
if (prop.vt != VT_UI4) return E_INVALIDARG; props.numThreads = prop.ulVal; break;
default:
RINOK(SetLzmaProp(propID, prop, props));
}
}
return SResToHRESULT(LzmaEnc_SetProps(_encoder, &props));
}
static SRes Encode(FILE *inFile, FILE *outFile, char *rs)
{
CLzmaEncHandle enc;
SRes res;
CFileSeqInStream inStream;
CFileSeqOutStream outStream;
CLzmaEncProps props;
enc = LzmaEnc_Create(&g_Alloc);
if (enc == 0)
return SZ_ERROR_MEM;
inStream.funcTable.Read = MyRead;
inStream.file = inFile;
outStream.funcTable.Write = MyWrite;
outStream.file = outFile;
LzmaEncProps_Init(&props);
res = LzmaEnc_SetProps(enc, &props);
if (res == SZ_OK)
{
Byte header[LZMA_PROPS_SIZE + 8];
size_t headerSize = LZMA_PROPS_SIZE;
UInt64 fileSize;
int i;
res = LzmaEnc_WriteProperties(enc, header, &headerSize);
fileSize = MyGetFileLength(inFile);
for (i = 0; i < 8; i++)
header[headerSize++] = (Byte)(fileSize >> (8 * i));
if (!MyWriteFileAndCheck(outFile, header, headerSize))
return PrintError(rs, "writing error");
if (res == SZ_OK)
res = LzmaEnc_Encode(enc, &outStream.funcTable, &inStream.funcTable,
NULL, &g_Alloc, &g_Alloc);
}
inline TLzmaCompressBase(size_t level)
: H_(LzmaEnc_Create(Alloc()))
{
if (!H_) {
ythrow yexception() << "can not init lzma engine";
}
LzmaEncProps_Init(&Props_);
Props_.level = level;
Props_.dictSize = 0;
Props_.lc = -1;
Props_.lp = -1;
Props_.pb = -1;
Props_.fb = -1;
Props_.numThreads = -1;
Props_.writeEndMark = 1;
Check(LzmaEnc_SetProps(H_, &Props_));
size_t bufLen = sizeof(PropsBuf_);
Zero(PropsBuf_);
Check(LzmaEnc_WriteProperties(H_, PropsBuf_, &bufLen));
}
size32_t compress(const void* input, size32_t inlength, void* output)
{ // returns (size32_t)-1 if cannot compress
if (!enc) {
enc = LzmaEnc_Create(&g_Alloc);
if (enc == 0)
throw MakeStringException(-1,"LzmaEnc_Create failed");
LzmaEncProps_Init(&props);
if (LzmaEnc_SetProps(enc, &props)!=SZ_OK)
throw MakeStringException(-1,"LzmaEnc_SetProps failed");
}
if (inlength+LZMA_PROPS_SIZE+sizeof(size32_t)<1024)
return (size32_t)-1; // don't compress less than 1K
SizeT propsize = LZMA_PROPS_SIZE;
LzmaEnc_WriteProperties(enc, (byte *)output+sizeof(size32_t), &propsize);
*(size32_t *)output = (size32_t)propsize;
SizeT reslen = inlength-1-propsize-sizeof(size32_t);
SRes res = LzmaEnc_MemEncode(enc, (byte *)output+propsize+sizeof(size32_t), &reslen, (const byte *)input, inlength, true, NULL, &g_Alloc, &g_Alloc);
if (res==SZ_ERROR_OUTPUT_EOF)
return (size32_t)-1;
if (res!=SZ_OK)
throw MakeStringException(-1,"LzmaEnc_MemEncode failed(%d)",(int)res);
return reslen+propsize+sizeof(size32_t);
}
static int
pylzma_compfile_init(CCompressionFileObject *self, PyObject *args, PyObject *kwargs)
{
PyObject *inFile;
CLzmaEncProps props;
Byte header[LZMA_PROPS_SIZE];
size_t headerSize = LZMA_PROPS_SIZE;
int result = -1;
// possible keywords for this function
static char *kwlist[] = {"infile", "dictionary", "fastBytes", "literalContextBits",
"literalPosBits", "posBits", "algorithm", "eos", "multithreading", "matchfinder", NULL};
int dictionary = 23; // [0,28], default 23 (8MB)
int fastBytes = 128; // [5,255], default 128
int literalContextBits = 3; // [0,8], default 3
int literalPosBits = 0; // [0,4], default 0
int posBits = 2; // [0,4], default 2
int eos = 1; // write "end of stream" marker?
int multithreading = 1; // use multithreading if available?
char *matchfinder = NULL; // matchfinder algorithm
int algorithm = 2;
int res;
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|iiiiiiiis", kwlist, &inFile, &dictionary, &fastBytes,
&literalContextBits, &literalPosBits, &posBits, &algorithm, &eos, &multithreading, &matchfinder))
return -1;
CHECK_RANGE(dictionary, 0, 28, "dictionary must be between 0 and 28");
CHECK_RANGE(fastBytes, 5, 255, "fastBytes must be between 5 and 255");
CHECK_RANGE(literalContextBits, 0, 8, "literalContextBits must be between 0 and 8");
CHECK_RANGE(literalPosBits, 0, 4, "literalPosBits must be between 0 and 4");
CHECK_RANGE(posBits, 0, 4, "posBits must be between 0 and 4");
CHECK_RANGE(algorithm, 0, 2, "algorithm must be between 0 and 2");
if (matchfinder != NULL) {
#if (PY_VERSION_HEX >= 0x02050000)
PyErr_WarnEx(PyExc_DeprecationWarning, "matchfinder selection is deprecated and will be ignored", 1);
#else
PyErr_Warn(PyExc_DeprecationWarning, "matchfinder selection is deprecated and will be ignored");
#endif
}
if (PyString_Check(inFile)) {
// create new cStringIO object from string
inFile = PycStringIO->NewInput(inFile);
if (inFile == NULL)
{
PyErr_NoMemory();
return -1;
}
} else if (!PyObject_HasAttrString(inFile, "read")) {
PyErr_SetString(PyExc_ValueError, "first parameter must be a file-like object");
return -1;
} else {
// protect object from being refcounted out...
Py_INCREF(inFile);
}
self->encoder = LzmaEnc_Create(&allocator);
if (self->encoder == NULL) {
Py_DECREF(inFile);
PyErr_NoMemory();
return -1;
}
LzmaEncProps_Init(&props);
props.dictSize = 1 << dictionary;
props.lc = literalContextBits;
props.lp = literalPosBits;
props.pb = posBits;
props.algo = algorithm;
props.fb = fastBytes;
// props.btMode = 1;
// props.numHashBytes = 4;
// props.mc = 32;
props.writeEndMark = eos ? 1 : 0;
props.numThreads = multithreading ? 2 : 1;
LzmaEncProps_Normalize(&props);
res = LzmaEnc_SetProps(self->encoder, &props);
if (res != SZ_OK) {
Py_DECREF(inFile);
PyErr_Format(PyExc_TypeError, "could not set encoder properties: %d", res);
return -1;
}
self->inFile = inFile;
CreatePythonInStream(&self->inStream, inFile);
CreateMemoryOutStream(&self->outStream);
LzmaEnc_WriteProperties(self->encoder, header, &headerSize);
if (self->outStream.s.Write(&self->outStream, header, headerSize) != headerSize) {
PyErr_SetString(PyExc_TypeError, "could not generate stream header");
goto exit;
}
LzmaEnc_Prepare(self->encoder, &self->inStream.s, &self->outStream.s, &allocator, &allocator);
result = 0;
exit:
return result;
}
PyObject *
pylzma_compress(PyObject *self, PyObject *args, PyObject *kwargs)
{
PyObject *result = NULL;
CLzmaEncProps props;
CLzmaEncHandle encoder=NULL;
CMemoryOutStream outStream;
CMemoryInStream inStream;
Byte header[LZMA_PROPS_SIZE];
size_t headerSize = LZMA_PROPS_SIZE;
int res;
// possible keywords for this function
static char *kwlist[] = {"data", "dictionary", "fastBytes", "literalContextBits",
"literalPosBits", "posBits", "algorithm", "eos", "multithreading", "matchfinder", NULL};
int dictionary = 23; // [0,27], default 23 (8MB)
int fastBytes = 128; // [5,273], default 128
int literalContextBits = 3; // [0,8], default 3
int literalPosBits = 0; // [0,4], default 0
int posBits = 2; // [0,4], default 2
int eos = 1; // write "end of stream" marker?
int multithreading = 1; // use multithreading if available?
char *matchfinder = NULL; // matchfinder algorithm
int algorithm = 2;
char *data;
int length;
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#|iiiiiiiis", kwlist, &data, &length, &dictionary, &fastBytes,
&literalContextBits, &literalPosBits, &posBits, &algorithm, &eos, &multithreading, &matchfinder))
return NULL;
outStream.data = NULL;
CHECK_RANGE(dictionary, 0, 27, "dictionary must be between 0 and 27");
CHECK_RANGE(fastBytes, 5, 273, "fastBytes must be between 5 and 273");
CHECK_RANGE(literalContextBits, 0, 8, "literalContextBits must be between 0 and 8");
CHECK_RANGE(literalPosBits, 0, 4, "literalPosBits must be between 0 and 4");
CHECK_RANGE(posBits, 0, 4, "posBits must be between 0 and 4");
CHECK_RANGE(algorithm, 0, 2, "algorithm must be between 0 and 2");
if (matchfinder != NULL) {
#if (PY_VERSION_HEX >= 0x02050000)
PyErr_WarnEx(PyExc_DeprecationWarning, "matchfinder selection is deprecated and will be ignored", 1);
#else
PyErr_Warn(PyExc_DeprecationWarning, "matchfinder selection is deprecated and will be ignored");
#endif
}
encoder = LzmaEnc_Create(&allocator);
if (encoder == NULL)
return PyErr_NoMemory();
CreateMemoryInStream(&inStream, (Byte *) data, length);
CreateMemoryOutStream(&outStream);
LzmaEncProps_Init(&props);
props.dictSize = 1 << dictionary;
props.lc = literalContextBits;
props.lp = literalPosBits;
props.pb = posBits;
props.algo = algorithm;
props.fb = fastBytes;
// props.btMode = 1;
// props.numHashBytes = 4;
// props.mc = 32;
props.writeEndMark = eos ? 1 : 0;
props.numThreads = multithreading ? 2 : 1;
LzmaEncProps_Normalize(&props);
res = LzmaEnc_SetProps(encoder, &props);
if (res != SZ_OK) {
PyErr_Format(PyExc_TypeError, "could not set encoder properties: %d", res);
goto exit;
}
Py_BEGIN_ALLOW_THREADS
LzmaEnc_WriteProperties(encoder, header, &headerSize);
if (outStream.s.Write(&outStream, header, headerSize) != headerSize) {
res = SZ_ERROR_WRITE;
} else {
res = LzmaEnc_Encode(encoder, &outStream.s, &inStream.s, NULL, &allocator, &allocator);
}
Py_END_ALLOW_THREADS
if (res != SZ_OK) {
PyErr_Format(PyExc_TypeError, "Error during compressing: %d", res);
goto exit;
}
result = PyBytes_FromStringAndSize((const char *) outStream.data, outStream.size);
exit:
if (encoder != NULL) {
LzmaEnc_Destroy(encoder, &allocator, &allocator);
}
if (outStream.data != NULL) {
free(outStream.data);
}
return result;
}