int64_t lzbench_lzma_compress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t level, size_t, size_t)
{
char rs[800] = { 0 };
CLzmaEncProps props;
int res;
size_t headerSize = LZMA_PROPS_SIZE;
SizeT out_len = outsize - LZMA_PROPS_SIZE;
LzmaEncProps_Init(&props);
props.level = level;
LzmaEncProps_Normalize(&props);
/*
p->level = 5;
p->dictSize = p->mc = 0;
p->reduceSize = (UInt64)(Int64)-1;
p->lc = p->lp = p->pb = p->algo = p->fb = p->btMode = p->numHashBytes = p->numThreads = -1;
p->writeEndMark = 0;
*/
res = LzmaEncode((uint8_t*)outbuf+LZMA_PROPS_SIZE, &out_len, (uint8_t*)inbuf, insize, &props, (uint8_t*)outbuf, &headerSize, 0/*int writeEndMark*/, NULL, &g_Alloc, &g_Alloc);
if (res != SZ_OK) return 0;
// printf("out_len=%u LZMA_PROPS_SIZE=%d headerSize=%d\n", (int)(out_len + LZMA_PROPS_SIZE), LZMA_PROPS_SIZE, (int)headerSize);
return LZMA_PROPS_SIZE + out_len;
}
/*
* The two functions below are not thread-safe, by design.
*/
int
lzma_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op)
{
if (!p && op == COMPRESS) {
p = (CLzmaEncProps *)slab_alloc(NULL, sizeof (CLzmaEncProps));
LzmaEncProps_Init(p);
/*
* Set the dictionary size and fast bytes based on level.
*/
if (*level < 8) {
/*
* Choose a dict size with a balance between perf and
* compression.
*/
p->dictSize = LZMA_DEFAULT_DICT;
} else {
/*
* Let LZMA determine best dict size.
*/
p->dictSize = 0;
}
/* Determine the fast bytes value and also adjust dict size further. */
if (*level < 7) {
p->fb = 32;
} else if (*level < 10) {
p->fb = 64;
} else if (*level == 11) {
p->fb = 64;
p->mc = 128;
} else if (*level == 12) {
p->fb = 128;
p->mc = 256;
} else if (*level == 13) {
p->fb = 64;
p->mc = 128;
p->dictSize = (1 << 27);
} else if (*level == 14) {
p->fb = 128;
p->mc = 256;
p->dictSize = (1 << 28);
}
if (*level > 9) *level = 9;
p->level = *level;
p->numThreads = nthreads;
LzmaEncProps_Normalize(p);
slab_cache_add(p->litprob_sz);
}
if (*level > 9) *level = 9;
*data = p;
return (0);
}
static int
pylzma_compfile_init(CCompressionFileObject *self, PyObject *args, PyObject *kwargs)
{
PyObject *inFile;
CLzmaEncProps props;
Byte header[LZMA_PROPS_SIZE];
size_t headerSize = LZMA_PROPS_SIZE;
int result = -1;
// possible keywords for this function
static char *kwlist[] = {"infile", "dictionary", "fastBytes", "literalContextBits",
"literalPosBits", "posBits", "algorithm", "eos", "multithreading", "matchfinder", NULL};
int dictionary = 23; // [0,28], default 23 (8MB)
int fastBytes = 128; // [5,255], default 128
int literalContextBits = 3; // [0,8], default 3
int literalPosBits = 0; // [0,4], default 0
int posBits = 2; // [0,4], default 2
int eos = 1; // write "end of stream" marker?
int multithreading = 1; // use multithreading if available?
char *matchfinder = NULL; // matchfinder algorithm
int algorithm = 2;
int res;
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|iiiiiiiis", kwlist, &inFile, &dictionary, &fastBytes,
&literalContextBits, &literalPosBits, &posBits, &algorithm, &eos, &multithreading, &matchfinder))
return -1;
CHECK_RANGE(dictionary, 0, 28, "dictionary must be between 0 and 28");
CHECK_RANGE(fastBytes, 5, 255, "fastBytes must be between 5 and 255");
CHECK_RANGE(literalContextBits, 0, 8, "literalContextBits must be between 0 and 8");
CHECK_RANGE(literalPosBits, 0, 4, "literalPosBits must be between 0 and 4");
CHECK_RANGE(posBits, 0, 4, "posBits must be between 0 and 4");
CHECK_RANGE(algorithm, 0, 2, "algorithm must be between 0 and 2");
if (matchfinder != NULL) {
#if (PY_VERSION_HEX >= 0x02050000)
PyErr_WarnEx(PyExc_DeprecationWarning, "matchfinder selection is deprecated and will be ignored", 1);
#else
PyErr_Warn(PyExc_DeprecationWarning, "matchfinder selection is deprecated and will be ignored");
#endif
}
if (PyString_Check(inFile)) {
// create new cStringIO object from string
inFile = PycStringIO->NewInput(inFile);
if (inFile == NULL)
{
PyErr_NoMemory();
return -1;
}
} else if (!PyObject_HasAttrString(inFile, "read")) {
PyErr_SetString(PyExc_ValueError, "first parameter must be a file-like object");
return -1;
} else {
// protect object from being refcounted out...
Py_INCREF(inFile);
}
self->encoder = LzmaEnc_Create(&allocator);
if (self->encoder == NULL) {
Py_DECREF(inFile);
PyErr_NoMemory();
return -1;
}
LzmaEncProps_Init(&props);
props.dictSize = 1 << dictionary;
props.lc = literalContextBits;
props.lp = literalPosBits;
props.pb = posBits;
props.algo = algorithm;
props.fb = fastBytes;
// props.btMode = 1;
// props.numHashBytes = 4;
// props.mc = 32;
props.writeEndMark = eos ? 1 : 0;
props.numThreads = multithreading ? 2 : 1;
LzmaEncProps_Normalize(&props);
res = LzmaEnc_SetProps(self->encoder, &props);
if (res != SZ_OK) {
Py_DECREF(inFile);
PyErr_Format(PyExc_TypeError, "could not set encoder properties: %d", res);
return -1;
}
self->inFile = inFile;
CreatePythonInStream(&self->inStream, inFile);
CreateMemoryOutStream(&self->outStream);
LzmaEnc_WriteProperties(self->encoder, header, &headerSize);
if (self->outStream.s.Write(&self->outStream, header, headerSize) != headerSize) {
PyErr_SetString(PyExc_TypeError, "could not generate stream header");
goto exit;
}
LzmaEnc_Prepare(self->encoder, &self->inStream.s, &self->outStream.s, &allocator, &allocator);
result = 0;
exit:
return result;
}
static SRes Encode(ISeqOutStream *outStream, ISeqInStream *inStream, UInt64 fileSize)
{
SRes res;
size_t inSize = (size_t)fileSize;
Byte *inBuffer = 0;
Byte *outBuffer = 0;
Byte *filteredStream = 0;
size_t outSize;
CLzmaEncProps props;
LzmaEncProps_Init(&props);
LzmaEncProps_Normalize(&props);
if (inSize != 0) {
inBuffer = (Byte *)MyAlloc(inSize);
if (inBuffer == 0)
return SZ_ERROR_MEM;
} else {
return SZ_ERROR_INPUT_EOF;
}
if (SeqInStream_Read(inStream, inBuffer, inSize) != SZ_OK) {
res = SZ_ERROR_READ;
goto Done;
}
// we allocate 105% of original size + 64KB for output buffer
outSize = (size_t)fileSize / 20 * 21 + (1 << 16);
outBuffer = (Byte *)MyAlloc(outSize);
if (outBuffer == 0) {
res = SZ_ERROR_MEM;
goto Done;
}
{
int i;
for (i = 0; i < 8; i++)
outBuffer[i + LZMA_PROPS_SIZE] = (Byte)(fileSize >> (8 * i));
}
if (mConType != NoConverter)
{
filteredStream = (Byte *)MyAlloc(inSize);
if (filteredStream == 0) {
res = SZ_ERROR_MEM;
goto Done;
}
memcpy(filteredStream, inBuffer, inSize);
if (mConType == X86Converter) {
{
UInt32 x86State;
x86_Convert_Init(x86State);
x86_Convert(filteredStream, (SizeT) inSize, 0, &x86State, 1);
}
}
}
{
size_t outSizeProcessed = outSize - LZMA_HEADER_SIZE;
size_t outPropsSize = LZMA_PROPS_SIZE;
res = LzmaEncode(outBuffer + LZMA_HEADER_SIZE, &outSizeProcessed,
mConType != NoConverter ? filteredStream : inBuffer, inSize,
&props, outBuffer, &outPropsSize, 0,
NULL, &g_Alloc, &g_Alloc);
if (res != SZ_OK)
goto Done;
outSize = LZMA_HEADER_SIZE + outSizeProcessed;
}
if (outStream->Write(outStream, outBuffer, outSize) != outSize)
res = SZ_ERROR_WRITE;
Done:
MyFree(outBuffer);
MyFree(inBuffer);
MyFree(filteredStream);
return res;
}
PyObject *
pylzma_compress(PyObject *self, PyObject *args, PyObject *kwargs)
{
PyObject *result = NULL;
CLzmaEncProps props;
CLzmaEncHandle encoder=NULL;
CMemoryOutStream outStream;
CMemoryInStream inStream;
Byte header[LZMA_PROPS_SIZE];
size_t headerSize = LZMA_PROPS_SIZE;
int res;
// possible keywords for this function
static char *kwlist[] = {"data", "dictionary", "fastBytes", "literalContextBits",
"literalPosBits", "posBits", "algorithm", "eos", "multithreading", "matchfinder", NULL};
int dictionary = 23; // [0,27], default 23 (8MB)
int fastBytes = 128; // [5,273], default 128
int literalContextBits = 3; // [0,8], default 3
int literalPosBits = 0; // [0,4], default 0
int posBits = 2; // [0,4], default 2
int eos = 1; // write "end of stream" marker?
int multithreading = 1; // use multithreading if available?
char *matchfinder = NULL; // matchfinder algorithm
int algorithm = 2;
char *data;
int length;
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#|iiiiiiiis", kwlist, &data, &length, &dictionary, &fastBytes,
&literalContextBits, &literalPosBits, &posBits, &algorithm, &eos, &multithreading, &matchfinder))
return NULL;
outStream.data = NULL;
CHECK_RANGE(dictionary, 0, 27, "dictionary must be between 0 and 27");
CHECK_RANGE(fastBytes, 5, 273, "fastBytes must be between 5 and 273");
CHECK_RANGE(literalContextBits, 0, 8, "literalContextBits must be between 0 and 8");
CHECK_RANGE(literalPosBits, 0, 4, "literalPosBits must be between 0 and 4");
CHECK_RANGE(posBits, 0, 4, "posBits must be between 0 and 4");
CHECK_RANGE(algorithm, 0, 2, "algorithm must be between 0 and 2");
if (matchfinder != NULL) {
#if (PY_VERSION_HEX >= 0x02050000)
PyErr_WarnEx(PyExc_DeprecationWarning, "matchfinder selection is deprecated and will be ignored", 1);
#else
PyErr_Warn(PyExc_DeprecationWarning, "matchfinder selection is deprecated and will be ignored");
#endif
}
encoder = LzmaEnc_Create(&allocator);
if (encoder == NULL)
return PyErr_NoMemory();
CreateMemoryInStream(&inStream, (Byte *) data, length);
CreateMemoryOutStream(&outStream);
LzmaEncProps_Init(&props);
props.dictSize = 1 << dictionary;
props.lc = literalContextBits;
props.lp = literalPosBits;
props.pb = posBits;
props.algo = algorithm;
props.fb = fastBytes;
// props.btMode = 1;
// props.numHashBytes = 4;
// props.mc = 32;
props.writeEndMark = eos ? 1 : 0;
props.numThreads = multithreading ? 2 : 1;
LzmaEncProps_Normalize(&props);
res = LzmaEnc_SetProps(encoder, &props);
if (res != SZ_OK) {
PyErr_Format(PyExc_TypeError, "could not set encoder properties: %d", res);
goto exit;
}
Py_BEGIN_ALLOW_THREADS
LzmaEnc_WriteProperties(encoder, header, &headerSize);
if (outStream.s.Write(&outStream, header, headerSize) != headerSize) {
res = SZ_ERROR_WRITE;
} else {
res = LzmaEnc_Encode(encoder, &outStream.s, &inStream.s, NULL, &allocator, &allocator);
}
Py_END_ALLOW_THREADS
if (res != SZ_OK) {
PyErr_Format(PyExc_TypeError, "Error during compressing: %d", res);
goto exit;
}
result = PyBytes_FromStringAndSize((const char *) outStream.data, outStream.size);
exit:
if (encoder != NULL) {
LzmaEnc_Destroy(encoder, &allocator, &allocator);
}
if (outStream.data != NULL) {
free(outStream.data);
}
return result;
}
Vector<uint8_t> Image::serialize() const
{
Vector<uint8_t> result;
#define A(...) appendToVector(result, __VA_ARGS__);
//imageinfo
A(info.architecture);
A(info.baseAddress);
A(info.entryPoint);
A(info.flag);
A(info.platformData);
A(info.platformData1);
A(info.size);
A(fileName);
A(static_cast<uint32_t>(exports.size()));
for(auto &i : exports)
{
A(i.address);
A(i.forward);
A(i.name);
A(i.nameHash);
A(i.ordinal);
}
A(static_cast<uint32_t>(sections.size()));
for(auto &i : sections)
{
A(i.name);
A(i.baseAddress);
A(i.size);
A(i.flag);
}
A(static_cast<uint32_t>(imports.size()));
for(auto &i : imports)
{
A(i.libraryName);
A(static_cast<uint32_t>(i.functions.size()));
for(auto &j : i.functions)
{
A(j.iat);
A(j.name);
A(j.nameHash);
A(j.ordinal);
}
}
A(static_cast<uint32_t>(relocations.size()));
for(auto &i : relocations)
A(i);
for(auto &i : sections)
{
A(i.data->get(), i.data->size());
if((i.flag & SectionFlagCode) && i.data->size() > 5)
{
uint8_t *codeStart = result.end() - i.data->size();
for(size_t j = 0; j < i.data->size() - 5; j ++)
{
if(codeStart[j] == 0xE8 || codeStart[j] == 0xE9) //call rel32, jmp rel32
{
*reinterpret_cast<int32_t *>(codeStart + j + 1) += (j + 5);
j += 4;
}
else if(codeStart[j] == 0x0f && (codeStart[j + 1] >= 0x80 && codeStart[j + 1] <= 0x8f)) //conditional jmp rel32
{
*reinterpret_cast<int32_t *>(codeStart + j + 2) += (j + 6);
j += 5;
}
}
}
}
A(header->get(), header->size());
#undef A
CLzmaEncProps props;
LzmaEncProps_Init(&props);
props.numThreads = 1;
LzmaEncProps_Normalize(&props);
uint32_t sizeSize = sizeof(uint32_t) * 2;
uint32_t propsSize = LZMA_PROPS_SIZE;
uint32_t outSize = result.size() + result.size() / 40 + (1 << 12); //igor recommends (http://sourceforge.net/p/sevenzip/discussion/45798/thread/dd3b392c/)
Vector<uint8_t> compressed(outSize + propsSize + sizeSize);
LzmaEncode(&compressed[propsSize + sizeSize], &outSize, &result[0], result.size(), &props, &compressed[sizeSize], &propsSize, 0, nullptr, &g_Alloc, &g_Alloc);
*reinterpret_cast<uint32_t *>(&compressed[0]) = result.size();
*reinterpret_cast<uint32_t *>(&compressed[sizeof(uint32_t)]) = outSize;
compressed.resize(outSize + propsSize + sizeSize);
return compressed;
}
