MY_STDAPI LzmaCompress(unsigned char *dest, size_t *destLen, const unsigned char *src, size_t srcLen,
unsigned char *outProps, size_t *outPropsSize,
int level, /* 0 <= level <= 9, default = 5 */
unsigned dictSize, /* use (1 << N) or (3 << N). 4 KB < dictSize <= 128 MB */
int lc, /* 0 <= lc <= 8, default = 3 */
int lp, /* 0 <= lp <= 4, default = 0 */
int pb, /* 0 <= pb <= 4, default = 2 */
int fb, /* 5 <= fb <= 273, default = 32 */
int numThreads, /* 1 or 2, default = 2 */
int algo /* 0 = fast, 1 = normal */
)
{
CLzmaEncProps props;
LzmaEncProps_Init(&props);
props.level = level;
props.dictSize = dictSize;
props.lc = lc;
props.lp = lp;
props.pb = pb;
props.fb = fb;
props.numThreads = numThreads;
props.algo = algo;
return LzmaEncode(dest, destLen, src, srcLen, &props, outProps, outPropsSize, 0,
NULL, &g_Alloc, &g_Alloc);
}
static SRes Encode(ISeqOutStream *outStream, ISeqInStream *inStream, UInt64 fileSize, ICompressProgress *pg, int lvl)
{
CLzmaEncHandle enc;
SRes res;
CLzmaEncProps props;
enc = LzmaEnc_Create(&g_Alloc);
if (enc == 0)
return SZ_ERROR_MEM;
LzmaEncProps_Init(&props);
props.level = lvl;
res = LzmaEnc_SetProps(enc, &props);
if(res == SZ_OK) {
Byte header[LZMA_PROPS_SIZE + 8];
size_t headerSize = LZMA_PROPS_SIZE;
int i;
res = LzmaEnc_WriteProperties(enc, header, &headerSize);
for(i = 0; i < 8; i++)
header[headerSize++] = (Byte)(fileSize >> (8 * i));
if(outStream->Write(outStream, header, headerSize) != headerSize)
res = SZ_ERROR_WRITE;
else {
if(res == SZ_OK)
res = LzmaEnc_Encode(enc, outStream, inStream, pg, &g_Alloc, &g_Alloc);
}
}
int64_t lzbench_lzma_compress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t level, size_t, size_t)
{
char rs[800] = { 0 };
CLzmaEncProps props;
int res;
size_t headerSize = LZMA_PROPS_SIZE;
SizeT out_len = outsize - LZMA_PROPS_SIZE;
LzmaEncProps_Init(&props);
props.level = level;
LzmaEncProps_Normalize(&props);
/*
p->level = 5;
p->dictSize = p->mc = 0;
p->reduceSize = (UInt64)(Int64)-1;
p->lc = p->lp = p->pb = p->algo = p->fb = p->btMode = p->numHashBytes = p->numThreads = -1;
p->writeEndMark = 0;
*/
res = LzmaEncode((uint8_t*)outbuf+LZMA_PROPS_SIZE, &out_len, (uint8_t*)inbuf, insize, &props, (uint8_t*)outbuf, &headerSize, 0/*int writeEndMark*/, NULL, &g_Alloc, &g_Alloc);
if (res != SZ_OK) return 0;
// printf("out_len=%u LZMA_PROPS_SIZE=%d headerSize=%d\n", (int)(out_len + LZMA_PROPS_SIZE), LZMA_PROPS_SIZE, (int)headerSize);
return LZMA_PROPS_SIZE + out_len;
}
bool CCCrypto::compressData( const unsigned char* pInData, unsigned int nInSize, unsigned char* pOutData, unsigned int& nOutSize )
{
if ( nOutSize <= sizeof(stHead) )
return false;
nOutSize -= sizeof(stHead);
unsigned int propsSize = LZMA_PROPS_SIZE;
CLzmaEncProps props;
LzmaEncProps_Init(&props);
props.level = 1;
// props.dictSize = 1 << 26; // 64 MB
// props.fb = 64;
// props.level = 1;
// props.dictSize = 1 << 12;
// props.algo = 0;
props.writeEndMark = 0; // 0 or 1
stHead* phead = (stHead*)pOutData;
int res = LzmaEncode( &(pOutData[sizeof(stHead)]), (SizeT*)&nOutSize, pInData, nInSize, &props, (phead->m_szPorp), (SizeT*)&propsSize, props.writeEndMark, NULL, &SzAllocForLzma, &SzAllocForLzma ); // 隐含的要求pOutData必须大于LZMA_PROPS_SIZE+4
if ( res == SZ_OK && propsSize == LZMA_PROPS_SIZE )
{
nOutSize += sizeof(stHead);
phead->m_szPorp[3] ^= 5;
phead->m_OriginalSize = nInSize^1002;
phead->m_sign = '@Fml';
phead->m_nRandomValue = rand()%256;
for ( unsigned i = 0; i < 16 && i < nOutSize; i ++ )
{
pOutData[sizeof(stHead)+i] ^= 6636;
}
return true;
}
return false;
}
/*
* The two functions below are not thread-safe, by design.
*/
int
lzma_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op)
{
if (!p && op == COMPRESS) {
p = (CLzmaEncProps *)slab_alloc(NULL, sizeof (CLzmaEncProps));
LzmaEncProps_Init(p);
/*
* Set the dictionary size and fast bytes based on level.
*/
if (*level < 8) {
/*
* Choose a dict size with a balance between perf and
* compression.
*/
p->dictSize = LZMA_DEFAULT_DICT;
} else {
/*
* Let LZMA determine best dict size.
*/
p->dictSize = 0;
}
/* Determine the fast bytes value and also adjust dict size further. */
if (*level < 7) {
p->fb = 32;
} else if (*level < 10) {
p->fb = 64;
} else if (*level == 11) {
p->fb = 64;
p->mc = 128;
} else if (*level == 12) {
p->fb = 128;
p->mc = 256;
} else if (*level == 13) {
p->fb = 64;
p->mc = 128;
p->dictSize = (1 << 27);
} else if (*level == 14) {
p->fb = 128;
p->mc = 256;
p->dictSize = (1 << 28);
}
if (*level > 9) *level = 9;
p->level = *level;
p->numThreads = nthreads;
LzmaEncProps_Normalize(p);
slab_cache_add(p->litprob_sz);
}
if (*level > 9) *level = 9;
*data = p;
return (0);
}
uint8_t* encodeLZMA(System* s,
avian::util::Alloc* a,
uint8_t* in,
unsigned inSize,
unsigned* outSize)
{
const unsigned PropHeaderSize = 5;
const unsigned HeaderSize = 13;
unsigned bufferSize = inSize * 2;
uint8_t* buffer = static_cast<uint8_t*>(a->allocate(bufferSize));
LzmaAllocator allocator(a);
CLzmaEncProps props;
LzmaEncProps_Init(&props);
props.level = 9;
props.writeEndMark = 1;
ICompressProgress progress = {myProgress};
SizeT propsSize = PropHeaderSize;
int32_t inSize32 = inSize;
memcpy(buffer + PropHeaderSize, &inSize32, 4);
SizeT outSizeT = bufferSize;
int result = LzmaEncode(buffer + HeaderSize,
&outSizeT,
in,
inSize,
&props,
buffer,
&propsSize,
1,
&progress,
&(allocator.allocator),
&(allocator.allocator));
expect(s, result == SZ_OK);
*outSize = outSizeT + HeaderSize;
uint8_t* out = static_cast<uint8_t*>(a->allocate(*outSize));
memcpy(out, buffer, *outSize);
a->free(buffer, bufferSize);
return out;
}
EFI_STATUS
EFIAPI
LzmaCompress (
CONST UINT8 *Source,
UINT32 SourceSize,
UINT8 *Destination,
UINT32 *DestinationSize,
UINT32 DictionarySize
)
{
SRes LzmaResult;
CLzmaEncProps props;
SizeT propsSize = LZMA_PROPS_SIZE;
SizeT destLen = SourceSize + SourceSize / 3 + 128;
if (*DestinationSize < (UINT32)destLen)
{
*DestinationSize = (UINT32)destLen;
return EFI_BUFFER_TOO_SMALL;
}
LzmaEncProps_Init(&props);
props.dictSize = DictionarySize;
props.level = 9;
props.fb = 273;
LzmaResult = LzmaEncode(
(Byte*)((UINT8*)Destination + LZMA_HEADER_SIZE),
&destLen,
Source,
(SizeT)SourceSize,
&props,
(UINT8*)Destination,
&propsSize,
props.writeEndMark,
&g_ProgressCallback,
&SzAllocForLzma,
&SzAllocForLzma);
*DestinationSize = (UINT32)(destLen + LZMA_HEADER_SIZE);
SetEncodedSizeOfBuf(SourceSize, Destination);
if (LzmaResult == SZ_OK) {
return EFI_SUCCESS;
}
else {
return EFI_INVALID_PARAMETER;
}
}
bool zmodifyer::compress( unsigned char const * src, size_t srcLen, unsigned char * dest, size_t & destLen, int level /* = 5 */ )
{
if( !src || !dest || destLen < 10 )
return false;
CLzmaEncProps props;
LzmaEncProps_Init(&props);
props.level = level;
props.algo = 0;
props.lc = 3;
props.lp = 0;
props.pb = 2;
static const unsigned int kb = 1024;
if( srcLen > 1024 * 1024 * kb )
{
props.dictSize = 16 * 1024 * kb;
props.fb = 32;
}
else if( srcLen > 1024 * kb )
{
props.dictSize = 4 * 1024 * kb;
props.fb = 16;
}
else if( srcLen > kb )
{
props.dictSize = 64 * kb;
props.fb = 8;
}
else
{
props.dictSize = 64 * kb;
props.fb = 8;
}
props.numThreads = 2;
size_t outPropsSize = 5;
int result = LzmaEncode(dest+10, &destLen, src, srcLen, &props, dest, &outPropsSize, 0,
NULL, &alloctator, &alloctator);
*(dest+5) = (unsigned char)outPropsSize;
*(unsigned int*)(dest+6) = srcLen;
destLen += 10;
return (result == SZ_OK) ? true : false;
}
int LzmaCompress(char *in_data, int in_size, char *out_data, int out_size, unsigned long *total_out)
{
CLzmaEncProps props;
size_t headerSize = LZMA_PROPS_SIZE;
int ret;
SizeT outProcess;
LzmaEncProps_Init(&props);
props.algo = 1;
outProcess = out_size - LZMA_PROPS_SIZE;
ret = LzmaEncode(out_data+LZMA_PROPS_SIZE, &outProcess, in_data, in_size, &props, out_data,
&headerSize, 0, NULL, &g_Alloc, &g_Alloc);
*total_out = outProcess + LZMA_PROPS_SIZE;
return ret;
}
EFI_STATUS
LzmaCompress (
IN UINT8 *Source,
IN UINT32 SourceSize,
IN UINT8 *Destination,
IN OUT UINT32 *DestinationSize
)
{
SRes LzmaResult;
CLzmaEncProps props;
SizeT propsSize = LZMA_PROPS_SIZE;
SizeT destLen = SourceSize + SourceSize / 3 + 128;
if (*DestinationSize < destLen)
{
*DestinationSize = destLen;
return EFI_BUFFER_TOO_SMALL;
}
LzmaEncProps_Init(&props);
props.dictSize = LZMA_DICTIONARY_SIZE;
props.level = 9;
props.fb = 273;
LzmaResult = LzmaEncode(
(Byte*)((UINT8*)Destination + LZMA_HEADER_SIZE),
&destLen,
(VOID*)Source,
SourceSize,
&props,
(UINT8*)Destination,
&propsSize,
props.writeEndMark,
&g_ProgressCallback,
&SzAllocForLzma,
&SzAllocForLzma);
*DestinationSize = destLen + LZMA_HEADER_SIZE;
SetEncodedSizeOfBuf((UINT64)SourceSize, Destination);
if (LzmaResult == SZ_OK) {
return EFI_SUCCESS;
} else {
return EFI_INVALID_PARAMETER;
}
}
static bool Compress(const char *uncompressed, size_t uncompressedSize, char *compressed, size_t *compressedSize)
{
ISzCrtAlloc lzmaAlloc;
CrashIf(*compressedSize < uncompressedSize + 1);
if (*compressedSize < uncompressedSize + 1)
return false;
if (*compressedSize < LZMA_HEADER_SIZE)
goto StoreUncompressed;
CLzmaEncProps props;
LzmaEncProps_Init(&props);
// always apply the BCJ filter for speed (else two or three compression passes would be required)
size_t lzma_size = (size_t)-1;
uint8_t *bcj_enc = (uint8_t *)malloc(uncompressedSize);
if (bcj_enc) {
memcpy(bcj_enc, uncompressed, uncompressedSize);
UInt32 x86State;
x86_Convert_Init(x86State);
x86_Convert(bcj_enc, uncompressedSize, 0, &x86State, 1);
}
SizeT outSize = *compressedSize - LZMA_HEADER_SIZE;
SizeT propsSize = LZMA_PROPS_SIZE;
SRes res = LzmaEncode((Byte *)compressed + LZMA_HEADER_SIZE, &outSize,
bcj_enc ? bcj_enc : (const Byte *)uncompressed, uncompressedSize,
&props, (Byte *)compressed + 1, &propsSize,
TRUE /* add EOS marker */, NULL, &lzmaAlloc, &lzmaAlloc);
if (SZ_OK == res && propsSize == LZMA_PROPS_SIZE)
lzma_size = outSize + LZMA_HEADER_SIZE;
free(bcj_enc);
if (lzma_size < uncompressedSize + 1) {
compressed[0] = bcj_enc ? 1 : 0;
*compressedSize = lzma_size;
}
else {
StoreUncompressed:
compressed[0] = (uint8_t)-1;
memcpy(compressed + 1, uncompressed, uncompressedSize);
*compressedSize = uncompressedSize + 1;
}
return true;
}
void CompressInc(std::vector<unsigned char> &outBuf, const std::vector<unsigned char> &inBuf)
{
CLzmaEncHandle enc = LzmaEnc_Create(&g_Alloc);
assert(enc);
CLzmaEncProps props;
LzmaEncProps_Init(&props);
//props.writeEndMark = 1; // 0 or 1
SRes res = LzmaEnc_SetProps(enc, &props);
assert(res == SZ_OK);
unsigned char tempprops[5];
unsigned propsSize = LZMA_PROPS_SIZE;
//outBuf.resize(propsSize); //fill with zero, which cost me a week to check the mistake.
res = LzmaEnc_WriteProperties(enc, tempprops, &propsSize);
assert(res == SZ_OK && propsSize == LZMA_PROPS_SIZE);
VectorInStream inStream = { &VectorInStream_Read, &inBuf, 0 };
VectorOutStream outStream = { &VectorOutStream_Write, &outBuf };
res = LzmaEnc_Encode(enc,
(ISeqOutStream*)&outStream, (ISeqInStream*)&inStream,
0, &g_Alloc, &g_Alloc);
assert(res == SZ_OK);
UInt64 resLen = inBuf.size();
//printf("uint64 resLen : %" PRIu64 "\n", resLen); //print uint64
Byte header[8];
for (int i = 0; i < 8; i++)
header[i] = (Byte)(resLen >> (8 * i));
FILE *file = fopen("data3.dat", "wb+");
fwrite(tempprops, 1, propsSize, file);
fwrite(&header[0], 1, 8, file);
fwrite(&outBuf[0], 1, outBuf.size(), file);
fclose(file);
LzmaEnc_Destroy(enc, &g_Alloc, &g_Alloc);
}
STDMETHODIMP CEncoder::SetCoderProperties(const PROPID *propIDs,
const PROPVARIANT *coderProps, UInt32 numProps)
{
CLzmaEncProps props;
LzmaEncProps_Init(&props);
for (UInt32 i = 0; i < numProps; i++)
{
const PROPVARIANT &prop = coderProps[i];
switch (propIDs[i])
{
case NCoderPropID::kNumFastBytes:
if (prop.vt != VT_UI4) return E_INVALIDARG; props.fb = prop.ulVal; break;
case NCoderPropID::kMatchFinderCycles:
if (prop.vt != VT_UI4) return E_INVALIDARG; props.mc = prop.ulVal; break;
case NCoderPropID::kAlgorithm:
if (prop.vt != VT_UI4) return E_INVALIDARG; props.algo = prop.ulVal; break;
case NCoderPropID::kDictionarySize:
if (prop.vt != VT_UI4) return E_INVALIDARG; props.dictSize = prop.ulVal; break;
case NCoderPropID::kPosStateBits:
if (prop.vt != VT_UI4) return E_INVALIDARG; props.pb = prop.ulVal; break;
case NCoderPropID::kLitPosBits:
if (prop.vt != VT_UI4) return E_INVALIDARG; props.lp = prop.ulVal; break;
case NCoderPropID::kLitContextBits:
if (prop.vt != VT_UI4) return E_INVALIDARG; props.lc = prop.ulVal; break;
case NCoderPropID::kNumThreads:
if (prop.vt != VT_UI4) return E_INVALIDARG; props.numThreads = prop.ulVal; break;
case NCoderPropID::kMultiThread:
if (prop.vt != VT_BOOL) return E_INVALIDARG; props.numThreads = ((prop.boolVal == VARIANT_TRUE) ? 2 : 1); break;
case NCoderPropID::kEndMarker:
if (prop.vt != VT_BOOL) return E_INVALIDARG; props.writeEndMark = (prop.boolVal == VARIANT_TRUE); break;
case NCoderPropID::kMatchFinder:
if (prop.vt != VT_BSTR) return E_INVALIDARG;
if (!ParseMatchFinder(prop.bstrVal, &props.btMode, &props.numHashBytes /* , &_matchFinderBase.skipModeBits */))
return E_INVALIDARG; break;
default:
return E_INVALIDARG;
}
}
return SResToHRESULT(LzmaEnc_SetProps(_encoder, &props));
}
STDMETHODIMP CEncoder::SetCoderProperties(const PROPID *propIDs,
const PROPVARIANT *coderProps, UInt32 numProps)
{
CLzmaEncProps props;
LzmaEncProps_Init(&props);
for (UInt32 i = 0; i < numProps; i++)
{
const PROPVARIANT &prop = coderProps[i];
PROPID propID = propIDs[i];
switch (propID)
{
case NCoderPropID::kEndMarker:
if (prop.vt != VT_BOOL) return E_INVALIDARG; props.writeEndMark = (prop.boolVal == VARIANT_TRUE); break;
case NCoderPropID::kNumThreads:
if (prop.vt != VT_UI4) return E_INVALIDARG; props.numThreads = prop.ulVal; break;
default:
RINOK(SetLzmaProp(propID, prop, props));
}
}
return SResToHRESULT(LzmaEnc_SetProps(_encoder, &props));
}
static SRes Encode(FILE *inFile, FILE *outFile, char *rs)
{
CLzmaEncHandle enc;
SRes res;
CFileSeqInStream inStream;
CFileSeqOutStream outStream;
CLzmaEncProps props;
enc = LzmaEnc_Create(&g_Alloc);
if (enc == 0)
return SZ_ERROR_MEM;
inStream.funcTable.Read = MyRead;
inStream.file = inFile;
outStream.funcTable.Write = MyWrite;
outStream.file = outFile;
LzmaEncProps_Init(&props);
res = LzmaEnc_SetProps(enc, &props);
if (res == SZ_OK)
{
Byte header[LZMA_PROPS_SIZE + 8];
size_t headerSize = LZMA_PROPS_SIZE;
UInt64 fileSize;
int i;
res = LzmaEnc_WriteProperties(enc, header, &headerSize);
fileSize = MyGetFileLength(inFile);
for (i = 0; i < 8; i++)
header[headerSize++] = (Byte)(fileSize >> (8 * i));
if (!MyWriteFileAndCheck(outFile, header, headerSize))
return PrintError(rs, "writing error");
if (res == SZ_OK)
res = LzmaEnc_Encode(enc, &outStream.funcTable, &inStream.funcTable,
NULL, &g_Alloc, &g_Alloc);
}
size32_t compress(const void* input, size32_t inlength, void* output)
{ // returns (size32_t)-1 if cannot compress
if (!enc) {
enc = LzmaEnc_Create(&g_Alloc);
if (enc == 0)
throw MakeStringException(-1,"LzmaEnc_Create failed");
LzmaEncProps_Init(&props);
if (LzmaEnc_SetProps(enc, &props)!=SZ_OK)
throw MakeStringException(-1,"LzmaEnc_SetProps failed");
}
if (inlength+LZMA_PROPS_SIZE+sizeof(size32_t)<1024)
return (size32_t)-1; // don't compress less than 1K
SizeT propsize = LZMA_PROPS_SIZE;
LzmaEnc_WriteProperties(enc, (byte *)output+sizeof(size32_t), &propsize);
*(size32_t *)output = (size32_t)propsize;
SizeT reslen = inlength-1-propsize-sizeof(size32_t);
SRes res = LzmaEnc_MemEncode(enc, (byte *)output+propsize+sizeof(size32_t), &reslen, (const byte *)input, inlength, true, NULL, &g_Alloc, &g_Alloc);
if (res==SZ_ERROR_OUTPUT_EOF)
return (size32_t)-1;
if (res!=SZ_OK)
throw MakeStringException(-1,"LzmaEnc_MemEncode failed(%d)",(int)res);
return reslen+propsize+sizeof(size32_t);
}
inline TLzmaCompressBase(size_t level)
: H_(LzmaEnc_Create(Alloc()))
{
if (!H_) {
ythrow yexception() << "can not init lzma engine";
}
LzmaEncProps_Init(&Props_);
Props_.level = level;
Props_.dictSize = 0;
Props_.lc = -1;
Props_.lp = -1;
Props_.pb = -1;
Props_.fb = -1;
Props_.numThreads = -1;
Props_.writeEndMark = 1;
Check(LzmaEnc_SetProps(H_, &Props_));
size_t bufLen = sizeof(PropsBuf_);
Zero(PropsBuf_);
Check(LzmaEnc_WriteProperties(H_, PropsBuf_, &bufLen));
}
int INIT jffs2_lzma_init(void)
{
int ret;
CLzmaEncProps props;
LzmaEncProps_Init(&props);
props.dictSize = LZMA_BEST_DICT(0x2000);
props.level = LZMA_BEST_LEVEL;
props.lc = LZMA_BEST_LC;
props.lp = LZMA_BEST_LP;
props.pb = LZMA_BEST_PB;
props.fb = LZMA_BEST_FB;
ret = lzma_alloc_workspace(&props);
if (ret < 0)
return ret;
ret = jffs2_register_compressor(&jffs2_lzma_comp);
if (ret)
lzma_free_workspace();
return ret;
}
void* CBAWMeshWriter::compressWithLzma(const void* _input, size_t _inputSize, size_t& _outComprSize) const
{
ISzAlloc alloc{&asset::LzmaMemMngmnt::alloc, &asset::LzmaMemMngmnt::release};
SizeT propsSize = LZMA_PROPS_SIZE;
UInt32 dictSize = _inputSize; // next nearest (to input size) power of two times two
--dictSize;
for (uint32_t p = 1; p < 32; p <<= 1)
dictSize |= dictSize>>p;
++dictSize;
dictSize <<= 1;
// Lzma props: https://stackoverflow.com/a/21384797/5538150
CLzmaEncProps props;
LzmaEncProps_Init(&props);
props.dictSize = dictSize;
props.level = 5; // compression level [0;9]
props.algo = 0; // fast algo: a little worse compression, a little less loading time
props.lp = 2; // 2^2==sizeof(float)
const SizeT heapSize = _inputSize + LZMA_PROPS_SIZE;
uint8_t* data = (uint8_t*)_IRR_ALIGNED_MALLOC(heapSize,_IRR_SIMD_ALIGNMENT);
SizeT destSize = heapSize;
SRes res = LzmaEncode(data+propsSize, &destSize, (const Byte*)_input, _inputSize, &props, data, &propsSize, props.writeEndMark, NULL, &alloc, &alloc);
if (res != SZ_OK)
{
_IRR_ALIGNED_FREE(data);
data = (uint8_t*)const_cast<void*>(_input);
destSize = _inputSize;
#ifdef _DEBUG
os::Printer::log("Failed to compress (lzma). Blob exported without compression.", ELL_WARNING);
#endif
}
_outComprSize = destSize + propsSize;
return data;
}
void lzmaCompress(
unsigned char* destBuf,
int &destSize,
unsigned char *srcBuf,
int srcSize,
int compressionLevel
)
{
SizeT propsSize = LZMA_PROPS_SIZE;
SizeT lzmaDestSize = (SizeT)destSize;
CLzmaEncProps props;
LzmaEncProps_Init(&props);
props.level = compressionLevel; //compression level
//props.dictSize = 1 << 16;
//props.dictSize = 1 << 24;
props.writeEndMark = 1; // 0 or 1
int res = LzmaEncode(
destBuf+LZMA_PROPS_SIZE, &lzmaDestSize,
srcBuf, srcSize,
&props, destBuf, &propsSize, props.writeEndMark,
&g_ProgressCallback, &SzAllocForLzma, &SzAllocForLzma);
destSize = (int)lzmaDestSize + LZMA_PROPS_SIZE;
cout << "COMPRESSED " << srcSize << " BYTES DOWN TO " << destSize << endl;
if(res != SZ_OK || propsSize != LZMA_PROPS_SIZE)
{
cout << "ERROR COMPRESSING DATA\n";
cout << res << ',' << propsSize << endl;
exit(1);
}
}
Vector<uint8_t> Image::serialize() const
{
Vector<uint8_t> result;
#define A(...) appendToVector(result, __VA_ARGS__);
//imageinfo
A(info.architecture);
A(info.baseAddress);
A(info.entryPoint);
A(info.flag);
A(info.platformData);
A(info.platformData1);
A(info.size);
A(fileName);
A(static_cast<uint32_t>(exports.size()));
for(auto &i : exports)
{
A(i.address);
A(i.forward);
A(i.name);
A(i.nameHash);
A(i.ordinal);
}
A(static_cast<uint32_t>(sections.size()));
for(auto &i : sections)
{
A(i.name);
A(i.baseAddress);
A(i.size);
A(i.flag);
}
A(static_cast<uint32_t>(imports.size()));
for(auto &i : imports)
{
A(i.libraryName);
A(static_cast<uint32_t>(i.functions.size()));
for(auto &j : i.functions)
{
A(j.iat);
A(j.name);
A(j.nameHash);
A(j.ordinal);
}
}
A(static_cast<uint32_t>(relocations.size()));
for(auto &i : relocations)
A(i);
for(auto &i : sections)
{
A(i.data->get(), i.data->size());
if((i.flag & SectionFlagCode) && i.data->size() > 5)
{
uint8_t *codeStart = result.end() - i.data->size();
for(size_t j = 0; j < i.data->size() - 5; j ++)
{
if(codeStart[j] == 0xE8 || codeStart[j] == 0xE9) //call rel32, jmp rel32
{
*reinterpret_cast<int32_t *>(codeStart + j + 1) += (j + 5);
j += 4;
}
else if(codeStart[j] == 0x0f && (codeStart[j + 1] >= 0x80 && codeStart[j + 1] <= 0x8f)) //conditional jmp rel32
{
*reinterpret_cast<int32_t *>(codeStart + j + 2) += (j + 6);
j += 5;
}
}
}
}
A(header->get(), header->size());
#undef A
CLzmaEncProps props;
LzmaEncProps_Init(&props);
props.numThreads = 1;
LzmaEncProps_Normalize(&props);
uint32_t sizeSize = sizeof(uint32_t) * 2;
uint32_t propsSize = LZMA_PROPS_SIZE;
uint32_t outSize = result.size() + result.size() / 40 + (1 << 12); //igor recommends (http://sourceforge.net/p/sevenzip/discussion/45798/thread/dd3b392c/)
Vector<uint8_t> compressed(outSize + propsSize + sizeSize);
LzmaEncode(&compressed[propsSize + sizeSize], &outSize, &result[0], result.size(), &props, &compressed[sizeSize], &propsSize, 0, nullptr, &g_Alloc, &g_Alloc);
*reinterpret_cast<uint32_t *>(&compressed[0]) = result.size();
*reinterpret_cast<uint32_t *>(&compressed[sizeof(uint32_t)]) = outSize;
compressed.resize(outSize + propsSize + sizeSize);
return compressed;
}
int Lzma86_Encode(Byte *dest, size_t *destLen, const Byte *src, size_t srcLen,
int level, UInt32 dictSize, int filterMode)
{
ISzAlloc g_Alloc = { SzAlloc, SzFree };
size_t outSize2 = *destLen;
Byte *filteredStream;
Bool useFilter;
int mainResult = SZ_ERROR_OUTPUT_EOF;
CLzmaEncProps props;
LzmaEncProps_Init(&props);
props.level = level;
props.dictSize = dictSize;
*destLen = 0;
if (outSize2 < LZMA86_HEADER_SIZE)
return SZ_ERROR_OUTPUT_EOF;
{
int i;
UInt64 t = srcLen;
for (i = 0; i < 8; i++, t >>= 8)
dest[LZMA86_SIZE_OFFSET + i] = (Byte)t;
}
filteredStream = 0;
useFilter = (filterMode != SZ_FILTER_NO);
if (useFilter)
{
if (srcLen != 0)
{
filteredStream = (Byte *)MyAlloc(srcLen);
if (filteredStream == 0)
return SZ_ERROR_MEM;
memcpy(filteredStream, src, srcLen);
}
{
UInt32 x86State;
x86_Convert_Init(x86State);
x86_Convert(filteredStream, srcLen, 0, &x86State, 1);
}
}
{
size_t minSize = 0;
Bool bestIsFiltered = False;
/* passes for SZ_FILTER_AUTO:
0 - BCJ + LZMA
1 - LZMA
2 - BCJ + LZMA agaian, if pass 0 (BCJ + LZMA) is better.
*/
int numPasses = (filterMode == SZ_FILTER_AUTO) ? 3 : 1;
int i;
for (i = 0; i < numPasses; i++)
{
size_t outSizeProcessed = outSize2 - LZMA86_HEADER_SIZE;
size_t outPropsSize = 5;
SRes curRes;
Bool curModeIsFiltered = (numPasses > 1 && i == numPasses - 1);
if (curModeIsFiltered && !bestIsFiltered)
break;
if (useFilter && i == 0)
curModeIsFiltered = True;
curRes = LzmaEncode(dest + LZMA86_HEADER_SIZE, &outSizeProcessed,
curModeIsFiltered ? filteredStream : src, srcLen,
&props, dest + 1, &outPropsSize, 0,
NULL, &g_Alloc, &g_Alloc);
if (curRes != SZ_ERROR_OUTPUT_EOF)
{
if (curRes != SZ_OK)
{
mainResult = curRes;
break;
}
if (outSizeProcessed <= minSize || mainResult != SZ_OK)
{
minSize = outSizeProcessed;
bestIsFiltered = curModeIsFiltered;
mainResult = SZ_OK;
}
}
}
dest[0] = (Byte)(bestIsFiltered ? 1 : 0);
*destLen = LZMA86_HEADER_SIZE + minSize;
}
if (useFilter)
MyFree(filteredStream);
return mainResult;
}
/*static */ void Compress_LZMA(
char * pbOutBuffer,
int * pcbOutBuffer,
char * pbInBuffer,
int cbInBuffer,
int * /* pCmpType */,
int /* nCmpLevel */)
{
ICompressProgress Progress;
CLzmaEncProps props;
ISzAlloc SzAlloc;
Byte * destBuffer;
SizeT destLen = *pcbOutBuffer;
SizeT srcLen = cbInBuffer;
Byte encodedProps[LZMA_PROPS_SIZE];
size_t encodedPropsSize = LZMA_PROPS_SIZE;
SRes nResult;
// Fill the callbacks in structures
Progress.Progress = LZMA_Callback_Progress;
SzAlloc.Alloc = LZMA_Callback_Alloc;
SzAlloc.Free = LZMA_Callback_Free;
// Initialize properties
LzmaEncProps_Init(&props);
// Perform compression
destBuffer = (Byte *)pbOutBuffer + LZMA_HEADER_SIZE;
destLen = *pcbOutBuffer - LZMA_HEADER_SIZE;
nResult = LzmaEncode(destBuffer,
&destLen,
(Byte *)pbInBuffer,
srcLen,
&props,
encodedProps,
&encodedPropsSize,
0,
&Progress,
&SzAlloc,
&SzAlloc);
if(nResult != SZ_OK)
return;
// If we failed to compress the data
if(destLen >= (SizeT)(*pcbOutBuffer - LZMA_HEADER_SIZE))
return;
// Write "useFilter" variable. Blizzard MPQ must not use filter.
*pbOutBuffer++ = 0;
// Copy the encoded properties to the output buffer
memcpy(pbOutBuffer, encodedProps, encodedPropsSize);
pbOutBuffer += encodedPropsSize;
// Copy the size of the data
*pbOutBuffer++ = (unsigned char)(srcLen >> 0x00);
*pbOutBuffer++ = (unsigned char)(srcLen >> 0x08);
*pbOutBuffer++ = (unsigned char)(srcLen >> 0x10);
*pbOutBuffer++ = (unsigned char)(srcLen >> 0x18);
*pbOutBuffer++ = 0;
*pbOutBuffer++ = 0;
*pbOutBuffer++ = 0;
*pbOutBuffer++ = 0;
// Give the size of the data to the caller
*pcbOutBuffer = (unsigned int)(destLen + LZMA_HEADER_SIZE);
}
PyObject *
pylzma_compress(PyObject *self, PyObject *args, PyObject *kwargs)
{
PyObject *result = NULL;
CLzmaEncProps props;
CLzmaEncHandle encoder=NULL;
CMemoryOutStream outStream;
CMemoryInStream inStream;
Byte header[LZMA_PROPS_SIZE];
size_t headerSize = LZMA_PROPS_SIZE;
int res;
// possible keywords for this function
static char *kwlist[] = {"data", "dictionary", "fastBytes", "literalContextBits",
"literalPosBits", "posBits", "algorithm", "eos", "multithreading", "matchfinder", NULL};
int dictionary = 23; // [0,27], default 23 (8MB)
int fastBytes = 128; // [5,273], default 128
int literalContextBits = 3; // [0,8], default 3
int literalPosBits = 0; // [0,4], default 0
int posBits = 2; // [0,4], default 2
int eos = 1; // write "end of stream" marker?
int multithreading = 1; // use multithreading if available?
char *matchfinder = NULL; // matchfinder algorithm
int algorithm = 2;
char *data;
int length;
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#|iiiiiiiis", kwlist, &data, &length, &dictionary, &fastBytes,
&literalContextBits, &literalPosBits, &posBits, &algorithm, &eos, &multithreading, &matchfinder))
return NULL;
outStream.data = NULL;
CHECK_RANGE(dictionary, 0, 27, "dictionary must be between 0 and 27");
CHECK_RANGE(fastBytes, 5, 273, "fastBytes must be between 5 and 273");
CHECK_RANGE(literalContextBits, 0, 8, "literalContextBits must be between 0 and 8");
CHECK_RANGE(literalPosBits, 0, 4, "literalPosBits must be between 0 and 4");
CHECK_RANGE(posBits, 0, 4, "posBits must be between 0 and 4");
CHECK_RANGE(algorithm, 0, 2, "algorithm must be between 0 and 2");
if (matchfinder != NULL) {
#if (PY_VERSION_HEX >= 0x02050000)
PyErr_WarnEx(PyExc_DeprecationWarning, "matchfinder selection is deprecated and will be ignored", 1);
#else
PyErr_Warn(PyExc_DeprecationWarning, "matchfinder selection is deprecated and will be ignored");
#endif
}
encoder = LzmaEnc_Create(&allocator);
if (encoder == NULL)
return PyErr_NoMemory();
CreateMemoryInStream(&inStream, (Byte *) data, length);
CreateMemoryOutStream(&outStream);
LzmaEncProps_Init(&props);
props.dictSize = 1 << dictionary;
props.lc = literalContextBits;
props.lp = literalPosBits;
props.pb = posBits;
props.algo = algorithm;
props.fb = fastBytes;
// props.btMode = 1;
// props.numHashBytes = 4;
// props.mc = 32;
props.writeEndMark = eos ? 1 : 0;
props.numThreads = multithreading ? 2 : 1;
LzmaEncProps_Normalize(&props);
res = LzmaEnc_SetProps(encoder, &props);
if (res != SZ_OK) {
PyErr_Format(PyExc_TypeError, "could not set encoder properties: %d", res);
goto exit;
}
Py_BEGIN_ALLOW_THREADS
LzmaEnc_WriteProperties(encoder, header, &headerSize);
if (outStream.s.Write(&outStream, header, headerSize) != headerSize) {
res = SZ_ERROR_WRITE;
} else {
res = LzmaEnc_Encode(encoder, &outStream.s, &inStream.s, NULL, &allocator, &allocator);
}
Py_END_ALLOW_THREADS
if (res != SZ_OK) {
PyErr_Format(PyExc_TypeError, "Error during compressing: %d", res);
goto exit;
}
result = PyBytes_FromStringAndSize((const char *) outStream.data, outStream.size);
exit:
if (encoder != NULL) {
LzmaEnc_Destroy(encoder, &allocator, &allocator);
}
if (outStream.data != NULL) {
free(outStream.data);
}
return result;
}
static int
pylzma_compfile_init(CCompressionFileObject *self, PyObject *args, PyObject *kwargs)
{
PyObject *inFile;
CLzmaEncProps props;
Byte header[LZMA_PROPS_SIZE];
size_t headerSize = LZMA_PROPS_SIZE;
int result = -1;
// possible keywords for this function
static char *kwlist[] = {"infile", "dictionary", "fastBytes", "literalContextBits",
"literalPosBits", "posBits", "algorithm", "eos", "multithreading", "matchfinder", NULL};
int dictionary = 23; // [0,28], default 23 (8MB)
int fastBytes = 128; // [5,255], default 128
int literalContextBits = 3; // [0,8], default 3
int literalPosBits = 0; // [0,4], default 0
int posBits = 2; // [0,4], default 2
int eos = 1; // write "end of stream" marker?
int multithreading = 1; // use multithreading if available?
char *matchfinder = NULL; // matchfinder algorithm
int algorithm = 2;
int res;
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|iiiiiiiis", kwlist, &inFile, &dictionary, &fastBytes,
&literalContextBits, &literalPosBits, &posBits, &algorithm, &eos, &multithreading, &matchfinder))
return -1;
CHECK_RANGE(dictionary, 0, 28, "dictionary must be between 0 and 28");
CHECK_RANGE(fastBytes, 5, 255, "fastBytes must be between 5 and 255");
CHECK_RANGE(literalContextBits, 0, 8, "literalContextBits must be between 0 and 8");
CHECK_RANGE(literalPosBits, 0, 4, "literalPosBits must be between 0 and 4");
CHECK_RANGE(posBits, 0, 4, "posBits must be between 0 and 4");
CHECK_RANGE(algorithm, 0, 2, "algorithm must be between 0 and 2");
if (matchfinder != NULL) {
#if (PY_VERSION_HEX >= 0x02050000)
PyErr_WarnEx(PyExc_DeprecationWarning, "matchfinder selection is deprecated and will be ignored", 1);
#else
PyErr_Warn(PyExc_DeprecationWarning, "matchfinder selection is deprecated and will be ignored");
#endif
}
if (PyString_Check(inFile)) {
// create new cStringIO object from string
inFile = PycStringIO->NewInput(inFile);
if (inFile == NULL)
{
PyErr_NoMemory();
return -1;
}
} else if (!PyObject_HasAttrString(inFile, "read")) {
PyErr_SetString(PyExc_ValueError, "first parameter must be a file-like object");
return -1;
} else {
// protect object from being refcounted out...
Py_INCREF(inFile);
}
self->encoder = LzmaEnc_Create(&allocator);
if (self->encoder == NULL) {
Py_DECREF(inFile);
PyErr_NoMemory();
return -1;
}
LzmaEncProps_Init(&props);
props.dictSize = 1 << dictionary;
props.lc = literalContextBits;
props.lp = literalPosBits;
props.pb = posBits;
props.algo = algorithm;
props.fb = fastBytes;
// props.btMode = 1;
// props.numHashBytes = 4;
// props.mc = 32;
props.writeEndMark = eos ? 1 : 0;
props.numThreads = multithreading ? 2 : 1;
LzmaEncProps_Normalize(&props);
res = LzmaEnc_SetProps(self->encoder, &props);
if (res != SZ_OK) {
Py_DECREF(inFile);
PyErr_Format(PyExc_TypeError, "could not set encoder properties: %d", res);
return -1;
}
self->inFile = inFile;
CreatePythonInStream(&self->inStream, inFile);
CreateMemoryOutStream(&self->outStream);
LzmaEnc_WriteProperties(self->encoder, header, &headerSize);
if (self->outStream.s.Write(&self->outStream, header, headerSize) != headerSize) {
PyErr_SetString(PyExc_TypeError, "could not generate stream header");
goto exit;
}
LzmaEnc_Prepare(self->encoder, &self->inStream.s, &self->outStream.s, &allocator, &allocator);
result = 0;
exit:
return result;
}
static SRes Encode(ISeqOutStream *outStream, ISeqInStream *inStream, UInt64 fileSize)
{
SRes res;
size_t inSize = (size_t)fileSize;
Byte *inBuffer = 0;
Byte *outBuffer = 0;
Byte *filteredStream = 0;
size_t outSize;
CLzmaEncProps props;
LzmaEncProps_Init(&props);
LzmaEncProps_Normalize(&props);
if (inSize != 0) {
inBuffer = (Byte *)MyAlloc(inSize);
if (inBuffer == 0)
return SZ_ERROR_MEM;
} else {
return SZ_ERROR_INPUT_EOF;
}
if (SeqInStream_Read(inStream, inBuffer, inSize) != SZ_OK) {
res = SZ_ERROR_READ;
goto Done;
}
// we allocate 105% of original size + 64KB for output buffer
outSize = (size_t)fileSize / 20 * 21 + (1 << 16);
outBuffer = (Byte *)MyAlloc(outSize);
if (outBuffer == 0) {
res = SZ_ERROR_MEM;
goto Done;
}
{
int i;
for (i = 0; i < 8; i++)
outBuffer[i + LZMA_PROPS_SIZE] = (Byte)(fileSize >> (8 * i));
}
if (mConType != NoConverter)
{
filteredStream = (Byte *)MyAlloc(inSize);
if (filteredStream == 0) {
res = SZ_ERROR_MEM;
goto Done;
}
memcpy(filteredStream, inBuffer, inSize);
if (mConType == X86Converter) {
{
UInt32 x86State;
x86_Convert_Init(x86State);
x86_Convert(filteredStream, (SizeT) inSize, 0, &x86State, 1);
}
}
}
{
size_t outSizeProcessed = outSize - LZMA_HEADER_SIZE;
size_t outPropsSize = LZMA_PROPS_SIZE;
res = LzmaEncode(outBuffer + LZMA_HEADER_SIZE, &outSizeProcessed,
mConType != NoConverter ? filteredStream : inBuffer, inSize,
&props, outBuffer, &outPropsSize, 0,
NULL, &g_Alloc, &g_Alloc);
if (res != SZ_OK)
goto Done;
outSize = LZMA_HEADER_SIZE + outSizeProcessed;
}
if (outStream->Write(outStream, outBuffer, outSize) != outSize)
res = SZ_ERROR_WRITE;
Done:
MyFree(outBuffer);
MyFree(inBuffer);
MyFree(filteredStream);
return res;
}
