/* compress from THE BEGINNING up to the cursor.
everything after the cursor is thrown away */
inline int
mummy_string_compress(mummy_string *str) {
char *output, *temp;
int compressed;
/* already been compressed */
if (str->data[0] & 0x80) return 0;
/* too small. don't bother compressing, it can't possibly be worth it */
if (str->offset <= 6) return 0;
if (!(output = malloc(str->offset - 1))) return ENOMEM;
if (0 >= (compressed = lzf_compress(str->data + 1, str->offset - 1,
output + 5, str->offset - 6))) {
free(output);
return 0;
}
/* realloc the output buffer down to be a snug fit */
if (compressed < str->offset - 6) {
temp = realloc(output, compressed + 5);
if (NULL != temp) output = temp;
}
output[0] = str->data[0] | 0x80;
*(uint32_t *)(output + 1) = htonl(str->offset - 1);
free(str->data);
str->data = output;
str->offset = str->len = compressed + 5;
return 0;
}
bool KOSocket::SendCompressed(Packet * pkt)
{
if (pkt->size() < 500)
return Send(pkt);
Packet result(WIZ_COMPRESS_PACKET);
uint32 inLength = pkt->size() + 1, outLength = inLength + LZF_MARGIN, crc;
uint8 *buffer = new uint8[inLength], *outBuffer = new uint8[outLength];
*buffer = pkt->GetOpcode();
if (pkt->size() > 0)
memcpy(buffer + 1, pkt->contents(), pkt->size());
crc = (uint32)crc32(buffer, inLength);
outLength = lzf_compress(buffer, inLength, outBuffer, outLength);
#if __VERSION >= 1800 // 32-bit
result << outLength << inLength;
#else // 16-bit
result << uint16(outLength) << uint16(inLength);
#endif
result << uint32(crc);
result.append(outBuffer, outLength);
delete [] buffer;
delete [] outBuffer;
return Send(&result);
}
bool SFCompressLzf::Compress(BYTE* pDest, int& DestLen, BYTE* pSrc, int SrcLen)
{
int Size = lzf_compress(pSrc, SrcLen, pDest, DestLen);
DestLen = Size;
//int Result = fastlz_compress_level(2, (pSrc, SrcLen, pDest, DestLen);
return Size > 0;
}
extern int
lzf_encode
(const unsigned char *in_start, size_t in_len, unsigned char *out_start, size_t *pout_len, size_t out_max)
{
size_t x = lzf_compress (in_start, (unsigned)in_len, out_start, (unsigned)(in_len-1) /* ensures best compression */);
TB_DUMMY_unused = out_max;
if (x != 0)
*pout_len = (size_t) x;
return x != 0;
}
static bool lzfZipCol(int i, cr_t *cr, cz_t *cz, uint32 *tlen, uint32 *mtlen) {
INCRBY(*tlen, cr->slens[i]);
uint32 mlen = MAX(4, cr->slens[i] + 4);
uint32 n = cz->lzf_n;
cz->lzf_s[n] = malloc(mlen); /* FREE ME 034 */
cz->lzf_l[n] = lzf_compress(cr->strs[i], cr->slens[i], cz->lzf_s[n], mlen);
if (!cz->lzf_l[n]) return 0;
cz->lsocl[n] = _cr8Icol(cr->slens[i], &ucdum, &cz->socl[n]);
INCRBY(*mtlen, (cz->lsocl[cz->lzf_n] + cz->lzf_l[cz->lzf_n]));
INCR(cz->lzf_n)
return 1;
}
static PyObject *
python_compress(PyObject *self, PyObject *args) {
char *input, *output;
Py_ssize_t inlen;
PyObject *pyoutlen = Py_None;
long outlen;
PyObject *result;
if (!PyArg_ParseTuple(args, "s#|O", &input, &inlen, &pyoutlen))
return NULL;
if (pyoutlen == Py_None)
outlen = inlen - 1;
else if (PyInt_CheckExact(pyoutlen))
outlen = PyInt_AsLong(pyoutlen);
else if (PyLong_CheckExact(pyoutlen))
outlen = PyLong_AsLong(pyoutlen);
else {
PyErr_SetString(PyExc_TypeError, "max_len must be an integer");
return NULL;
}
if (inlen == 1) outlen++; /* work around for what looks like a liblzf bug */
if (outlen <= 0) {
PyErr_SetString(PyExc_ValueError, "max_len must be > 0");
return NULL;
}
output = (char *)malloc(outlen + 1);
if (output == NULL) {
PyErr_SetString(PyExc_MemoryError, "out of memory");
return NULL;
}
outlen = lzf_compress(input, inlen, output, outlen + 1);
if (outlen)
result = PYBYTES_FSAS(output, outlen);
else {
Py_XINCREF(Py_None);
result = Py_None;
}
free(output);
return result;
}
static int
compress_fd (int from, int to)
{
ssize_t us, cs, len;
u8 buf1[MAX_BLOCKSIZE + MAX_HDR_SIZE + 16];
u8 buf2[MAX_BLOCKSIZE + MAX_HDR_SIZE + 16];
u8 *header;
nr_read = nr_written = 0;
while ((us = rread (from, &buf1[MAX_HDR_SIZE], blocksize)) > 0)
{
cs = lzf_compress (&buf1[MAX_HDR_SIZE], us, &buf2[MAX_HDR_SIZE], us > 4 ? us - 4 : us);
if (cs)
{
header = &buf2[MAX_HDR_SIZE - TYPE1_HDR_SIZE];
header[0] = 'Z';
header[1] = 'V';
header[2] = 1;
header[3] = cs >> 8;
header[4] = cs & 0xff;
header[5] = us >> 8;
header[6] = us & 0xff;
len = cs + TYPE1_HDR_SIZE;
}
else
{ // write uncompressed
header = &buf1[MAX_HDR_SIZE - TYPE0_HDR_SIZE];
header[0] = 'Z';
header[1] = 'V';
header[2] = 0;
header[3] = us >> 8;
header[4] = us & 0xff;
len = us + TYPE0_HDR_SIZE;
}
if (raw)
{
header = &buf2[MAX_HDR_SIZE];
len = cs;
}
if (wwrite (to, header, len) == -1)
return -1;
}
int64_t lzbench_lzf_compress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t level, size_t, char*)
{
if (level == 0)
return lzf_compress(inbuf, insize, outbuf, outsize);
return lzf_compress_very(inbuf, insize, outbuf, outsize);
}
bool CompressionSuite::Compress(const char* inputFileName, const Algorithm algorithm, const char* outputFileName, CompressionStats* pStats)
{
if (pStats)
{
pStats->UncompressedDataSize = 0;
pStats->CompressedDataSize = 0;
pStats->TemporaryBufferSize = 0;
pStats->ElapsedTime = 0.0;
}
struct _stat st;
_stat(inputFileName, &st);
int uncompressedDataSize = st.st_size;
if (pStats)
pStats->UncompressedDataSize = uncompressedDataSize;
char* pInputBuffer = new char[uncompressedDataSize];
{
FILE* f = fopen(inputFileName, "rb");
if (!f)
{
delete[] pInputBuffer;
return false;
}
fread(pInputBuffer, uncompressedDataSize, 1, f);
fclose(f);
}
int temporaryBufferSize = 0;
if (algorithm == ALG_DOBOZ)
{
temporaryBufferSize = (int)doboz::Compressor::getMaxCompressedSize(uncompressedDataSize);
}
else if (algorithm == ALG_YAPPY)
{
// A guess... the extra 2b per block is for the 'compressed block length' for decompression.
// The extra 16 is because the sample code allocates 16 more per block, and we get memory
// stomping otherwise. :/
temporaryBufferSize = uncompressedDataSize + (uncompressedDataSize*2/kYappyBlockSize) + (uncompressedDataSize*16/kYappyBlockSize);
// And this is because we still get stomping. :/
temporaryBufferSize *= 2;
}
else if (algorithm == ALG_QUICKLZ)
{
temporaryBufferSize = uncompressedDataSize * 2;
}
else if (algorithm == ALG_FASTLZ)
{
// Must be no smaller than 66b, and at least 5% larger than uncompressed data
temporaryBufferSize = 66 + uncompressedDataSize * 2;
}
else if (algorithm == ALG_LZF)
{
temporaryBufferSize = uncompressedDataSize * 2;
}
else if (algorithm == ALG_SNAPPY)
{
temporaryBufferSize = (int)snappy::MaxCompressedLength(uncompressedDataSize);
}
else if (algorithm == ALG_LZ4)
{
temporaryBufferSize = uncompressedDataSize * 2;
}
if (pStats)
pStats->TemporaryBufferSize = temporaryBufferSize;
char* pOutputBuffer = new char[temporaryBufferSize];
size_t outputSize;
double elapsedTime = 0.0;
bool result = false;
if (algorithm == ALG_DOBOZ)
{
doboz::Compressor compressor;
Timer timer;
timer.delta();
result = (compressor.compress(pInputBuffer, uncompressedDataSize, pOutputBuffer, temporaryBufferSize, outputSize) == doboz::RESULT_OK);
elapsedTime = timer.delta();
}
else if (algorithm == ALG_YAPPY)
{
Yappy_FillTables();
unsigned char* pSrcData = (unsigned char*)pInputBuffer;
unsigned char* pDstData = (unsigned char*)pOutputBuffer;
Timer timer;
timer.delta();
for (int offset=0; offset<uncompressedDataSize; offset+=kYappyBlockSize, pSrcData+=kYappyBlockSize)
{
unsigned int blockSize = ((uncompressedDataSize-offset) >= kYappyBlockSize) ? kYappyBlockSize : uncompressedDataSize-offset;
unsigned short* pCompressedBlockSize = (unsigned short*)pDstData;
pDstData += sizeof(unsigned short);
unsigned char* pBlockEnd = Yappy_Compress(pSrcData, pDstData, blockSize);
unsigned int compressedBlockSize = pBlockEnd - pDstData;
*pCompressedBlockSize = static_cast<unsigned short>(compressedBlockSize);
pDstData = pBlockEnd;
}
elapsedTime = timer.delta();
outputSize = pDstData - (unsigned char*)pOutputBuffer;
result = (outputSize <= (unsigned int)temporaryBufferSize) && (outputSize != 0);
}
else if (algorithm == ALG_QUICKLZ)
{
qlz_state_compress qlzState;
memset(&qlzState, 0, sizeof(qlzState));
Timer timer;
timer.delta();
outputSize = qlz_compress(pInputBuffer, pOutputBuffer, uncompressedDataSize, &qlzState);
elapsedTime = timer.delta();
result = (outputSize <= (unsigned int)temporaryBufferSize) && (outputSize != 0);
}
else if (algorithm == ALG_FASTLZ)
{
Timer timer;
timer.delta();
outputSize = fastlz_compress(pInputBuffer, uncompressedDataSize, pOutputBuffer);
elapsedTime = timer.delta();
result = (outputSize <= (unsigned int)temporaryBufferSize) && (outputSize != 0);
}
else if (algorithm == ALG_LZF)
{
Timer timer;
timer.delta();
outputSize = lzf_compress(pInputBuffer, uncompressedDataSize, pOutputBuffer, temporaryBufferSize);
elapsedTime = timer.delta();
result = (outputSize <= (unsigned int)temporaryBufferSize) && (outputSize != 0);
}
else if (algorithm == ALG_SNAPPY)
{
Timer timer;
timer.delta();
snappy::RawCompress(pInputBuffer, uncompressedDataSize, pOutputBuffer, &outputSize);
elapsedTime = timer.delta();
result = (outputSize <= (unsigned int)temporaryBufferSize) && (outputSize != 0);
}
else if (algorithm == ALG_LZ4)
{
Timer timer;
timer.delta();
outputSize = LZ4_compress(pInputBuffer, pOutputBuffer, uncompressedDataSize);
elapsedTime = timer.delta();
result = (outputSize <= (unsigned int)temporaryBufferSize) && (outputSize != 0);
}
if (pStats)
pStats->ElapsedTime = elapsedTime;
if (!result)
{
delete[] pOutputBuffer;
delete[] pInputBuffer;
return false;
}
int compressedDataSize = (int)outputSize;
if (pStats)
pStats->CompressedDataSize = compressedDataSize;
FILE* of = fopen(outputFileName, "wb");
if (!of)
{
delete[] pOutputBuffer;
delete[] pInputBuffer;
return false;
}
FileHeader header;
header.Algorithm = algorithm;
header.UncompressedDataSize = uncompressedDataSize;
fwrite(&header, sizeof(header), 1, of);
fwrite(pOutputBuffer, compressedDataSize, 1, of);
fclose(of);
delete[] pOutputBuffer;
delete[] pInputBuffer;
return true;
}
unsigned int LZFcompress_data (const char* input, u8 *outputBuffer)
{
unsigned int us, cs, len;
size_t length;
u8 buf1[MAX_BLOCKSIZE + MAX_HDR_SIZE + 16];
u8 buf2[MAX_BLOCKSIZE + MAX_HDR_SIZE + 16];
u8 *header;
size_t read = 0, byteToRead = 0, chunkSize = 3000;
int i = 0;
unsigned int outputPtr = 0;
length = strlen(input);
do
{
while(read < length)
{
if((length-read)>chunkSize)
{
byteToRead=chunkSize;
}
else
{
byteToRead=length-read;
}
memcpy(&buf1[MAX_HDR_SIZE],input+read,byteToRead);
read += byteToRead;
us = byteToRead;
cs = lzf_compress (&buf1[MAX_HDR_SIZE], us, &buf2[MAX_HDR_SIZE], us > 4 ? us - 4 : us);
if (cs)
{
header = &buf2[MAX_HDR_SIZE - TYPE1_HDR_SIZE];
header[0] = 'Z';
header[1] = 'V';
header[2] = 1;
header[3] = cs >> 8;
header[4] = cs & 0xff;
header[5] = us >> 8;
header[6] = us & 0xff;
len = cs + TYPE1_HDR_SIZE;
}
else
{ // write uncompressed
header = &buf1[MAX_HDR_SIZE - TYPE0_HDR_SIZE];
header[0] = 'Z';
header[1] = 'V';
header[2] = 0;
header[3] = us >> 8;
header[4] = us & 0xff;
len = us + TYPE0_HDR_SIZE;
}
for(i = 0; i<len;i++,outputPtr++)
{
outputBuffer[outputPtr] = header[i];
}
}
}while(read==BLOCKSIZE);
outputBuffer[outputPtr] = '\0';
return outputPtr;
}