DataRecord* decompress_record(DataRecord *r)
{
if (r->flag & COMPRESS_FLAG) {
char scratch[QLZ_SCRATCH_DECOMPRESS];
int csize = qlz_size_compressed(r->value);
if (csize != r->vsz) {
fprintf(stderr, "broken compressed data: %d != %d, flag=%x\n", csize, r->vsz, r->flag);
goto DECOMP_END;
}
int size = qlz_size_decompressed(r->value);
char *v = malloc(size);
if (v == NULL) {
fprintf(stderr, "malloc(%d)\n", size);
goto DECOMP_END;
}
int ret = qlz_decompress(r->value, v, scratch);
if (ret != size) {
fprintf(stderr, "decompress %s failed: %d != %d\n", r->key, ret, size);
goto DECOMP_END;
}
if (r->free_value) {
free(r->value);
}
r->value = v;
r->free_value = true;
r->vsz = size;
r->flag &= ~COMPRESS_FLAG;
}
return r;
DECOMP_END:
free_record(r);
return NULL;
}
int tst_decomp(FILE *ifile, FILE *ofile, int opt_verbose)
{
char *src, *dst, *scratch;
unsigned int len;
// allocate source buffer
fseek(ifile, 0, SEEK_END);
len = ftell(ifile);
fseek(ifile, 0, SEEK_SET);
src = (char*) malloc(len);
// read file and allocate destination buffer
len = fread(src, 1, len, ifile);
len = qlz_size_decompressed(src);
if (opt_verbose)
printf("%s: uncomp block len: %d\n", progname, len);
dst = (char*) malloc(len);
// QLZ_SCRATCH_DECOMPRESS is defined in the beginning of the quicklz.h file
scratch = (char*) malloc(QLZ_SCRATCH_DECOMPRESS);
// decompress and write result
len = qlz_decompress(src, dst, scratch);
if (opt_verbose)
printf("%s: decompd len: %d\n", progname, len);
fwrite(dst, len, 1, ofile);
return 0;
}
void ness_decompress(const char *src,
uint32_t src_size,
char *dst,
uint32_t dst_size)
{
switch (src[0] & 0xF) {
case NESS_NO_COMPRESS:
memcpy(dst, src + 1, src_size - 1);
break;
case NESS_QUICKLZ_METHOD: {
uint32_t raw_size;
qlz_state_decompress *qsd;
qsd = xcalloc(1, sizeof(*qsd));
raw_size = qlz_decompress(src + 1, dst, qsd);
nassert(raw_size == dst_size);
(void)raw_size;
(void)dst_size;
xfree(qsd);
}
break;
default:
printf("no decompress support!\n");
nassert(1);
break;
}
}
static size_t
quicklz_decompressor(int level, const char *source, void *destination,
void *state)
{
if (level == 1)
{
return qlz_decompress(source, destination,
(qlz_state_decompress *)state);
}
Insist(false);
}
/*
* Decompress the TCP data of an SKB.
*/
int tcp_decompress(__u8 *ippacket, __u8 *lzbuffer,
qlz_state_decompress *state_decompress) {
struct iphdr *iph = NULL;
struct tcphdr *tcph = NULL;
__u16 oldsize = 0, newsize = 0; /* Store old, and new size of the TCP data. */
__u8 *tcpdata = NULL; /* Starting location for the TCP data. */
char message[LOGSZ];
if (DEBUG_COMPRESSION == true) {
sprintf(message, "[OpenNOP]: Entering into TCP DECOMPRESS \n");
logger(LOG_INFO, message);
}
if ((ippacket != NULL) && (NULL != state_decompress)) { // If the skb or state_decompress is NULL abort compression.
iph = (struct iphdr *) ippacket; // Access ip header.
memset(state_decompress, 0, sizeof(qlz_state_decompress));
if ((iph->protocol == IPPROTO_TCP)) { // If this is not a TCP segment abort compression.
tcph = (struct tcphdr *) (((u_int32_t *) ippacket) + iph->ihl); // Access tcp header.
oldsize = (__u16)(ntohs(iph->tot_len) - iph->ihl * 4) - tcph->doff
* 4;
tcpdata = (__u8 *) tcph + tcph->doff * 4; // Find starting location of the TCP data.
if ((oldsize > 0) && (lzbuffer != NULL)) {
newsize = qlz_decompress((char *) tcpdata, (char *) lzbuffer,
state_decompress);
memmove(tcpdata, lzbuffer, newsize); // Move decompressed data to packet.
iph->tot_len = htons(ntohs(iph->tot_len) + (newsize - oldsize));// Fix packet length.
__set_tcp_option((__u8 *) iph, 33, 3, 0); // Set compression flag to 0.
/*
tcph->seq = htonl(ntohl(tcph->seq) - 8000); // Decrease SEQ number.
*/
if (DEBUG_COMPRESSION == true) {
sprintf(
message,
"[OpenNOP] Decompressing [%d] size of data to [%d] \n",
oldsize, newsize);
logger(LOG_INFO, message);
}
// return 1;
// fruiz return amount of data expansion
return newsize-oldsize;
}
}
}
return -1;
}
int stream_decompress(FILE *ifile, FILE *ofile)
{
char *file_data, *decompressed;
size_t d, c, dc, fd_size, d_size;
qlz_state_decompress *state_decompress = (qlz_state_decompress *)malloc(sizeof(qlz_state_decompress));
// a compressed packet can be at most MAX_BUF_SIZE + BUF_BUFFER bytes if it
// was compressed with this program.
fd_size = MAX_BUF_SIZE + BUF_BUFFER;
file_data = (char*) malloc(fd_size);
// allocate decompression buffer
d_size = fd_size - BUF_BUFFER;
decompressed = (char*) malloc(d_size);
// allocate and initially zero out the scratch buffer. After this, make sure it is
// preserved across calls and never modified manually
memset(state_decompress, 0, sizeof(qlz_state_decompress));
// read 9-byte header to find the size of the entire compressed packet, and
// then read remaining packet
while((c = fread(file_data, 1, 9, ifile)) != 0)
{
// Do we need a bigger decompressed buffer? If the file was compressed
// with segments larger than the default in this program.
dc = qlz_size_decompressed(file_data);
if (dc > (fd_size - BUF_BUFFER)) {
free(file_data);
fd_size = dc + BUF_BUFFER;
file_data = (char*)malloc(fd_size);
}
// Do we need a bigger compressed buffer?
c = qlz_size_compressed(file_data);
if (c > d_size) {
free (decompressed);
d_size = c;
decompressed = (char*)malloc(d_size);
}
fread(file_data + 9, 1, c - 9, ifile);
d = qlz_decompress(file_data, decompressed, state_decompress);
fwrite(decompressed, d, 1, ofile);
}
free(decompressed);
free(state_decompress);
free(file_data);
return 0;
}
int unpack_write_callback(glc_thread_state_t *state)
{
unpack_t unpack = (unpack_t) state->ptr;
if (state->header.type == GLC_MESSAGE_LZO) {
#ifdef __LZO
__sync_fetch_and_add(&unpack->stats.pack_size, state->read_size - sizeof(glc_lzo_header_t));
memcpy(&state->header, &((glc_lzo_header_t *) state->read_data)->header,
sizeof(glc_message_header_t));
__lzo_decompress((unsigned char *) &state->read_data[sizeof(glc_lzo_header_t)],
state->read_size - sizeof(glc_lzo_header_t),
(unsigned char *) state->write_data,
(lzo_uintp) &state->write_size,
NULL);
#else
return ENOTSUP;
#endif
} else if (state->header.type == GLC_MESSAGE_QUICKLZ) {
#ifdef __QUICKLZ
__sync_fetch_and_add(&unpack->stats.pack_size,
state->read_size - sizeof(glc_quicklz_header_t));
memcpy(&state->header, &((glc_quicklz_header_t *) state->read_data)->header,
sizeof(glc_message_header_t));
if (!state->threadptr)
state->threadptr = malloc(sizeof(qlz_state_decompress));
qlz_decompress((const void *) &state->read_data[sizeof(glc_quicklz_header_t)],
(void *) state->write_data,
(qlz_state_decompress *) state->threadptr);
#else
return ENOTSUP;
#endif
} else if (state->header.type == GLC_MESSAGE_LZJB) {
#ifdef __LZJB
__sync_fetch_and_add(&unpack->stats.pack_size, state->read_size - sizeof(glc_lzjb_header_t));
memcpy(&state->header, &((glc_quicklz_header_t *) state->read_data)->header,
sizeof(glc_message_header_t));
lzjb_decompress(&state->read_data[sizeof(glc_lzjb_header_t)],
state->write_data,
state->read_size - sizeof(glc_lzjb_header_t),
state->write_size);
#else
return ENOTSUP;
#endif
} else
return ENOTSUP;
__sync_fetch_and_add(&unpack->stats.unpack_size, state->write_size);
return 0;
}
int64_t lzbench_quicklz_decompress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t level, size_t dstate, size_t)
{
switch (level)
{
default:
case 1:
return qlz_decompress_1(inbuf, outbuf, (qlz150_state_decompress*)dstate);
break;
case 2:
return qlz_decompress_2(inbuf, outbuf, (qlz150_state_decompress*)dstate);
break;
case 3:
return qlz_decompress_3(inbuf, outbuf, (qlz150_state_decompress*)dstate);
break;
case 4:
return qlz_decompress(inbuf, outbuf, (qlz_state_decompress*)dstate);
break;
}
}
void AdvImageLayout::GetDataFromDataBytes(enum GetByteMode mode, unsigned char* data, unsigned int* prevFrame, unsigned int* pixels, int sectionDataLength, int startOffset)
{
unsigned char* layoutData;
if (!m_UsesCompression)
{
layoutData = data + startOffset;
}
else if (0 == strcmp(Compression, "QUICKLZ"))
{
size_t size = qlz_size_decompressed((char*)(data + startOffset));
// MaxFrameBufferSize
qlz_decompress((char*)(data + startOffset), m_DecompressedPixels, m_StateDecompress);
layoutData = (unsigned char*)m_DecompressedPixels;
}
else if (0 == strcmp(Compression, "LAGARITH16"))
{
int size = m_Lagarith16Decompressor->DecompressData((char*)(data + startOffset), (unsigned short*)m_DecompressedPixels);
layoutData = (unsigned char*)m_DecompressedPixels;
}
bool crcOkay;
int readIndex = 0;
if (Bpp == 12)
{
GetPixelsFrom12BitByteArray(layoutData, prevFrame, pixels, mode, &readIndex, &crcOkay);
}
else if (Bpp == 16)
{
if (IsDiffCorrLayout)
GetPixelsFrom16BitByteArrayDiffCorrLayout(layoutData, prevFrame, pixels, &readIndex, &crcOkay);
else
GetPixelsFrom16BitByteArrayRawLayout(layoutData, prevFrame, pixels, &readIndex, &crcOkay);
}
else if (Bpp == 8)
{
if (IsDiffCorrLayout)
GetPixelsFrom8BitByteArrayDiffCorrLayout(layoutData, prevFrame, pixels, &readIndex, &crcOkay);
else
GetPixelsFrom8BitByteArrayRawLayout(layoutData, prevFrame, pixels, &readIndex, &crcOkay);
}
}
int64_t lzbench_quicklz_decompress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t level, size_t , char*)
{
int64_t res;
qlz150_state_compress* dstate = (qlz150_state_compress*) calloc(1, MAX(qlz_get_setting_3(2),MAX(qlz_get_setting_1(2), qlz_get_setting_2(2))));
if (!dstate)
return 0;
switch (level)
{
default:
case 1: res = qlz_decompress_1(inbuf, outbuf, (qlz150_state_decompress*)dstate); break;
case 2: res = qlz_decompress_2(inbuf, outbuf, (qlz150_state_decompress*)dstate); break;
case 3: res = qlz_decompress_3(inbuf, outbuf, (qlz150_state_decompress*)dstate); break;
case 4: res = qlz_decompress(inbuf, outbuf, (qlz_state_decompress*)dstate); break;
}
free(dstate);
return res;
}
int main(int argc, char* argv[])
{
FILE *ifile, *ofile;
char *file_data, *decompressed;
size_t d, c;
qlz_state_decompress *state_decompress = (qlz_state_decompress *)malloc(sizeof(qlz_state_decompress));
// read source file
ifile = fopen(argv[1], "rb");
ofile = fopen(argv[2], "wb");
// a compressed packet can be at most "uncompressed size" + 400 bytes large where
// "uncompressed size" = 10000 in worst case in this sample demo
file_data = (char*) malloc(10000 + 400);
// allocate decompression buffer
decompressed = (char*) malloc(10000);
// allocate and initially zero out the scratch buffer. After this, make sure it is
// preserved across calls and never modified manually
memset(state_decompress, 0, sizeof(qlz_state_decompress));
// read 9-byte header to find the size of the entire compressed packet, and
// then read remaining packet
while((c = fread(file_data, 1, 9, ifile)) != 0)
{
c = qlz_size_compressed(file_data);
fread(file_data + 9, 1, c - 9, ifile);
d = qlz_decompress(file_data, decompressed, state_decompress);
printf("%u bytes decompressed into %u.\n", (unsigned int)c, (unsigned int)d);
fwrite(decompressed, d, 1, ofile);
}
fclose(ifile);
fclose(ofile);
return 0;
}
PyObject *qlz_decompress_py(PyObject *self, PyObject *args)
{
PyObject *result = NULL;
PyObject *state = NULL;
char* buffer;
char* decompressed_buffer;
int buffer_length;
int size_decompressed;
#if PY_MAJOR_VERSION >= 3
if (PyArg_ParseTuple(args, "y#O!",
&buffer, &buffer_length,
&QLZStateDecompressType, &state))
#else
if (PyArg_ParseTuple(args, "s#O!",
&buffer, &buffer_length,
&QLZStateDecompressType, &state))
#endif
{
size_decompressed = qlz_size_decompressed(buffer);
decompressed_buffer = (char*)malloc(size_decompressed);
qlz_decompress(buffer, decompressed_buffer,
((qlz_state_decompress_Type*)state)->value);
#if PY_MAJOR_VERSION >= 3
result = Py_BuildValue("y#", decompressed_buffer, size_decompressed);
#else
result = Py_BuildValue("s#", decompressed_buffer, size_decompressed);
#endif
free(decompressed_buffer);
} /* otherwise there is an error,
* the exception already raised by PyArg_ParseTuple, and NULL is
* returned.
*/
return result;
}
/*
VAR file decompress
*/
int var_decompress(FILE *fi, FILE *fo, int verbose)
{
int r = 0;
int ok;
char *in_buf = NULL;
char *out_buf = NULL;
char *scratch = NULL;
unsigned char m [ sizeof(magic) ];
unsigned int block_size;
md5_state_t md5_state;
md5_byte_t md5_digest[16];
static const char *const block_salt = ":lzo:";
//VAR ext
var_header file;
char file_chk_str [ 33 ];
//VAR ext block
var_block_header block;
qlz_block_header qlz_block;
unsigned long new_len;
unsigned long block_next_num;
unsigned long block_count;
unsigned long block_empty;
unsigned long long block_real_pos;
unsigned long block_last_nonzero;
total_in = total_out = 0;
block_count = block_empty = 0;
block_next_num = block_last_nonzero = 0;
/*
* Step 1: check magic header, read flags & block size
*/
if (xread(fi, m, sizeof(magic),1) != sizeof(magic) ||
memcmp(m, magic, sizeof(magic)) != 0)
{
printf("%s: header error - this file is not a .var\n", progname);
r = 1;
goto err;
}
//read header
var_read_header(fi, &file);
block_size = file.block_size;
if (block_size < 1024 || block_size > 8*1024*1024L)
{
printf("%s: header error - invalid block size %u\n",
progname, block_size);
r = 3;
goto err;
}
if (verbose > 0)
printf("%s: block size %u\n",
progname, block_size);
//header debug
if (verbose > 0)
{
str_to_hexstr(file.uuid, file_chk_str);
printf("%s: version %u, bin_len %u, txt_len %u, header_end x%llx\n\tfile_flags %lx, file_uuid %s\n",
progname, file.version, file.bin_len, file.txt_len, xtell(fi), file.flags, file_chk_str);
}
/*
* Step 2: allocate buffer for in-place decompression
*/
// QLZ uses one input & one ouput buffer + a scratch one
// The decompressed length can be obtained with "qlz_size_decompressed(src)"
// but we use the fixed block size and reuse the same buffers for all blocks
//out_len = qlz_size_decompressed(in);
if (opt_debug > 0)
printf("%s: allocate 2 x %d\n", progname, block_size);
in_buf = (char*) malloc(block_size);
out_buf = (char*) malloc(block_size);
scratch = (char*) malloc(QLZ_SCRATCH_DECOMPRESS);
if (in_buf == NULL || out_buf == NULL || scratch == NULL)
{
printf("%s: out of memory\n", progname);
r = 4;
goto err;
}
/*
* Step 3: process blocks
*/
for (;;)
{
/*
Parse the VAR block header
*/
block_real_pos = xtell(fi);
var_read_block_header(fi, &block);
//NOTE: files can have no data blocks at all (all zero disk...)
//if (block_next_num > 0 && block.num == 0) {
if (block.in_len == 0) {
// file end
if (verbose > 0)
printf("%s: last block at %llx\n",\
progname, block_real_pos);
break;
}
// block header debug
if (opt_debug > 0) {
printf("%s: reading block\t%lu, pos\t%llu\tin/out: %lu/%lu\n",\
progname, block.num, xtell(fi), block.in_len, block.out_len);
str_to_hexstr(block.chk, file_chk_str);
printf("%s: checksum: %s\n", progname, file_chk_str);
}
if (block.num != block_next_num)
{
if (block.num < block_next_num)
{
printf("%s: block number error - expected >= %lu / got %lu (x%llx)\n",\
progname, block_next_num, block.num, block_real_pos);
r = 5;
goto err;
}
// bloc_num > block_next_num actually means empty blocks...
block_next_num = block.num - block_next_num;
if (opt_debug > 0)
printf("%s: writing %ld empty blocks\n", progname, block_next_num);
xwrite_empty_blocks(fo, block.out_len, block_next_num, out_buf);
//record empty blocks + reset next_num
block_empty += block_next_num;
block_next_num = block.num;
}
block_next_num += 1;
// sanity check of the size values
if (block.in_len-9 > block_size || block.out_len > block_size ||
block.in_len == 0 || block.in_len-9 > block.out_len)
{
printf("%s: block size error - data corrupted\n", progname);
printf("%s: len in / out: %lu / %lu\n",
progname, block.in_len, block.out_len);
r = 6;
goto err;
}
//check block pos
//NOTE: this this doesn't seem to be the set in v5 files...
if (block.pos && block.pos != block_real_pos)
{
printf("%s: block position error - expected %llu / got %llu\n",\
progname, block_real_pos, block.pos);
r = 7;
goto err;
}
/*
Parse the QLZ header
actually we don't need to (it's part of a standard QLZ block),
it's just for additionale checks,
but we need to rewind after that...
*/
var_read_qlz_block_header(fi, &qlz_block);
if (qlz_block.type != 0x4b && qlz_block.type != 0x4a)
{
printf("%s: block flag error - expected 4b/4a / got %x\n",\
progname, (int)qlz_block.type);
r = 8;
goto err;
}
// read qlz block uncompressed & compressed size
// it should be the same as in the VAR header
if (qlz_block.out_len != block.out_len || qlz_block.in_len != block.in_len)
{
printf("%s: block qlz len error - in %lu/%lu, out %lu/%lu\n",\
progname, block.in_len, qlz_block.in_len, block.out_len, qlz_block.out_len);
r = 9;
goto err;
}
/*
Read & decompress the QLZ block
*/
block_count++;
// rewind at the begining of the QLZ block / header
// 1 x QLZ marker + 4 x comp size + 4 x uncomp size
xskip(fi, -9);
if (opt_debug > 1)
{
// skip the block
xskip(fi, block.in_len);
}
else
{
// read the compressed block in "in_buf"
xread(fi, in_buf, block.in_len, 0);
//real decompress
// debug
//xwrite(fo, in, block.in_len);
//todo: clean scratch buffer (if QLZ_STREAMING_BUFFER > 0)
//memset(scratch_buf, 0, QLZ_SCRATCH_DECOMPRESS);
// decompress
//NOTE: define QLZ_MEMORY_SAFE to get checks...
new_len = qlz_decompress(in_buf, out_buf, scratch);
if (opt_debug > 0)
printf("%s: decomp len: %lu / %lu\n", progname, new_len, block.out_len);
if (new_len != block.out_len)
{
printf("%s: compressed data violation\n", progname);
r = 10;
goto err;
}
// write decompressed block
xwrite(fo, out_buf, block.out_len);
//check block checksum
md5_init(&md5_state);
md5_append(&md5_state, (const md5_byte_t *)block_salt, 5);
md5_append(&md5_state, (const md5_byte_t *)out_buf, block.out_len);
md5_finish(&md5_state, md5_digest);
if (opt_debug > 0)
{
str_to_hexstr((char *)md5_digest, file_chk_str);
printf("%s: verified: %s\n", progname, file_chk_str);
}
if (memcmp(block.chk, md5_digest, 16) != 0)
{
printf("%s: checksum error - block %lu corrupted\n", progname, block.num);
r = 11;
goto err;
}
}
//debug
if (opt_debug > 1)
{
if(block_next_num>1)
{
printf("bail, pos x%llx\n", xtell(fi));
goto err;
}
}
}
/*
* Step 4: process map / index
*/
//read map header
var_read_map_header(fi, &file);
//debug
if (verbose > 0)
{
str_to_hexstr(file.map_chk, file_chk_str);
printf("%s: map: len %lu, len2 %lu, pos %llx, check %s\n",
progname, file.map_len, file.map_len2, xtell(fi), file_chk_str);
}
//read all before parsing to verify...
//TODO: compute while reading blocks ? anyway it's useless...
if (opt_debug>0)
{
ok = file_check_part(fi, file.map_len, file.map_chk);
fseeko64(fi, -1 * file.map_len, SEEK_CUR);
printf("%s: map checksum %s\n", progname, (ok==0)?"OK":"ERROR");
}
/*
* Step 5: process blocks map
*/
//TODO: check // blocks...
block_next_num = 0;
block_last_nonzero = -1;
for(;;)
{
var_read_block_header(fi, &block);
//FIXME: files can have no data blocks at all (all zero disk...)
if (block_next_num > 0 && block.num == 0) {
//if (block.num == 0) {
// file end
if (verbose > 0)
printf("%s: last map entry (total %lu blocks) at %llx\n",\
progname, block_next_num, xtell(fi));
//write last empty blocks
block_next_num = block_next_num - 1 - block_last_nonzero;
if (opt_debug > 0)
printf("%s: writing %ld trailing empty blocks\n", progname, block_next_num);
xwrite_empty_blocks(fo, block_size, block_next_num, out_buf);
block_empty += block_next_num;
//end
break;
}
if (block.num != block_next_num)
{
printf("%s: map block number error - expected >= %lu / got %lu (x%llx)\n",\
progname, block_next_num, block.num, xtell(fi));
r = 12;
goto err;
}
block_next_num += 1;
block_size = block.out_len; //store previous block size
//TODO check // real last block processed
if (block.in_len > 0)
block_last_nonzero = block.num;
//debug
//if (block.out_len != block_size)
if (block.out_len > 0x40000)
printf("%s: WARNING: block_size too large: %lu\n", progname, block.out_len);
// block map debug
if (opt_debug > 0)
printf("%s: reading map\t%lu, in/out: %lu/%lu, end @ x%llx\n",\
progname, block.num, block.in_len, block.out_len, xtell(fi));
}
//read footer
var_read_footer(fi, &file);
//should be the end of file...
//debug...
if (verbose > 0)
{
const char *const chk_names[4] = {"bin", "txt", "map", "foot"};
int i;
for (i = 0; i<4; i++) {
str_to_hexstr(file.chks[i], file_chk_str);
printf("%s: %s chk: %s\n", progname, chk_names[i], file_chk_str);
}
printf("%s: footer info %lu / %lu / %lu\n",
progname, file.foot1, file.foot2, file.foot3);
}
//TODO: check those md5 before reading the file...
//...but that implies reading the file end first
if (opt_debug > 0)
{
//chk 0: bin header: from start for bin_len
fseeko64(fi, 0, SEEK_SET);
ok = file_check_part(fi, file.bin_len, file.chks[0]);
printf("%s: binary header checksum %s\n", progname, (ok==0)?"OK":"ERROR");
//chk 1: txt header: from bin header for txt_len
ok = file_check_part(fi, file.txt_len, file.chks[1]);
printf("%s: text header checksum %s\n", progname, (ok==0)?"OK":"ERROR");
//chk 2: map dupe: full map...
ok = memcmp(file.map_chk, file.chks[2], 16);
printf("%s: map dupe checksum %s\n", progname, (ok==0)?"OK":"ERROR");
//chk 3: footer: last 12 bytes
fseeko64(fi, -12, SEEK_END);
ok = file_check_part(fi, 12, file.chks[3]);
printf("%s: footer checksum %s\n", progname, (ok==0)?"OK":"ERROR");
}
if (verbose > 0)
printf("%s: blocks decomp / empty / total: %lu / %lu / %lu\n",\
progname, block_count, block_empty, block_count + block_empty);
r = 0;
err:
free(in_buf);
free(out_buf);
free(scratch);
return r;
}
int RecvPack(T_Connect *connect,T_NetHead *nethead)
{
char headbuf[HEADPACKLENGTH+1],addr[16];
int i,n;
u_int crc;
char *zbuf;
memset(nethead->para,0,sizeof(nethead->para));
nethead->data=NULL;
peeraddr(connect->Socket,addr);
i=RecvNet(connect->Socket,headbuf,HEADPACKLENGTH,connect->timeout);
if(i<HEADPACKLENGTH){
if(i==TIMEOUTERR) {
ShowLog(1,"%s:head TIMEOUT %d second's",__FUNCTION__,
connect->timeout);
return i;
}
ShowLog(1,"RecvPack Head LENERR i=%d,err=%d,%s",i,errno,strerror(errno));
return LENGERR;
}
if(connect->CryptFlg & DO_CRYPT)
enigma2_decrypt(&connect->t,headbuf,HEADPACKLENGTH);
i=NetHeadDispack(nethead,headbuf,connect->family[29]);
if(i!=0) {
ShowLog(1,"aft NetHeadDispack len=%d,PKGERR %s",i, addr);
showpack(1,headbuf,HEADPACKLENGTH);
return(FORMATERR);
}
if(!nethead->T_LEN) return 0;
if(connect->CryptFlg&UNDO_ZIP) {
i=nethead->T_LEN+1;
} else {
i=nethead->PKG_LEN+1;
if(nethead->T_LEN <nethead->PKG_LEN) i+=nethead->T_LEN+1;
}
n=connect->RecvLen-i;
if(!connect->RecvBuffer || n<0 || n>32768){
if(connect->RecvBuffer) {
free(connect->RecvBuffer);
connect->RecvBuffer=0;
}
connect->RecvLen=0;
connect->RecvBuffer=malloc(i);
if(!connect->RecvBuffer) return MEMERR;
connect->RecvLen=i;
}
if(!(connect->CryptFlg&UNDO_ZIP) && nethead->T_LEN != nethead->PKG_LEN) {
zbuf=connect->RecvBuffer+nethead->PKG_LEN+1;
} else zbuf=connect->RecvBuffer;
i=RecvNet(connect->Socket,zbuf,nethead->T_LEN,5);
if(i != (nethead->T_LEN)) {
if(TIMEOUTERR == i) {
ShowLog(1,"%s:recv body TIMEOUT",__FUNCTION__);
return i;
}
ShowLog(1,"%s,Recv Body T_LEN=%d i=%d,errno=%d",__FUNCTION__,
nethead->T_LEN,i,errno);
free(connect->RecvBuffer);
connect->RecvBuffer=0;
connect->RecvLen=0;
return i<0?SYSERR:LENGERR;
}
crc=ssh_crc32((const unsigned char *)zbuf, nethead->T_LEN);
if((connect->CryptFlg & CHECK_CRC) && (crc != nethead->PKG_CRC)) {
ShowLog(1,"RecvPack:PKG_CRC=%08X,crc=%08X,PKG_LEN=%d,T_LEN=%d,head=%s",
nethead->PKG_CRC,crc,nethead->PKG_LEN,nethead->T_LEN,headbuf);
return CRCERR;
}
pack_decode(zbuf, nethead->T_LEN,connect);
if(zbuf != connect->RecvBuffer) {
size_t sz=qlz_size_decompressed(zbuf);
if(nethead->T_LEN<9 || nethead->PKG_LEN != sz) {
ShowLog(1,"unzip error T_LEN=%d,sz=%ld,PKG_LEN=%d,ADDR=%s",nethead->T_LEN, sz,nethead->PKG_LEN,addr);
return FORMATERR;
}
i=qlz_decompress(zbuf,connect->RecvBuffer);
if(i!=nethead->PKG_LEN) {
ShowLog(1,"RecvPack:PKG_LEN=%d,T_LEN=%d,unzip_size=%d",
nethead->PKG_LEN,nethead->T_LEN,i);
return LENGERR;
}
} //else if(connect->CryptFlg & UNDO_ZIP)
// ShowLog(5,"%s:UNDO_ZIP %s,T_LEN=%d,PKG_LEN=%d,data=%s",__FUNCTION__,
// *connect->Host?connect->Host:"Client",nethead->T_LEN,nethead->PKG_LEN,connect->RecvBuffer);
if(!(connect->CryptFlg & UNDO_ZIP)) connect->RecvBuffer[nethead->PKG_LEN]=0;
nethead->data=connect->RecvBuffer;
return 0;
}
int RecvPack(T_Connect *connect,T_NetHead *nethead)
{
char headbuf[HEADPACKLENGTH+1],addr[16];
int i,n;
u_int crc;
char *zbuf;
n_zero(PARANUM,nethead->para);
nethead->data=0;
memset(headbuf,0,sizeof(headbuf));
i=RecvNet(connect->Socket,headbuf,HEADPACKLENGTH,connect->timeout);
//free SendBuffer
if(i<HEADPACKLENGTH){
if(i==TIMEOUTERR) {
ShowLog(1,"%s:head TIMEOUT %d second's",__FUNCTION__,
connect->timeout);
return i;
}
ShowLog(1,"RecvPack Head LENERR i=%d,err=%d,%s",i,errno,strerror(errno));
return LENGERR;
}
headbuf[HEADPACKLENGTH]=0;
i=NetHeadDispack(nethead,headbuf,connect->family[29]);
if(i!=0) {
peeraddr(connect->Socket,addr);
ShowLog(1,"aft NetHeadDispack len=%d,PKGERR %s:%.48s",i, addr,headbuf);
showpack(1,headbuf,HEADPACKLENGTH);
return(FORMATERR);
}
if(!nethead->T_LEN) return 0;
i=((connect->CryptFlg&UNDO_ZIP)?nethead->T_LEN:nethead->PKG_LEN)+1;
n=connect->RecvLen-i;
if(n<0 || n>SDBC_BLKSZ){
if( connect->RecvBuffer)
free(connect->RecvBuffer);
connect->RecvBuffer=0;
connect->RecvLen=0;
connect->RecvBuffer=malloc(i);
if(!connect->RecvBuffer) return MEMERR;
connect->RecvLen=i;
}
if(!(connect->CryptFlg&UNDO_ZIP) && nethead->T_LEN != nethead->PKG_LEN) {
zbuf=malloc(nethead->T_LEN);
if(!zbuf) {
RecvNet(connect->Socket,connect->RecvBuffer,nethead->T_LEN,3);
return MEMERR;
}
} else zbuf=connect->RecvBuffer;
i=RecvNet(connect->Socket,zbuf,nethead->T_LEN,3);
if(i < (nethead->T_LEN)) {
if(TIMEOUTERR == i) {
ShowLog(1,"%s:recv body TIMEOUT",__FUNCTION__);
return i;
}
if(zbuf!=connect->RecvBuffer) free(zbuf);
ShowLog(1,"%s,Recv Body T_LEN=%d i=%d,errno=%d",__FUNCTION__,
nethead->T_LEN,i,errno);
free(connect->RecvBuffer);
connect->RecvBuffer=0;
connect->RecvLen=0;
return i<0?SYSERR:LENGERR;
}
crc=ssh_crc32((const unsigned char *)zbuf, nethead->T_LEN);
if((connect->CryptFlg & CHECK_CRC) && (crc != nethead->PKG_CRC)) {
ShowLog(1,"RecvPack:PKG_CRC=%08X,crc=%08X,PKG_LEN=%d,T_LEN=%d,head=%s",
nethead->PKG_CRC,crc,nethead->PKG_LEN,nethead->T_LEN,headbuf);
return CRCERR;
}
pack_decode(zbuf, nethead->T_LEN,connect);
if(zbuf != connect->RecvBuffer) {
if(nethead->T_LEN<9 || nethead->PKG_LEN != qlz_size_decompressed(zbuf)) {
ShowLog(1,"unzip error T_LEN=%d,ADDR=%s",nethead->T_LEN, addr);
return FORMATERR;
}
i=qlz_decompress(zbuf,connect->RecvBuffer);
free(zbuf);
if(i!=nethead->PKG_LEN) {
ShowLog(1,"RecvPack:PKG_LEN=%d,T_LEN=%d,unzip_size=%d",
nethead->PKG_LEN,nethead->T_LEN,i);
return LENGERR;
}
}
connect->RecvBuffer[nethead->PKG_LEN]=0;
nethead->data=connect->RecvBuffer;
return 0;
}
int MDFN_StateEvil(int rewind)
{
if(!EvilEnabled)
return(0);
if(rewind)
{
int32 next_bcspos = bcspos;
bool NeedDataFlush = FALSE;
bcspos--;
if(bcspos < 0) bcspos += SRW_NUM;
if(!bcs[bcspos].data)
bcspos = (bcspos + 1) % SRW_NUM;
else
NeedDataFlush = TRUE;
if(bcs[bcspos].compressed_len)
{
uint8 *tmp_buf;
lzo_uint dst_len = bcs[bcspos].uncompressed_len;
tmp_buf = (uint8 *)malloc(bcs[bcspos].uncompressed_len);
if(SRWCompressor == SRW_COMPRESSOR_QUICKLZ)
{
//static char workmem[QLZ_SCRATCH_DECOMPRESS];
dst_len = qlz_decompress((char*)bcs[bcspos].data, tmp_buf); //, workmem);
}
else if(SRWCompressor == SRW_COMPRESSOR_MINILZO)
lzo1x_decompress(bcs[bcspos].data, bcs[bcspos].compressed_len, tmp_buf, &dst_len, NULL);
else if(SRWCompressor == SRW_COMPRESSOR_BLZ)
{
dst_len = blz_unpack(bcs[bcspos].data, tmp_buf);
}
for(uint32 x = 0; x < bcs[bcspos].uncompressed_len && x < bcs[next_bcspos].uncompressed_len; x++)
tmp_buf[x] ^= bcs[next_bcspos].data[x];
free(bcs[bcspos].data);
bcs[bcspos].data = tmp_buf;
bcs[bcspos].compressed_len = 0;
}
if(NeedDataFlush)
{
if(bcs[next_bcspos].MovieLove.data)
{
free(bcs[next_bcspos].MovieLove.data);
bcs[next_bcspos].MovieLove.data = NULL;
}
free(bcs[next_bcspos].data);
bcs[next_bcspos].data = NULL;
bcs[next_bcspos].compressed_len = 0;
bcs[next_bcspos].uncompressed_len = 0;
}
if(bcs[bcspos].uncompressed_len)
{
StateMem sm;
sm.data = bcs[bcspos].data;
sm.loc = 0;
sm.initial_malloc = 0;
sm.malloced = sm.len = bcs[bcspos].uncompressed_len;
MDFNSS_LoadSM(&sm, 0, 1);
// free(MDFNMOV_GrabRewindJoy().data);
return(1);
}
}
else
{
StateMem sm;
int32 prev_bcspos = bcspos;
bcspos = (bcspos + 1) % SRW_NUM;
// if(MDFNMOV_IsRecording())
{
if(bcs[bcspos].data && bcs[bcspos].MovieLove.data)
{
//printf("Force: %d\n", bcspos);
// MDFNMOV_ForceRecord(&bcs[bcspos].MovieLove);
free(bcs[bcspos].MovieLove.data);
bcs[bcspos].MovieLove.data = NULL;
}
}
if(bcs[bcspos].data)
{
free(bcs[bcspos].data);
bcs[bcspos].data = NULL;
}
if(bcs[bcspos].MovieLove.data)
{
free(bcs[bcspos].MovieLove.data);
bcs[bcspos].MovieLove.data = NULL;
}
memset(&sm, 0, sizeof(sm));
// MDFNSS_SaveSM(&sm, 0, 1);
bcs[bcspos].data = sm.data;
bcs[bcspos].compressed_len = 0;
bcs[bcspos].uncompressed_len = sm.len;
// Compress the previous save state.
if(bcs[prev_bcspos].data)
{
for(uint32 x = 0; x < bcs[prev_bcspos].uncompressed_len && x < sm.len; x++)
bcs[prev_bcspos].data[x] ^= sm.data[x];
if(SRWCompressor == SRW_COMPRESSOR_QUICKLZ)
{
//static char workmem[QLZ_SCRATCH_COMPRESS];
uint8 *tmp_buf = (uint8 *)malloc(bcs[prev_bcspos].uncompressed_len + 36000);
uint32 dst_len = bcs[prev_bcspos].uncompressed_len + 36000;
dst_len = qlz_compress(bcs[prev_bcspos].data, (char*)tmp_buf, bcs[prev_bcspos].uncompressed_len); //, workmem);
free(bcs[prev_bcspos].data);
bcs[prev_bcspos].data = (uint8 *)realloc(tmp_buf, dst_len);
bcs[prev_bcspos].compressed_len = dst_len;
}
else if(SRWCompressor == SRW_COMPRESSOR_MINILZO)
{
uint8 workmem[LZO1X_1_MEM_COMPRESS];
uint8 * tmp_buf = (uint8 *)malloc((size_t)(1.10 * bcs[prev_bcspos].uncompressed_len));
lzo_uint dst_len = (lzo_uint)(1.10 * bcs[prev_bcspos].uncompressed_len);
lzo1x_1_compress(bcs[prev_bcspos].data, bcs[prev_bcspos].uncompressed_len, tmp_buf, &dst_len, workmem);
free(bcs[prev_bcspos].data);
bcs[prev_bcspos].data = (uint8 *)realloc(tmp_buf, dst_len);
bcs[prev_bcspos].compressed_len = dst_len;
}
else if(SRWCompressor == SRW_COMPRESSOR_BLZ)
{
blz_pack_t workmem;
uint8 * tmp_buf = (uint8 *)malloc((size_t)(bcs[prev_bcspos].uncompressed_len + blz_pack_extra));
uint32 dst_len = bcs[prev_bcspos].uncompressed_len + blz_pack_extra;
dst_len = blz_pack(bcs[prev_bcspos].data, bcs[prev_bcspos].uncompressed_len, tmp_buf, &workmem);
free(bcs[prev_bcspos].data);
bcs[prev_bcspos].data = (uint8 *)realloc(tmp_buf, dst_len);
bcs[prev_bcspos].compressed_len = dst_len;
}
}
// if(MDFNMOV_IsRecording())
// bcs[bcspos].MovieLove = MDFNMOV_GrabRewindJoy();
}
return(0);
}
bool CompressionSuite::Decompressor::Decompress(char* pOutputBuffer, const int outputBufferSize, CompressionStats* pStats)
{
assert(mHeader.Algorithm != ALG_Invalid);
int uncompressedDataSize = mHeader.UncompressedDataSize;
if (pStats)
{
pStats->CompressedDataSize = mCompressedDataSize;
pStats->UncompressedDataSize = uncompressedDataSize;
}
if (uncompressedDataSize > outputBufferSize)
return false;
double elapsedTime = 0.0;
bool result = false;
if (mHeader.Algorithm == ALG_DOBOZ)
{
doboz::Decompressor decompressor;
Timer timer;
timer.delta();
result = (decompressor.decompress(mpInputBuffer, mCompressedDataSize, pOutputBuffer, uncompressedDataSize) == doboz::RESULT_OK);
elapsedTime = timer.delta();
}
else if (mHeader.Algorithm == ALG_YAPPY)
{
Timer timer;
timer.delta();
unsigned char* pSrcData = (unsigned char*)mpInputBuffer;
unsigned char* pSrcDataEnd = (unsigned char*)mpInputBuffer + mCompressedDataSize;
unsigned char* pDstData = (unsigned char*)pOutputBuffer;
while (pSrcData < pSrcDataEnd)
{
unsigned short compressedBlockSize = *reinterpret_cast<unsigned short*>(pSrcData);
pSrcData += sizeof(unsigned short);
unsigned char* pSrcBlockEnd = pSrcData + compressedBlockSize;
unsigned char* pDstBlockEnd = Yappy_UnCompress(pSrcData, pSrcBlockEnd, pDstData);
pDstData = pDstBlockEnd;
pSrcData = pSrcBlockEnd;
};
elapsedTime = timer.delta();
result = ((pDstData == ((unsigned char*)pOutputBuffer + uncompressedDataSize)) &&
(pSrcData == pSrcDataEnd));
}
else if (mHeader.Algorithm == ALG_QUICKLZ)
{
qlz_state_decompress qlzState;
memset(&qlzState, 0, sizeof(qlzState));
Timer timer;
timer.delta();
unsigned int outSize = qlz_decompress(mpInputBuffer, pOutputBuffer, &qlzState);
elapsedTime = timer.delta();
result = (outSize == uncompressedDataSize);
}
else if (mHeader.Algorithm == ALG_FASTLZ)
{
Timer timer;
timer.delta();
int outSize = fastlz_decompress(mpInputBuffer, mCompressedDataSize, pOutputBuffer, uncompressedDataSize);
elapsedTime = timer.delta();
result = (outSize == uncompressedDataSize);
}
else if (mHeader.Algorithm == ALG_LZF)
{
Timer timer;
timer.delta();
int outSize = lzf_decompress(mpInputBuffer, mCompressedDataSize, pOutputBuffer, uncompressedDataSize);
elapsedTime = timer.delta();
result = (outSize == uncompressedDataSize);
}
else if (mHeader.Algorithm == ALG_SNAPPY)
{
Timer timer;
timer.delta();
result = snappy::RawUncompress(mpInputBuffer, mCompressedDataSize, pOutputBuffer);
elapsedTime = timer.delta();
}
else if (mHeader.Algorithm == ALG_LZ4)
{
Timer timer;
timer.delta();
int outSize = LZ4_decode(mpInputBuffer, pOutputBuffer, mCompressedDataSize);
elapsedTime = timer.delta();
result = (outSize == uncompressedDataSize);
}
if (pStats)
pStats->ElapsedTime = elapsedTime;
FreeCompressedData();
return result;
}
int main(int argc, char* argv[])
{
input_file = stdin;
output_file = stdout;
char *src = 0;
char *dst = 0;
size_t input_file_size = 0;
size_t output_file_size = 0;
char inputBuffer[1];
size_t bytesRead = 0;
parseArgv(argc, argv);
if(show_help)
{
printf("Quicklz implementation version 1:\n");
printf("\n");
printf("usage: quicklz [options]:\n");
printf("\n");
printf(" -f file - input file that will be compress or decompress. This can be stdin.\n");
printf(" -h - show this help message.\n");
printf(" -o file - output file. This can be stdout.\n");
printf(" -d - will decompress the file given via stdin or via -f option.\n");
printf("\n");
printf("Examples:.\n");
printf("\n");
printf(" cat file.qlz | quicklz -d > file\n");
printf(" cat file | quicklz > file.qlz\n");
printf(" quicklz -i file -o file.qlz\n");
printf(" quicklz -d -i file.qlz -o file\n");
exit(0);
}
do
{
bytesRead = fread(
inputBuffer,
1,
1,
input_file
);
char *tmp = (char*)realloc(src, input_file_size + bytesRead);
if (tmp)
{
src = tmp;
memmove(&src[input_file_size], inputBuffer, bytesRead);
input_file_size += bytesRead;
}
else
{
free(src);
exit(1);
}
} while (!feof(input_file));
if(compress)
{
qlz_state_compress *state_compress = (qlz_state_compress *)malloc(sizeof(qlz_state_compress));
dst = (char*) malloc(input_file_size + 400);
output_file_size = qlz_compress(src, dst, input_file_size, state_compress);
}
else if(decompress)
{
qlz_state_decompress *state_decompress = (qlz_state_decompress *)malloc(sizeof(qlz_state_decompress));
output_file_size = qlz_size_decompressed(src);
dst = (char*) malloc(output_file_size);
output_file_size = qlz_decompress(src, dst, state_decompress);
}
fwrite(dst, output_file_size, 1, output_file);
fclose(input_file);
fclose(output_file);
return 0;
}
