コード例 #1
0
ファイル: record.c プロジェクト: chain710/beansdb
DataRecord* decompress_record(DataRecord *r)
{
    if (r->flag & COMPRESS_FLAG) {
        char scratch[QLZ_SCRATCH_DECOMPRESS];
        int csize = qlz_size_compressed(r->value);
        if (csize != r->vsz) {
            fprintf(stderr, "broken compressed data: %d != %d, flag=%x\n", csize, r->vsz, r->flag);
            goto DECOMP_END;
        }
        int size = qlz_size_decompressed(r->value);
        char *v = malloc(size);
        if (v == NULL) {
            fprintf(stderr, "malloc(%d)\n", size);
            goto DECOMP_END;
        }
        int ret = qlz_decompress(r->value, v, scratch);
        if (ret != size) {
            fprintf(stderr, "decompress %s failed: %d != %d\n", r->key, ret, size);
            goto DECOMP_END;
        }
        if (r->free_value) {
            free(r->value);
        }
        r->value = v;
        r->free_value = true;
        r->vsz = size;
        r->flag &= ~COMPRESS_FLAG;
    }
    return r;

DECOMP_END:
    free_record(r); 
    return NULL;
}
コード例 #2
0
ファイル: unvar.c プロジェクト: fandrieu/unvar
int tst_decomp(FILE *ifile, FILE *ofile, int opt_verbose)
{
    char *src, *dst, *scratch;
    unsigned int len;

    // allocate source buffer
    fseek(ifile, 0, SEEK_END);
    len = ftell(ifile);
    fseek(ifile, 0, SEEK_SET);
    src = (char*) malloc(len);

    // read file and allocate destination buffer
    len = fread(src, 1, len, ifile);
    len = qlz_size_decompressed(src);
    if (opt_verbose)
        printf("%s: uncomp block len: %d\n", progname, len);
    dst = (char*) malloc(len);

    // QLZ_SCRATCH_DECOMPRESS is defined in the beginning of the quicklz.h file
    scratch = (char*) malloc(QLZ_SCRATCH_DECOMPRESS);

    // decompress and write result
    len = qlz_decompress(src, dst, scratch);
    if (opt_verbose)
        printf("%s: decompd len: %d\n", progname, len);
    fwrite(dst, len, 1, ofile);
    return 0;
}
コード例 #3
0
ファイル: compress.c プロジェクト: CCoder123/nessDB
void ness_decompress(const char *src,
                     uint32_t src_size,
                     char *dst,
                     uint32_t dst_size)
{
	switch (src[0] & 0xF) {
	case NESS_NO_COMPRESS:
		memcpy(dst, src + 1, src_size - 1);
		break;

	case NESS_QUICKLZ_METHOD: {
			uint32_t raw_size;
			qlz_state_decompress *qsd;

			qsd = xcalloc(1, sizeof(*qsd));
			raw_size = qlz_decompress(src + 1, dst, qsd);
			nassert(raw_size == dst_size);
			(void)raw_size;
			(void)dst_size;

			xfree(qsd);
		}
		break;
	default:
		printf("no decompress support!\n");
		nassert(1);
		break;
	}
}
コード例 #4
0
ファイル: quicklz_compression.c プロジェクト: adam8157/gpdb
static size_t
quicklz_decompressor(int level, const char *source, void *destination,
					 void *state)
{
	if (level == 1)
	{
		return qlz_decompress(source, destination,
							   (qlz_state_decompress *)state);
	}
	Insist(false);
}
コード例 #5
0
ファイル: compression.c プロジェクト: xmcsi/solowan
/*
 * Decompress the TCP data of an SKB.
 */
int tcp_decompress(__u8 *ippacket, __u8 *lzbuffer,
		qlz_state_decompress *state_decompress) {
	struct iphdr *iph = NULL;
	struct tcphdr *tcph = NULL;
	__u16 oldsize = 0, newsize = 0; /* Store old, and new size of the TCP data. */
	__u8 *tcpdata = NULL; /* Starting location for the TCP data. */
	char message[LOGSZ];

	if (DEBUG_COMPRESSION == true) {
		sprintf(message, "[OpenNOP]: Entering into TCP DECOMPRESS \n");
		logger(LOG_INFO, message);
	}

	if ((ippacket != NULL) && (NULL != state_decompress)) { // If the skb or state_decompress is NULL abort compression.
		iph = (struct iphdr *) ippacket; // Access ip header.
		memset(state_decompress, 0, sizeof(qlz_state_decompress));

		if ((iph->protocol == IPPROTO_TCP)) { // If this is not a TCP segment abort compression.
			tcph = (struct tcphdr *) (((u_int32_t *) ippacket) + iph->ihl); // Access tcp header.
			oldsize = (__u16)(ntohs(iph->tot_len) - iph->ihl * 4) - tcph->doff
					* 4;

			tcpdata = (__u8 *) tcph + tcph->doff * 4; // Find starting location of the TCP data.

			if ((oldsize > 0) && (lzbuffer != NULL)) {

				newsize = qlz_decompress((char *) tcpdata, (char *) lzbuffer,
						state_decompress);
				memmove(tcpdata, lzbuffer, newsize); // Move decompressed data to packet.
				iph->tot_len = htons(ntohs(iph->tot_len) + (newsize - oldsize));// Fix packet length.
				__set_tcp_option((__u8 *) iph, 33, 3, 0); // Set compression flag to 0.
/*
				tcph->seq = htonl(ntohl(tcph->seq) - 8000); // Decrease SEQ number.
*/

				if (DEBUG_COMPRESSION == true) {
					sprintf(
							message,
							"[OpenNOP] Decompressing [%d] size of data to [%d] \n",
							oldsize, newsize);
					logger(LOG_INFO, message);
				}
				// return 1;
				// fruiz return amount of data expansion
				return newsize-oldsize;
			}
		}
	}
	return -1;
}
コード例 #6
0
ファイル: qzip.c プロジェクト: ArikaChen/quicklz
int stream_decompress(FILE *ifile, FILE *ofile)
{
    char *file_data, *decompressed;
    size_t d, c, dc, fd_size, d_size;
    qlz_state_decompress *state_decompress = (qlz_state_decompress *)malloc(sizeof(qlz_state_decompress));

    // a compressed packet can be at most MAX_BUF_SIZE + BUF_BUFFER bytes if it
    // was compressed with this program.
    fd_size = MAX_BUF_SIZE + BUF_BUFFER;
    file_data = (char*) malloc(fd_size);

    // allocate decompression buffer
    d_size = fd_size - BUF_BUFFER;
    decompressed = (char*) malloc(d_size);

    // allocate and initially zero out the scratch buffer. After this, make sure it is
    // preserved across calls and never modified manually
    memset(state_decompress, 0, sizeof(qlz_state_decompress));

    // read 9-byte header to find the size of the entire compressed packet, and
    // then read remaining packet
    while((c = fread(file_data, 1, 9, ifile)) != 0)
    {
        // Do we need a bigger decompressed buffer? If the file was compressed
        // with segments larger than the default in this program.
        dc = qlz_size_decompressed(file_data);
        if (dc > (fd_size - BUF_BUFFER)) {
            free(file_data);
            fd_size = dc + BUF_BUFFER;
            file_data = (char*)malloc(fd_size);
        }

        // Do we need a bigger compressed buffer?
        c = qlz_size_compressed(file_data);
        if (c > d_size) {
            free (decompressed);
            d_size = c;
            decompressed = (char*)malloc(d_size);
        }

        fread(file_data + 9, 1, c - 9, ifile);
        d = qlz_decompress(file_data, decompressed, state_decompress);
        fwrite(decompressed, d, 1, ofile);
    }

    free(decompressed);
    free(state_decompress);
    free(file_data);
    return 0;
}
コード例 #7
0
ファイル: pack.c プロジェクト: prodigeni/glcs
int unpack_write_callback(glc_thread_state_t *state)
{
	unpack_t unpack = (unpack_t) state->ptr;

	if (state->header.type == GLC_MESSAGE_LZO) {
#ifdef __LZO
		__sync_fetch_and_add(&unpack->stats.pack_size, state->read_size - sizeof(glc_lzo_header_t));
		memcpy(&state->header, &((glc_lzo_header_t *) state->read_data)->header,
		       sizeof(glc_message_header_t));
		__lzo_decompress((unsigned char *) &state->read_data[sizeof(glc_lzo_header_t)],
				state->read_size - sizeof(glc_lzo_header_t),
				(unsigned char *) state->write_data,
				(lzo_uintp) &state->write_size,
				NULL);
#else
		return ENOTSUP;
#endif
	} else if (state->header.type == GLC_MESSAGE_QUICKLZ) {
#ifdef __QUICKLZ
		__sync_fetch_and_add(&unpack->stats.pack_size,
					state->read_size - sizeof(glc_quicklz_header_t));
		memcpy(&state->header, &((glc_quicklz_header_t *) state->read_data)->header,
		       sizeof(glc_message_header_t));
		if (!state->threadptr)
			state->threadptr = malloc(sizeof(qlz_state_decompress));
		qlz_decompress((const void *) &state->read_data[sizeof(glc_quicklz_header_t)],
				(void *) state->write_data,
				(qlz_state_decompress *) state->threadptr);
#else
		return ENOTSUP;
#endif
	} else if (state->header.type == GLC_MESSAGE_LZJB) {
#ifdef __LZJB
		__sync_fetch_and_add(&unpack->stats.pack_size, state->read_size - sizeof(glc_lzjb_header_t));
		memcpy(&state->header, &((glc_quicklz_header_t *) state->read_data)->header,
		       sizeof(glc_message_header_t));
		lzjb_decompress(&state->read_data[sizeof(glc_lzjb_header_t)],
				state->write_data,
				state->read_size - sizeof(glc_lzjb_header_t),
				state->write_size);
#else
		return ENOTSUP;
#endif
	} else
		return ENOTSUP;
	__sync_fetch_and_add(&unpack->stats.unpack_size, state->write_size);
	return 0;
}
コード例 #8
0
ファイル: compressors.cpp プロジェクト: bobhansen/lzbench
int64_t lzbench_quicklz_decompress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t level, size_t dstate, size_t)
{
    switch (level)
    {
    default:
    case 1:
        return qlz_decompress_1(inbuf, outbuf, (qlz150_state_decompress*)dstate);
        break;
    case 2:
        return qlz_decompress_2(inbuf, outbuf, (qlz150_state_decompress*)dstate);
        break;
    case 3:
        return qlz_decompress_3(inbuf, outbuf, (qlz150_state_decompress*)dstate);
        break;
    case 4:
        return qlz_decompress(inbuf, outbuf, (qlz_state_decompress*)dstate);
        break;
    }
}
コード例 #9
0
ファイル: adv_image_layout.cpp プロジェクト: hpavlov/tangra3
void AdvImageLayout::GetDataFromDataBytes(enum GetByteMode mode, unsigned char* data, unsigned int* prevFrame, unsigned int* pixels, int sectionDataLength, int startOffset)
{
	unsigned char* layoutData;
	
	if (!m_UsesCompression)
	{
		layoutData = data + startOffset;
	}
	else if (0 == strcmp(Compression, "QUICKLZ"))
	{		
		size_t size = qlz_size_decompressed((char*)(data + startOffset));
		// MaxFrameBufferSize
		qlz_decompress((char*)(data + startOffset), m_DecompressedPixels, m_StateDecompress);		
		layoutData = (unsigned char*)m_DecompressedPixels;
	}
	else  if (0 == strcmp(Compression, "LAGARITH16"))
	{		
		int size = m_Lagarith16Decompressor->DecompressData((char*)(data + startOffset), (unsigned short*)m_DecompressedPixels);
		layoutData = (unsigned char*)m_DecompressedPixels;
	}

	bool crcOkay;
	int readIndex = 0;

	if (Bpp == 12)
	{
		GetPixelsFrom12BitByteArray(layoutData, prevFrame, pixels, mode, &readIndex, &crcOkay);
	}
	else if (Bpp == 16)
	{
		if (IsDiffCorrLayout)
			GetPixelsFrom16BitByteArrayDiffCorrLayout(layoutData, prevFrame, pixels, &readIndex, &crcOkay);
		else
			GetPixelsFrom16BitByteArrayRawLayout(layoutData, prevFrame, pixels, &readIndex, &crcOkay);
	}
	else if (Bpp == 8)
	{
		if (IsDiffCorrLayout)
			GetPixelsFrom8BitByteArrayDiffCorrLayout(layoutData, prevFrame, pixels, &readIndex, &crcOkay);
		else
			GetPixelsFrom8BitByteArrayRawLayout(layoutData, prevFrame, pixels, &readIndex, &crcOkay);
	}
}
コード例 #10
0
ファイル: compressors.cpp プロジェクト: TurtleSimos/lzbench
int64_t lzbench_quicklz_decompress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t level, size_t , char*)
{
    int64_t res;
    qlz150_state_compress* dstate = (qlz150_state_compress*) calloc(1, MAX(qlz_get_setting_3(2),MAX(qlz_get_setting_1(2), qlz_get_setting_2(2))));
    if (!dstate)
        return 0;
        
	switch (level)
	{
		default:
		case 1: res = qlz_decompress_1(inbuf, outbuf, (qlz150_state_decompress*)dstate); break;
		case 2: res = qlz_decompress_2(inbuf, outbuf, (qlz150_state_decompress*)dstate); break;
		case 3: res = qlz_decompress_3(inbuf, outbuf, (qlz150_state_decompress*)dstate); break;
		case 4: res = qlz_decompress(inbuf, outbuf, (qlz_state_decompress*)dstate); break;
	}

    free(dstate);
    return res;
}
コード例 #11
0
ファイル: stream_decompress.c プロジェクト: ArikaChen/quicklz
int main(int argc, char* argv[])
{
    FILE *ifile, *ofile;
    char *file_data, *decompressed;
    size_t d, c;
    qlz_state_decompress *state_decompress = (qlz_state_decompress *)malloc(sizeof(qlz_state_decompress));

    // read source file
    ifile = fopen(argv[1], "rb");
    ofile = fopen(argv[2], "wb");

    // a compressed packet can be at most "uncompressed size" + 400 bytes large where
    // "uncompressed size" = 10000 in worst case in this sample demo
    file_data = (char*) malloc(10000 + 400);

    // allocate decompression buffer
    decompressed = (char*) malloc(10000);

    // allocate and initially zero out the scratch buffer. After this, make sure it is
    // preserved across calls and never modified manually
	memset(state_decompress, 0, sizeof(qlz_state_decompress)); 


    // read 9-byte header to find the size of the entire compressed packet, and 
    // then read remaining packet
    while((c = fread(file_data, 1, 9, ifile)) != 0)
    {
        c = qlz_size_compressed(file_data);
        fread(file_data + 9, 1, c - 9, ifile);
        d = qlz_decompress(file_data, decompressed, state_decompress);
        printf("%u bytes decompressed into %u.\n", (unsigned int)c, (unsigned int)d);
        fwrite(decompressed, d, 1, ofile);
    }
    fclose(ifile);
    fclose(ofile);
    return 0;
}
コード例 #12
0
PyObject *qlz_decompress_py(PyObject *self, PyObject *args)
{ 
    PyObject *result = NULL;
    PyObject *state = NULL;
    char* buffer;
    char* decompressed_buffer;
    int buffer_length;
    int size_decompressed;

#if PY_MAJOR_VERSION >= 3
    if (PyArg_ParseTuple(args, "y#O!",
                        &buffer, &buffer_length,
                        &QLZStateDecompressType, &state))
#else
    if (PyArg_ParseTuple(args, "s#O!",
                        &buffer, &buffer_length,
                        &QLZStateDecompressType, &state))
#endif
    {
        size_decompressed = qlz_size_decompressed(buffer);
        decompressed_buffer = (char*)malloc(size_decompressed);

        qlz_decompress(buffer, decompressed_buffer,
                       ((qlz_state_decompress_Type*)state)->value);
        
#if PY_MAJOR_VERSION >= 3
        result = Py_BuildValue("y#", decompressed_buffer, size_decompressed);
#else
        result = Py_BuildValue("s#", decompressed_buffer, size_decompressed);
#endif
        free(decompressed_buffer);
    } /* otherwise there is an error,
       * the exception already raised by PyArg_ParseTuple, and NULL is
       * returned.
       */
    return result;
}
コード例 #13
0
ファイル: unvar.c プロジェクト: fandrieu/unvar
/*
    VAR file decompress
*/
int var_decompress(FILE *fi, FILE *fo, int verbose)
{
    int r = 0;
	int ok;
    char *in_buf = NULL;
    char *out_buf = NULL;
    char *scratch = NULL;
    unsigned char m [ sizeof(magic) ];
    unsigned int block_size;
    md5_state_t md5_state;
    md5_byte_t md5_digest[16];
    static const char *const block_salt = ":lzo:";
    
    //VAR ext
    var_header file;    
    char file_chk_str [ 33 ];

    //VAR ext block
    var_block_header block;
    qlz_block_header qlz_block;
    unsigned long new_len;
    unsigned long block_next_num;
    unsigned long block_count;
    unsigned long block_empty;
    unsigned long long block_real_pos;
    unsigned long block_last_nonzero;
    
    total_in = total_out = 0;
    block_count = block_empty = 0;
    block_next_num = block_last_nonzero = 0;

/*
 * Step 1: check magic header, read flags & block size
 */
    if (xread(fi, m, sizeof(magic),1) != sizeof(magic) ||
        memcmp(m, magic, sizeof(magic)) != 0)
    {
        printf("%s: header error - this file is not a .var\n", progname);
        r = 1;
        goto err;
    }
    //read header
    var_read_header(fi, &file);
    block_size = file.block_size;

    if (block_size < 1024 || block_size > 8*1024*1024L)
    {
        printf("%s: header error - invalid block size %u\n",
                progname, block_size);
        r = 3;
        goto err;
    }
    if (verbose > 0)
        printf("%s: block size %u\n",
                progname, block_size);
    
    //header debug
    if (verbose > 0)
    {
        str_to_hexstr(file.uuid, file_chk_str);
        printf("%s: version %u, bin_len %u, txt_len %u, header_end x%llx\n\tfile_flags %lx, file_uuid %s\n",
                progname, file.version, file.bin_len, file.txt_len, xtell(fi), file.flags, file_chk_str);
    }

/*
 * Step 2: allocate buffer for in-place decompression
 */
    // QLZ uses one input & one ouput buffer + a scratch one
    // The decompressed length can be obtained with "qlz_size_decompressed(src)"
    // but we use the fixed block size and reuse the same buffers for all blocks
    //out_len = qlz_size_decompressed(in);    
    if (opt_debug > 0)
        printf("%s: allocate 2 x %d\n", progname, block_size);
    in_buf = (char*) malloc(block_size);
    out_buf = (char*) malloc(block_size);
    scratch = (char*) malloc(QLZ_SCRATCH_DECOMPRESS);
    if (in_buf == NULL || out_buf == NULL || scratch == NULL)
    {
        printf("%s: out of memory\n", progname);
        r = 4;
        goto err;
    }
    
/*
 * Step 3: process blocks
 */
    for (;;)
    {
        
        /*
            Parse the VAR block header
        */

        block_real_pos = xtell(fi);
        
        var_read_block_header(fi, &block);

        //NOTE: files can have no data blocks at all (all zero disk...)
        //if (block_next_num > 0 && block.num == 0) {
		if (block.in_len == 0) {
            // file end
            if (verbose > 0)
                printf("%s: last block at %llx\n",\
                    progname, block_real_pos);
            
            break;
        }
        
        // block header debug
        if (opt_debug > 0) {
            printf("%s: reading block\t%lu, pos\t%llu\tin/out: %lu/%lu\n",\
                progname, block.num, xtell(fi), block.in_len, block.out_len); 
            str_to_hexstr(block.chk, file_chk_str);
            printf("%s: checksum: %s\n", progname, file_chk_str);
        }

        if (block.num != block_next_num)
        {
            if (block.num < block_next_num)
            {
                printf("%s: block number error - expected >= %lu / got %lu (x%llx)\n",\
                    progname, block_next_num, block.num, block_real_pos);
                r = 5;
                goto err;
            }
            // bloc_num >  block_next_num actually means empty blocks...
            block_next_num = block.num - block_next_num;
            if (opt_debug > 0)
                printf("%s: writing %ld empty blocks\n", progname, block_next_num);
            
            xwrite_empty_blocks(fo, block.out_len, block_next_num, out_buf);
            
            //record empty blocks + reset next_num
            block_empty += block_next_num;
            block_next_num = block.num;
        }
        block_next_num += 1;

        // sanity check of the size values
        if (block.in_len-9 > block_size || block.out_len > block_size ||
            block.in_len == 0 || block.in_len-9 > block.out_len)
        {
            printf("%s: block size error - data corrupted\n", progname);
            printf("%s: len in / out:  %lu /  %lu\n",
                    progname, block.in_len, block.out_len);
            r = 6;
            goto err;
        }

        //check block pos
        //NOTE: this this doesn't seem to be the set in v5 files...
        if (block.pos && block.pos != block_real_pos)
        {
            printf("%s: block position error - expected %llu / got %llu\n",\
                progname, block_real_pos, block.pos);
            r = 7;
            goto err;
        }

        
        /*
            Parse the QLZ header
            actually we don't need to (it's part of a standard QLZ block),
            it's just for additionale checks,
            but we need to rewind after that...
        */
        
        var_read_qlz_block_header(fi, &qlz_block);
        if (qlz_block.type != 0x4b && qlz_block.type != 0x4a)
        {
            printf("%s: block flag error - expected 4b/4a / got %x\n",\
                progname, (int)qlz_block.type);
            r = 8;
            goto err;
        }

        // read qlz block uncompressed & compressed size
        // it should be the same as in the VAR header
        if (qlz_block.out_len != block.out_len || qlz_block.in_len != block.in_len)
        {
            printf("%s: block qlz len error - in %lu/%lu, out %lu/%lu\n",\
                progname, block.in_len, qlz_block.in_len, block.out_len, qlz_block.out_len);
            r = 9;
            goto err;
        }

        /*
            Read & decompress the QLZ block
        */
        block_count++;

        // rewind at the begining of the QLZ block / header
        // 1 x QLZ marker + 4 x comp size + 4 x uncomp size
        xskip(fi, -9);
        
        if (opt_debug > 1)
        {
            // skip the block
            xskip(fi, block.in_len);
        }
        else
        {

            // read the compressed block in "in_buf"
            xread(fi, in_buf, block.in_len, 0);
            
            //real decompress

            // debug
            //xwrite(fo, in, block.in_len);
            
            //todo: clean scratch buffer (if QLZ_STREAMING_BUFFER > 0)
            //memset(scratch_buf, 0, QLZ_SCRATCH_DECOMPRESS);

            // decompress
            //NOTE: define QLZ_MEMORY_SAFE to get checks...
            new_len = qlz_decompress(in_buf, out_buf, scratch);

            if (opt_debug > 0)
                printf("%s: decomp len: %lu / %lu\n", progname, new_len, block.out_len);

            if (new_len != block.out_len)
            {
                printf("%s: compressed data violation\n", progname);
                r = 10;
                goto err;
            }

            // write decompressed block
            xwrite(fo, out_buf, block.out_len);

            //check block checksum
            md5_init(&md5_state);
            md5_append(&md5_state, (const md5_byte_t *)block_salt, 5);
            md5_append(&md5_state, (const md5_byte_t *)out_buf, block.out_len);
            md5_finish(&md5_state, md5_digest);
            if (opt_debug > 0)
            {
                str_to_hexstr((char *)md5_digest, file_chk_str);
                printf("%s: verified: %s\n", progname, file_chk_str);
            }
            if (memcmp(block.chk, md5_digest, 16) != 0)
            {
                printf("%s: checksum error - block %lu corrupted\n", progname, block.num);
                r = 11;
                goto err;
            }
        }

        //debug
        if (opt_debug > 1)
        {
            if(block_next_num>1)
            {
                printf("bail, pos x%llx\n", xtell(fi));
                goto err;
            }
        }
        
    }

/*
 * Step 4: process map / index
 */
    
    //read map header
    var_read_map_header(fi, &file);

    //debug
    if (verbose > 0)
    {
        str_to_hexstr(file.map_chk, file_chk_str);
        printf("%s: map: len %lu, len2 %lu, pos %llx, check %s\n",
                progname, file.map_len, file.map_len2, xtell(fi), file_chk_str);
    }

	
	//read all before parsing to verify...
	//TODO: compute while reading blocks ? anyway it's useless...
	if (opt_debug>0)
	{
		ok = file_check_part(fi, file.map_len, file.map_chk);
		fseeko64(fi, -1 * file.map_len, SEEK_CUR);
		printf("%s: map checksum %s\n", progname, (ok==0)?"OK":"ERROR");
	}
    

/*
 * Step 5: process blocks map
 */
//TODO: check // blocks...

    block_next_num = 0;
    block_last_nonzero = -1;
    for(;;)
    {
        var_read_block_header(fi, &block);
        
        //FIXME: files can have no data blocks at all (all zero disk...)
        if (block_next_num > 0 && block.num == 0) {
		//if (block.num == 0) {
            // file end
            if (verbose > 0)
                printf("%s: last map entry (total %lu blocks) at %llx\n",\
                    progname, block_next_num, xtell(fi));
            
            //write last empty blocks
            block_next_num = block_next_num - 1 - block_last_nonzero;
            if (opt_debug > 0)
                printf("%s: writing %ld trailing empty blocks\n", progname, block_next_num);
            
            xwrite_empty_blocks(fo, block_size, block_next_num, out_buf);
            
            block_empty += block_next_num;
            
            //end
            break;
        }
        
        if (block.num != block_next_num)
        {
            printf("%s: map block number error - expected >= %lu / got %lu (x%llx)\n",\
                progname, block_next_num, block.num, xtell(fi));
            r = 12;
            goto err;
        }
        block_next_num += 1;
        block_size = block.out_len; //store previous block size
        
        //TODO check // real last block processed
        if (block.in_len > 0)
            block_last_nonzero = block.num;

//debug
//if (block.out_len != block_size)
if (block.out_len > 0x40000)
    printf("%s: WARNING: block_size too large: %lu\n", progname, block.out_len);

        // block map debug
        if (opt_debug > 0)
            printf("%s: reading map\t%lu, in/out: %lu/%lu, end @ x%llx\n",\
                progname, block.num, block.in_len, block.out_len, xtell(fi));
    }

    //read footer
    var_read_footer(fi, &file);
    //should be the end of file...

	//debug...
    if (verbose > 0)
    {   
        const char *const chk_names[4] = {"bin", "txt", "map", "foot"};
        int i;
        for (i = 0; i<4; i++) {
            str_to_hexstr(file.chks[i], file_chk_str);
            printf("%s: %s chk: %s\n", progname, chk_names[i], file_chk_str);
        }
        printf("%s: footer info %lu / %lu / %lu\n",
            progname, file.foot1, file.foot2, file.foot3);
    }
    
    //TODO: check those md5 before reading the file...
    //...but that implies reading the file end first
    if (opt_debug > 0)
    {
        //chk 0: bin header: from start for bin_len
        fseeko64(fi, 0, SEEK_SET);
        ok = file_check_part(fi, file.bin_len, file.chks[0]);
        printf("%s: binary header checksum %s\n", progname, (ok==0)?"OK":"ERROR");

        //chk 1: txt header: from bin header for txt_len
        ok = file_check_part(fi, file.txt_len, file.chks[1]);
        printf("%s: text header checksum %s\n", progname, (ok==0)?"OK":"ERROR");
        
        //chk 2: map dupe: full map...
        ok = memcmp(file.map_chk, file.chks[2], 16);
        printf("%s: map dupe checksum %s\n", progname, (ok==0)?"OK":"ERROR");
        
        //chk 3: footer: last 12 bytes
        fseeko64(fi, -12, SEEK_END);
        ok = file_check_part(fi, 12, file.chks[3]);
        printf("%s: footer checksum %s\n", progname, (ok==0)?"OK":"ERROR");
    }



    if (verbose > 0)
        printf("%s: blocks decomp / empty / total: %lu / %lu / %lu\n",\
            progname, block_count, block_empty, block_count + block_empty);

    r = 0;
err:
    free(in_buf);
    free(out_buf);
    free(scratch);
    return r;
}
コード例 #14
0
ファイル: net_pkg.c プロジェクト: hx71105417/sdbc_linux
int RecvPack(T_Connect *connect,T_NetHead *nethead)
{
char headbuf[HEADPACKLENGTH+1],addr[16];
int i,n;
u_int crc;
char *zbuf;

	memset(nethead->para,0,sizeof(nethead->para));
	nethead->data=NULL;
	peeraddr(connect->Socket,addr);
	i=RecvNet(connect->Socket,headbuf,HEADPACKLENGTH,connect->timeout);
	if(i<HEADPACKLENGTH){
		if(i==TIMEOUTERR) {
			ShowLog(1,"%s:head TIMEOUT %d second's",__FUNCTION__,
				connect->timeout);
			return i;
		}
		ShowLog(1,"RecvPack Head LENERR i=%d,err=%d,%s",i,errno,strerror(errno));
		return LENGERR;
	}
	if(connect->CryptFlg & DO_CRYPT)
		enigma2_decrypt(&connect->t,headbuf,HEADPACKLENGTH);
	i=NetHeadDispack(nethead,headbuf,connect->family[29]);
	if(i!=0) {
		ShowLog(1,"aft NetHeadDispack len=%d,PKGERR %s",i, addr);
		showpack(1,headbuf,HEADPACKLENGTH);
		return(FORMATERR);
	}
	if(!nethead->T_LEN) return 0;
	if(connect->CryptFlg&UNDO_ZIP) {
		i=nethead->T_LEN+1;
	} else {
		i=nethead->PKG_LEN+1;
		if(nethead->T_LEN <nethead->PKG_LEN) i+=nethead->T_LEN+1;
	}
	n=connect->RecvLen-i;
	if(!connect->RecvBuffer || n<0 || n>32768){
		if(connect->RecvBuffer) {
			free(connect->RecvBuffer);
			connect->RecvBuffer=0;
		}
		connect->RecvLen=0;
		connect->RecvBuffer=malloc(i);
		if(!connect->RecvBuffer) return MEMERR;
		connect->RecvLen=i;
	}
	if(!(connect->CryptFlg&UNDO_ZIP) && nethead->T_LEN != nethead->PKG_LEN) {
		zbuf=connect->RecvBuffer+nethead->PKG_LEN+1;
	} else zbuf=connect->RecvBuffer;
	i=RecvNet(connect->Socket,zbuf,nethead->T_LEN,5);
	if(i != (nethead->T_LEN)) {
		if(TIMEOUTERR == i) {
			ShowLog(1,"%s:recv body TIMEOUT",__FUNCTION__);
			return i;
		}
		ShowLog(1,"%s,Recv Body T_LEN=%d i=%d,errno=%d",__FUNCTION__,
					nethead->T_LEN,i,errno);
		free(connect->RecvBuffer);
		connect->RecvBuffer=0;
		connect->RecvLen=0;
		return i<0?SYSERR:LENGERR;
	}
	crc=ssh_crc32((const unsigned char *)zbuf, nethead->T_LEN);
	if((connect->CryptFlg & CHECK_CRC) && (crc != nethead->PKG_CRC)) {
		ShowLog(1,"RecvPack:PKG_CRC=%08X,crc=%08X,PKG_LEN=%d,T_LEN=%d,head=%s",
			nethead->PKG_CRC,crc,nethead->PKG_LEN,nethead->T_LEN,headbuf);
			return CRCERR;
	}
        pack_decode(zbuf, nethead->T_LEN,connect);
	if(zbuf != connect->RecvBuffer) {
		size_t sz=qlz_size_decompressed(zbuf);
		if(nethead->T_LEN<9 || nethead->PKG_LEN != sz) {
			ShowLog(1,"unzip error T_LEN=%d,sz=%ld,PKG_LEN=%d,ADDR=%s",nethead->T_LEN, sz,nethead->PKG_LEN,addr);
			return FORMATERR;
		}
		i=qlz_decompress(zbuf,connect->RecvBuffer);
		if(i!=nethead->PKG_LEN) {
		    ShowLog(1,"RecvPack:PKG_LEN=%d,T_LEN=%d,unzip_size=%d",
			nethead->PKG_LEN,nethead->T_LEN,i);
			return LENGERR;
		}
	} //else if(connect->CryptFlg & UNDO_ZIP)
//		ShowLog(5,"%s:UNDO_ZIP %s,T_LEN=%d,PKG_LEN=%d,data=%s",__FUNCTION__,
//		*connect->Host?connect->Host:"Client",nethead->T_LEN,nethead->PKG_LEN,connect->RecvBuffer);
	if(!(connect->CryptFlg & UNDO_ZIP)) connect->RecvBuffer[nethead->PKG_LEN]=0;
	nethead->data=connect->RecvBuffer;
	return 0;
}
コード例 #15
0
ファイル: net_quicklz.c プロジェクト: hx71105417/sdbc_linux
int RecvPack(T_Connect *connect,T_NetHead *nethead)
{
char headbuf[HEADPACKLENGTH+1],addr[16];
int i,n;
u_int crc;
char *zbuf;

	n_zero(PARANUM,nethead->para);
	nethead->data=0;
	memset(headbuf,0,sizeof(headbuf));
	i=RecvNet(connect->Socket,headbuf,HEADPACKLENGTH,connect->timeout);
//free SendBuffer
	if(i<HEADPACKLENGTH){
		if(i==TIMEOUTERR) {
			ShowLog(1,"%s:head TIMEOUT %d second's",__FUNCTION__,
				connect->timeout);
			return i;
		}
		ShowLog(1,"RecvPack Head LENERR i=%d,err=%d,%s",i,errno,strerror(errno));
		return LENGERR;
	}
	headbuf[HEADPACKLENGTH]=0;
	i=NetHeadDispack(nethead,headbuf,connect->family[29]);
	if(i!=0) {
		peeraddr(connect->Socket,addr);
		ShowLog(1,"aft NetHeadDispack len=%d,PKGERR %s:%.48s",i, addr,headbuf);
		showpack(1,headbuf,HEADPACKLENGTH);
		return(FORMATERR);
	}
	if(!nethead->T_LEN) return 0;
	i=((connect->CryptFlg&UNDO_ZIP)?nethead->T_LEN:nethead->PKG_LEN)+1;
	n=connect->RecvLen-i;
	if(n<0 || n>SDBC_BLKSZ){
		if( connect->RecvBuffer)
			free(connect->RecvBuffer);
		connect->RecvBuffer=0;
		connect->RecvLen=0;
		connect->RecvBuffer=malloc(i);
		if(!connect->RecvBuffer) return MEMERR;
		connect->RecvLen=i;
	}
	if(!(connect->CryptFlg&UNDO_ZIP) && nethead->T_LEN != nethead->PKG_LEN) {
		zbuf=malloc(nethead->T_LEN);
		if(!zbuf) {
			RecvNet(connect->Socket,connect->RecvBuffer,nethead->T_LEN,3);
			return MEMERR;
		}
	} else zbuf=connect->RecvBuffer;
	i=RecvNet(connect->Socket,zbuf,nethead->T_LEN,3);
	if(i < (nethead->T_LEN)) {
		if(TIMEOUTERR == i) {
			ShowLog(1,"%s:recv body TIMEOUT",__FUNCTION__);
			return i;
		}
		if(zbuf!=connect->RecvBuffer) free(zbuf);
		ShowLog(1,"%s,Recv Body T_LEN=%d i=%d,errno=%d",__FUNCTION__,
					nethead->T_LEN,i,errno);
		free(connect->RecvBuffer);
		connect->RecvBuffer=0;
		connect->RecvLen=0;
		return i<0?SYSERR:LENGERR;
	}
	crc=ssh_crc32((const unsigned char *)zbuf, nethead->T_LEN);
	if((connect->CryptFlg & CHECK_CRC) && (crc != nethead->PKG_CRC)) {
		ShowLog(1,"RecvPack:PKG_CRC=%08X,crc=%08X,PKG_LEN=%d,T_LEN=%d,head=%s",
			nethead->PKG_CRC,crc,nethead->PKG_LEN,nethead->T_LEN,headbuf);
			return CRCERR;
	}
        pack_decode(zbuf, nethead->T_LEN,connect);
	if(zbuf != connect->RecvBuffer) {
		if(nethead->T_LEN<9 || nethead->PKG_LEN != qlz_size_decompressed(zbuf)) {
			ShowLog(1,"unzip error T_LEN=%d,ADDR=%s",nethead->T_LEN, addr);
			return FORMATERR;
		}
		i=qlz_decompress(zbuf,connect->RecvBuffer);
		free(zbuf);
		if(i!=nethead->PKG_LEN) {
		    ShowLog(1,"RecvPack:PKG_LEN=%d,T_LEN=%d,unzip_size=%d",
			nethead->PKG_LEN,nethead->T_LEN,i);
			return LENGERR;
		}
	} 
	connect->RecvBuffer[nethead->PKG_LEN]=0;
	nethead->data=connect->RecvBuffer;
	return 0;
}
コード例 #16
0
ファイル: state.cpp プロジェクト: TOTOleHero/vbjin-ovr
int MDFN_StateEvil(int rewind)
{
 if(!EvilEnabled)
  return(0);

 if(rewind)
 {
  int32 next_bcspos = bcspos;
  bool NeedDataFlush = FALSE;

  bcspos--;
  if(bcspos < 0) bcspos += SRW_NUM;

  if(!bcs[bcspos].data)
   bcspos = (bcspos + 1) % SRW_NUM;
  else
   NeedDataFlush = TRUE;

  if(bcs[bcspos].compressed_len)
  {
   uint8 *tmp_buf;
   lzo_uint dst_len = bcs[bcspos].uncompressed_len;

   tmp_buf = (uint8 *)malloc(bcs[bcspos].uncompressed_len);

   if(SRWCompressor == SRW_COMPRESSOR_QUICKLZ)
   {
    //static char workmem[QLZ_SCRATCH_DECOMPRESS];
    dst_len = qlz_decompress((char*)bcs[bcspos].data, tmp_buf); //, workmem);
   }
   else if(SRWCompressor == SRW_COMPRESSOR_MINILZO)
    lzo1x_decompress(bcs[bcspos].data, bcs[bcspos].compressed_len, tmp_buf, &dst_len, NULL);
   else if(SRWCompressor == SRW_COMPRESSOR_BLZ)
   {
    dst_len = blz_unpack(bcs[bcspos].data, tmp_buf);
   }
   for(uint32 x = 0; x < bcs[bcspos].uncompressed_len && x < bcs[next_bcspos].uncompressed_len; x++)
    tmp_buf[x] ^= bcs[next_bcspos].data[x];

   free(bcs[bcspos].data);
   bcs[bcspos].data = tmp_buf;
   bcs[bcspos].compressed_len = 0;
  }

  if(NeedDataFlush)
  {
   if(bcs[next_bcspos].MovieLove.data)
   {
    free(bcs[next_bcspos].MovieLove.data);
    bcs[next_bcspos].MovieLove.data = NULL;
   }
   free(bcs[next_bcspos].data);
   bcs[next_bcspos].data = NULL;
   bcs[next_bcspos].compressed_len = 0;
   bcs[next_bcspos].uncompressed_len = 0;
  }

  if(bcs[bcspos].uncompressed_len)
  {
   StateMem sm;

   sm.data = bcs[bcspos].data;
   sm.loc = 0;
   sm.initial_malloc = 0;
   sm.malloced = sm.len = bcs[bcspos].uncompressed_len;

   MDFNSS_LoadSM(&sm, 0, 1);

//   free(MDFNMOV_GrabRewindJoy().data);
   return(1);
  }
 }
 else
 {
  StateMem sm;
  int32 prev_bcspos = bcspos;

  bcspos = (bcspos + 1) % SRW_NUM;

//  if(MDFNMOV_IsRecording())
  {
   if(bcs[bcspos].data && bcs[bcspos].MovieLove.data)
   {
    //printf("Force: %d\n", bcspos);
//    MDFNMOV_ForceRecord(&bcs[bcspos].MovieLove);
    free(bcs[bcspos].MovieLove.data);
    bcs[bcspos].MovieLove.data = NULL;
   }
  }
  if(bcs[bcspos].data)
  {
   free(bcs[bcspos].data);
   bcs[bcspos].data = NULL;
  }
  if(bcs[bcspos].MovieLove.data)
  {
   free(bcs[bcspos].MovieLove.data);
   bcs[bcspos].MovieLove.data = NULL;
  }

  memset(&sm, 0, sizeof(sm));

//  MDFNSS_SaveSM(&sm, 0, 1);

  bcs[bcspos].data = sm.data;
  bcs[bcspos].compressed_len = 0;
  bcs[bcspos].uncompressed_len = sm.len;

  // Compress the previous save state.
  if(bcs[prev_bcspos].data)
  {
   for(uint32 x = 0; x < bcs[prev_bcspos].uncompressed_len && x < sm.len; x++)
    bcs[prev_bcspos].data[x] ^= sm.data[x];

   if(SRWCompressor == SRW_COMPRESSOR_QUICKLZ)
   {
    //static char workmem[QLZ_SCRATCH_COMPRESS];
    uint8 *tmp_buf = (uint8 *)malloc(bcs[prev_bcspos].uncompressed_len + 36000);
    uint32 dst_len = bcs[prev_bcspos].uncompressed_len + 36000;

    dst_len = qlz_compress(bcs[prev_bcspos].data, (char*)tmp_buf, bcs[prev_bcspos].uncompressed_len); //, workmem);

    free(bcs[prev_bcspos].data);
    bcs[prev_bcspos].data = (uint8 *)realloc(tmp_buf, dst_len);
    bcs[prev_bcspos].compressed_len = dst_len;
   }
   else if(SRWCompressor == SRW_COMPRESSOR_MINILZO)
   {
    uint8 workmem[LZO1X_1_MEM_COMPRESS];
    uint8 * tmp_buf = (uint8 *)malloc((size_t)(1.10 * bcs[prev_bcspos].uncompressed_len));
    lzo_uint dst_len = (lzo_uint)(1.10 * bcs[prev_bcspos].uncompressed_len);

    lzo1x_1_compress(bcs[prev_bcspos].data, bcs[prev_bcspos].uncompressed_len, tmp_buf, &dst_len, workmem);

    free(bcs[prev_bcspos].data);
    bcs[prev_bcspos].data = (uint8 *)realloc(tmp_buf, dst_len);
    bcs[prev_bcspos].compressed_len = dst_len;
   }
   else if(SRWCompressor == SRW_COMPRESSOR_BLZ)
   {
    blz_pack_t workmem;

    uint8 * tmp_buf = (uint8 *)malloc((size_t)(bcs[prev_bcspos].uncompressed_len + blz_pack_extra));
    uint32 dst_len = bcs[prev_bcspos].uncompressed_len + blz_pack_extra;

    dst_len = blz_pack(bcs[prev_bcspos].data, bcs[prev_bcspos].uncompressed_len, tmp_buf, &workmem);

    free(bcs[prev_bcspos].data);
    bcs[prev_bcspos].data = (uint8 *)realloc(tmp_buf, dst_len);
    bcs[prev_bcspos].compressed_len = dst_len;
   }
  }

//  if(MDFNMOV_IsRecording())
//   bcs[bcspos].MovieLove = MDFNMOV_GrabRewindJoy();
 }
 return(0);
}
コード例 #17
0
bool CompressionSuite::Decompressor::Decompress(char* pOutputBuffer, const int outputBufferSize, CompressionStats* pStats)
{
    assert(mHeader.Algorithm != ALG_Invalid);

    int uncompressedDataSize = mHeader.UncompressedDataSize;
    if (pStats)
    {
        pStats->CompressedDataSize = mCompressedDataSize;
        pStats->UncompressedDataSize = uncompressedDataSize;
    }

    if (uncompressedDataSize > outputBufferSize)
        return false;

    double elapsedTime = 0.0;
    bool result = false;
    if (mHeader.Algorithm == ALG_DOBOZ)
    {
        doboz::Decompressor decompressor;
        Timer timer;

        timer.delta();
        result = (decompressor.decompress(mpInputBuffer, mCompressedDataSize, pOutputBuffer, uncompressedDataSize) == doboz::RESULT_OK);
        elapsedTime = timer.delta();
    }
    else if (mHeader.Algorithm == ALG_YAPPY)
    {
        Timer timer;
        timer.delta();

        unsigned char* pSrcData = (unsigned char*)mpInputBuffer;
        unsigned char* pSrcDataEnd = (unsigned char*)mpInputBuffer + mCompressedDataSize;
        unsigned char* pDstData = (unsigned char*)pOutputBuffer;

        while (pSrcData < pSrcDataEnd)
        {
            unsigned short compressedBlockSize = *reinterpret_cast<unsigned short*>(pSrcData);
            pSrcData += sizeof(unsigned short);

            unsigned char* pSrcBlockEnd = pSrcData + compressedBlockSize;
            unsigned char* pDstBlockEnd = Yappy_UnCompress(pSrcData, pSrcBlockEnd, pDstData);

            pDstData = pDstBlockEnd;
            pSrcData = pSrcBlockEnd;
        };

        elapsedTime = timer.delta();

        result = ((pDstData == ((unsigned char*)pOutputBuffer + uncompressedDataSize)) &&
                  (pSrcData == pSrcDataEnd));
    }
    else if (mHeader.Algorithm == ALG_QUICKLZ)
    {
        qlz_state_decompress qlzState;
        memset(&qlzState, 0, sizeof(qlzState));
        Timer timer;

        timer.delta();
        unsigned int outSize = qlz_decompress(mpInputBuffer, pOutputBuffer, &qlzState);
        elapsedTime = timer.delta();

        result = (outSize == uncompressedDataSize);
    }
    else if (mHeader.Algorithm == ALG_FASTLZ)
    {
        Timer timer;

        timer.delta();
        int outSize = fastlz_decompress(mpInputBuffer, mCompressedDataSize, pOutputBuffer, uncompressedDataSize);
        elapsedTime = timer.delta();

        result = (outSize == uncompressedDataSize);
    }
    else if (mHeader.Algorithm == ALG_LZF)
    {
        Timer timer;

        timer.delta();
        int outSize = lzf_decompress(mpInputBuffer, mCompressedDataSize, pOutputBuffer, uncompressedDataSize);
        elapsedTime = timer.delta();

        result = (outSize == uncompressedDataSize);
    }
    else if (mHeader.Algorithm == ALG_SNAPPY)
    {
        Timer timer;

        timer.delta();
        result = snappy::RawUncompress(mpInputBuffer, mCompressedDataSize, pOutputBuffer);
        elapsedTime = timer.delta();
    }
    else if (mHeader.Algorithm == ALG_LZ4)
    {
        Timer timer;

        timer.delta();
        int outSize = LZ4_decode(mpInputBuffer, pOutputBuffer, mCompressedDataSize);
        elapsedTime = timer.delta();

        result = (outSize == uncompressedDataSize);
    }

    if (pStats)
        pStats->ElapsedTime = elapsedTime;

    FreeCompressedData();

    return result;
}
コード例 #18
0
ファイル: main.c プロジェクト: nvanheuverzwijn/quicklz
int main(int argc, char* argv[])
{
    input_file = stdin;
    output_file = stdout;
    char *src = 0;
    char *dst = 0;
    size_t input_file_size = 0;
    size_t output_file_size = 0;
    char inputBuffer[1];
    size_t bytesRead = 0;

    parseArgv(argc, argv);

    if(show_help)
    {
        printf("Quicklz implementation version 1:\n");
        printf("\n");
        printf("usage: quicklz [options]:\n");
        printf("\n");
        printf("   -f file - input file that will be compress or decompress. This can be stdin.\n");
        printf("   -h      - show this help message.\n");
        printf("   -o file - output file. This can be stdout.\n");
        printf("   -d      - will decompress the file given via stdin or via -f option.\n");
        printf("\n");
        printf("Examples:.\n");
        printf("\n");
        printf("   cat file.qlz | quicklz -d > file\n");
        printf("   cat file | quicklz > file.qlz\n");
        printf("   quicklz -i file -o file.qlz\n");
        printf("   quicklz -d -i file.qlz -o file\n");

        exit(0);
    }

    do
    {
        bytesRead = fread(
            inputBuffer,
            1,
            1,
            input_file
        );
        char *tmp = (char*)realloc(src, input_file_size + bytesRead);
        if (tmp)
        {
            src = tmp;
            memmove(&src[input_file_size], inputBuffer, bytesRead);
            input_file_size += bytesRead;
        }
        else
        {
            free(src);
            exit(1);
        }
    } while (!feof(input_file));

    if(compress)
    {
        qlz_state_compress *state_compress = (qlz_state_compress *)malloc(sizeof(qlz_state_compress));

        dst = (char*) malloc(input_file_size + 400);
        output_file_size = qlz_compress(src, dst, input_file_size, state_compress);
    }
    else if(decompress)
    {
        qlz_state_decompress *state_decompress = (qlz_state_decompress *)malloc(sizeof(qlz_state_decompress));

        output_file_size = qlz_size_decompressed(src);
        dst = (char*) malloc(output_file_size);

        output_file_size = qlz_decompress(src, dst, state_decompress);
    }

    fwrite(dst, output_file_size, 1, output_file);

    fclose(input_file);
    fclose(output_file);
    return 0;
}