int __lzo_cdecl_main main(int argc, char *argv[]) { int r; lzo_bytep in; lzo_uint in_len; lzo_bytep out; lzo_uint out_bufsize; lzo_uint out_len = 0; lzo_voidp wrkmem; lzo_uint wrkmem_size; lzo_uint best_len; int best_compress = -1; lzo_uint orig_len; lzo_uint32_t uncompressed_checksum; lzo_uint32_t compressed_checksum; FILE *fp; const char *in_name = NULL; const char *out_name = NULL; long l; lzo_wildargv(&argc, &argv); printf("\nLZO real-time data compression library (v%s, %s).\n", lzo_version_string(), lzo_version_date()); printf("Copyright (C) 1996-2017 Markus Franz Xaver Johannes Oberhumer\nAll Rights Reserved.\n\n"); progname = argv[0]; if (argc < 2 || argc > 3) { printf("usage: %s file [output-file]\n", progname); exit(1); } in_name = argv[1]; if (argc > 2) out_name = argv[2]; /* * Step 1: initialize the LZO library */ if (lzo_init() != LZO_E_OK) { printf("internal error - lzo_init() failed !!!\n"); printf("(this usually indicates a compiler bug - try recompiling\nwithout optimizations, and enable '-DLZO_DEBUG' for diagnostics)\n"); exit(1); } /* * Step 2: allocate the work-memory */ wrkmem_size = 1; #ifdef USE_LZO1X wrkmem_size = (LZO1X_999_MEM_COMPRESS > wrkmem_size) ? LZO1X_999_MEM_COMPRESS : wrkmem_size; #endif #ifdef USE_LZO1Y wrkmem_size = (LZO1Y_999_MEM_COMPRESS > wrkmem_size) ? LZO1Y_999_MEM_COMPRESS : wrkmem_size; #endif wrkmem = (lzo_voidp) xmalloc(wrkmem_size); if (wrkmem == NULL) { printf("%s: out of memory\n", progname); exit(1); } /* * Step 3: open the input file */ fp = fopen(in_name,"rb"); if (fp == NULL) { printf("%s: cannot open file %s\n", progname, in_name); exit(1); } fseek(fp, 0, SEEK_END); l = ftell(fp); fseek(fp, 0, SEEK_SET); if (l <= 0) { printf("%s: %s: empty file\n", progname, in_name); fclose(fp); fp = NULL; exit(1); } in_len = (lzo_uint) l; out_bufsize = in_len + in_len / 16 + 64 + 3; best_len = in_len; /* * Step 4: allocate compression buffers and read the file */ in = (lzo_bytep) xmalloc(in_len); out = (lzo_bytep) xmalloc(out_bufsize); if (in == NULL || out == NULL) { printf("%s: out of memory\n", progname); exit(1); } in_len = (lzo_uint) lzo_fread(fp, in, in_len); printf("%s: loaded file %s: %ld bytes\n", progname, in_name, (long) in_len); fclose(fp); fp = NULL; /* * Step 5: compute a checksum of the uncompressed data */ uncompressed_checksum = lzo_adler32(0,NULL,0); uncompressed_checksum = lzo_adler32(uncompressed_checksum,in,in_len); /* * Step 6a: compress from 'in' to 'out' with LZO1X-999 */ #ifdef USE_LZO1X out_len = out_bufsize; r = lzo1x_999_compress(in,in_len,out,&out_len,wrkmem); if (r != LZO_E_OK) { /* this should NEVER happen */ printf("internal error - compression failed: %d\n", r); exit(1); } printf("LZO1X-999: %8lu -> %8lu\n", (unsigned long) in_len, (unsigned long) out_len); if (out_len < best_len) { best_len = out_len; best_compress = 1; /* LZO1X-999 */ } #endif /* USE_LZO1X */ /* * Step 6b: compress from 'in' to 'out' with LZO1Y-999 */ #ifdef USE_LZO1Y out_len = out_bufsize; r = lzo1y_999_compress(in,in_len,out,&out_len,wrkmem); if (r != LZO_E_OK) { /* this should NEVER happen */ printf("internal error - compression failed: %d\n", r); exit(1); } printf("LZO1Y-999: %8lu -> %8lu\n", (unsigned long) in_len, (unsigned long) out_len); if (out_len < best_len) { best_len = out_len; best_compress = 2; /* LZO1Y-999 */ } #endif /* USE_LZO1Y */ /* * Step 7: check if compressible */ if (best_len >= in_len) { printf("This file contains incompressible data.\n"); return 0; } /* * Step 8: compress data again using the best compressor found */ out_len = out_bufsize; if (best_compress == 1) r = lzo1x_999_compress(in,in_len,out,&out_len,wrkmem); else if (best_compress == 2) r = lzo1y_999_compress(in,in_len,out,&out_len,wrkmem); else r = -100; assert(r == LZO_E_OK); assert(out_len == best_len); /* * Step 9: optimize compressed data (compressed data is in 'out' buffer) */ #if 1 /* Optimization does not require any data in the buffer that will * hold the uncompressed data. To prove this, we clear the buffer. */ lzo_memset(in,0,in_len); #endif orig_len = in_len; r = -100; #ifdef USE_LZO1X if (best_compress == 1) r = lzo1x_optimize(out,out_len,in,&orig_len,NULL); #endif #ifdef USE_LZO1Y if (best_compress == 2) r = lzo1y_optimize(out,out_len,in,&orig_len,NULL); #endif if (r != LZO_E_OK || orig_len != in_len) { /* this should NEVER happen */ printf("internal error - optimization failed: %d\n", r); exit(1); } /* * Step 10: compute a checksum of the compressed data */ compressed_checksum = lzo_adler32(0,NULL,0); compressed_checksum = lzo_adler32(compressed_checksum,out,out_len); /* * Step 11: write compressed data to a file */ printf("%s: %s: %ld -> %ld, checksum 0x%08lx 0x%08lx\n", progname, in_name, (long) in_len, (long) out_len, (long) uncompressed_checksum, (long) compressed_checksum); if (out_name && out_name[0]) { printf("%s: writing to file %s\n", progname, out_name); fp = fopen(out_name,"wb"); if (fp == NULL) { printf("%s: cannot open output file %s\n", progname, out_name); exit(1); } if (lzo_fwrite(fp, out, out_len) != out_len || fclose(fp) != 0) { printf("%s: write error !!\n", progname); exit(1); } } /* * Step 12: verify decompression */ #ifdef PARANOID lzo_memset(in,0,in_len); /* paranoia - clear output buffer */ orig_len = in_len; r = -100; #ifdef USE_LZO1X if (best_compress == 1) r = lzo1x_decompress_safe(out,out_len,in,&orig_len,NULL); #endif #ifdef USE_LZO1Y if (best_compress == 2) r = lzo1y_decompress_safe(out,out_len,in,&orig_len,NULL); #endif if (r != LZO_E_OK || orig_len != in_len) { /* this should NEVER happen */ printf("internal error - decompression failed: %d\n", r); exit(1); } if (uncompressed_checksum != lzo_adler32(lzo_adler32(0,NULL,0),in,in_len)) { /* this should NEVER happen */ printf("internal error - decompression data error\n"); exit(1); } /* Now you could also verify decompression under similar conditions as in * your application, e.g. overlapping assembler decompression etc. */ #endif lzo_free(in); lzo_free(out); lzo_free(wrkmem); return 0; }
void xrCompressor::CompressOne(LPCSTR path) { filesTOTAL ++; if (testSKIP(path)) { filesSKIP ++; printf (" - a SKIP"); Msg ("%-80s - SKIP",path); return; } string_path fn; strconcat (sizeof(fn), fn, target_name.c_str(), "\\", path); if (::GetFileAttributes(fn)==u32(-1)) { filesSKIP ++; printf (" - CAN'T OPEN"); Msg ("%-80s - CAN'T OPEN",path); return; } IReader* src = FS.r_open (fn); if (0==src) { filesSKIP ++; printf (" - CAN'T OPEN"); Msg ("%-80s - CAN'T OPEN",path); return; } bytesSRC += src->length (); u32 c_crc32 = crc32 (src->pointer(),src->length()); u32 c_ptr = 0; u32 c_size_real = 0; u32 c_size_compressed = 0; u32 a_tests = 0; ALIAS* A = testALIAS (src,c_crc32,a_tests); printf ("%3da ",a_tests); if(A) { filesALIAS ++; printf ("ALIAS"); Msg ("%-80s - ALIAS (%s)",path,A->path); // Alias found c_ptr = A->c_ptr; c_size_real = A->c_size_real; c_size_compressed = A->c_size_compressed; } else { if (testVFS(path)) { filesVFS ++; // Write into BaseFS c_ptr = fs_pack_writer->tell (); c_size_real = src->length(); c_size_compressed = src->length(); fs_pack_writer->w (src->pointer(),c_size_real); printf ("VFS"); Msg ("%-80s - VFS",path); } else { //if(testVFS(path)) // Compress into BaseFS c_ptr = fs_pack_writer->tell(); c_size_real = src->length(); if (0!=c_size_real) { u32 c_size_max = rtc_csize (src->length()); u8* c_data = xr_alloc<u8> (c_size_max); t_compress.Begin (); c_size_compressed = c_size_max; if (bFast) { R_ASSERT(LZO_E_OK == lzo1x_1_compress ((u8*)src->pointer(),c_size_real,c_data,&c_size_compressed,c_heap)); }else { R_ASSERT(LZO_E_OK == lzo1x_999_compress ((u8*)src->pointer(),c_size_real,c_data,&c_size_compressed,c_heap)); } t_compress.End (); if ((c_size_compressed+16) >= c_size_real) { // Failed to compress - revert to VFS filesVFS ++; c_size_compressed = c_size_real; fs_pack_writer->w (src->pointer(),c_size_real); printf ("VFS (R)"); Msg ("%-80s - VFS (R)",path); } else { // Compressed OK - optimize if (!bFast) { u8* c_out = xr_alloc<u8> (c_size_real); u32 c_orig = c_size_real; R_ASSERT (LZO_E_OK == lzo1x_optimize (c_data,c_size_compressed,c_out,&c_orig, NULL)); R_ASSERT (c_orig == c_size_real ); xr_free (c_out); }//bFast fs_pack_writer->w (c_data,c_size_compressed); printf ("%3.1f%%", 100.f*float(c_size_compressed)/float(src->length())); Msg ("%-80s - OK (%3.1f%%)",path,100.f*float(c_size_compressed)/float(src->length())); } // cleanup xr_free (c_data); }else { //0!=c_size_real filesVFS ++; c_size_compressed = c_size_real; printf ("VFS (R)"); Msg ("%-80s - EMPTY FILE",path); } }//test VFS } //(A) // Write description write_file_header (path,c_crc32,c_ptr,c_size_real,c_size_compressed); if (0==A) { // Register for future aliasing ALIAS R; R.path = xr_strdup (fn); R.crc = c_crc32; R.c_ptr = c_ptr; R.c_size_real = c_size_real; R.c_size_compressed = c_size_compressed; aliases.insert (mk_pair(R.c_size_real,R)); } FS.r_close (src); }
//--------------------------------------------------------------------------- LZO1X & LZO1X::compress(AutoPtrBuffer & buf,uint8_t * & p,int32_t & len) { int r = 0; lzo_uint dst_len = 0; if( level_ > 0 ){ buf.realloc(wBufPos_ + wBufPos_ / 16 + 64 + 3 + sizeof(int32_t) * 2 + (crc_ != CRCNone) * sizeof(uint32_t)); switch( method_ ){ case LZO1X_1 : r = lzo1x_1_compress( (const lzo_bytep) (wBuf_.ptr() + sizeof(int32_t)), wBufPos_, (lzo_bytep) (buf.ptr() + sizeof(int32_t) * 2), &dst_len, (lzo_voidp) wWrkMem_.ptr() ); break; case LZO1X_1_11 : r = lzo1x_1_11_compress( (const lzo_bytep) (wBuf_.ptr() + sizeof(int32_t)), wBufPos_, (lzo_bytep) (buf.ptr() + sizeof(int32_t) * 2), &dst_len, (lzo_voidp) wWrkMem_.ptr() ); break; case LZO1X_1_12 : r = lzo1x_1_11_compress( (const lzo_bytep) (wBuf_.ptr() + sizeof(int32_t)), wBufPos_, (lzo_bytep) (buf.ptr() + sizeof(int32_t) * 2), &dst_len, (lzo_voidp) wWrkMem_.ptr() ); break; case LZO1X_1_15 : r = lzo1x_1_15_compress( (const lzo_bytep) (wBuf_.ptr() + sizeof(int32_t)), wBufPos_, (lzo_bytep) (buf.ptr() + sizeof(int32_t) * 2), &dst_len, (lzo_voidp) wWrkMem_.ptr() ); break; case LZO1X_999 : r = lzo1x_999_compress_level( (const lzo_bytep) (wBuf_.ptr() + sizeof(int32_t)), wBufPos_, (lzo_bytep) (buf.ptr() + sizeof(int32_t) * 2), &dst_len, (lzo_voidp) wWrkMem_.ptr(), NULL, 0, NULL, level_ ); break; default : assert( 0 ); } assert( r == LZO_E_OK ); if( r != LZO_E_OK ) newObjectV1C2<Exception>(EINVAL,__PRETTY_FUNCTION__)->throwSP(); if( dst_len >= wBufPos_ ) goto l1; if( optimize_ ){ lzo_uint orig_len = wBufPos_; r = lzo1x_optimize( (lzo_bytep) (buf.ptr() + sizeof(int32_t) * 2), dst_len, (lzo_bytep) (wBuf_.ptr() + sizeof(int32_t)), &orig_len, NULL ); assert( r == LZO_E_OK ); } if( crc_ != CRCNone ){ lzo_uint32 checksum = 0; if( crc_ == CRC32 ){ checksum = lzo_crc32(0,NULL,0); checksum = lzo_crc32(checksum,buf.ptr() + sizeof(int32_t) * 2,dst_len); } else if( crc_ == ADLER32 ){ checksum = lzo_adler32(0,NULL,0); checksum = lzo_adler32(checksum,buf.ptr() + sizeof(int32_t) * 2,dst_len); } *(uint32_t *)(buf.ptr() + sizeof(int32_t) * 2 + dst_len) = checksum; } dst_len += sizeof(int32_t) * 2 + (crc_ != CRCNone) * sizeof(uint32_t); ((int32_t *) buf.ptr())[0] = (int32_t) dst_len; ((int32_t *) buf.ptr())[1] = wBufPos_; p = buf; len = (int32_t) dst_len; } else { l1: ((int32_t *) wBuf_.ptr())[0] = -int32_t(wBufPos_); p = wBuf_; len = (int32_t) (wBufPos_ + sizeof(int32_t)); } return *this; }
int main(int argc, char *argv[]) { int r; int lazy; const int max_try_lazy = 5; const lzo_uint big = 65536L; /* can result in very slow compression */ const lzo_uint32 flags = 0x1; lzo_byte *in; lzo_uint in_len; lzo_byte *out; lzo_uint out_len = 0; lzo_byte *wrkmem; lzo_uint wrk_len; lzo_uint best_len; int best_compress = -1; int best_lazy = -1; lzo_uint orig_len; lzo_uint32 uncompressed_checksum; lzo_uint32 compressed_checksum; FILE *f; const char *progname = NULL; const char *in_name = NULL; const char *out_name = NULL; long l; #if defined(__EMX__) _response(&argc,&argv); _wildcard(&argc,&argv); #endif printf("\nLZO real-time data compression library (v%s, %s).\n", lzo_version_string(), lzo_version_date()); printf("Copyright (C) 1996-2002 Markus Franz Xaver Johannes Oberhumer\n\n"); progname = argv[0]; if (argc < 2 || argc > 3) { printf("usage: %s file [output-file]\n", progname); exit(1); } in_name = argv[1]; out_name = (argc > 2) ? argv[2] : NULL; /* * Step 1: initialize the LZO library */ if (lzo_init() != LZO_E_OK) { printf("lzo_init() failed !!!\n"); exit(1); } /* * Step 2: allocate the work-memory */ wrk_len = 1; #ifdef USE_LZO1X if (wrk_len < LZO1X_999_MEM_COMPRESS) wrk_len = LZO1X_999_MEM_COMPRESS; #endif #ifdef USE_LZO1Y if (wrk_len < LZO1Y_999_MEM_COMPRESS) wrk_len = LZO1Y_999_MEM_COMPRESS; #endif wrkmem = (lzo_bytep) lzo_malloc(wrk_len); if (wrkmem == NULL) { printf("%s: out of memory\n", progname); exit(1); } /* * Step 3: open the input file */ f = fopen(in_name,"rb"); if (f == NULL) { printf("%s: cannot open file %s\n", progname, in_name); exit(1); } fseek(f,0,SEEK_END); l = ftell(f); fseek(f,0,SEEK_SET); if (l <= 0) { printf("%s: %s: empty file\n", progname, in_name); fclose(f); exit(1); } in_len = (lzo_uint) l; best_len = in_len; /* * Step 4: allocate compression buffers and read the file */ in = (lzo_bytep) lzo_malloc(in_len); out = (lzo_bytep) lzo_malloc(in_len + in_len / 64 + 16 + 3); if (in == NULL || out == NULL) { printf("%s: out of memory\n", progname); exit(1); } in_len = lzo_fread(f,in,in_len); printf("%s: loaded file %s: %ld bytes\n", progname, in_name, (long) in_len); fclose(f); /* * Step 5: compute a checksum of the uncompressed data */ uncompressed_checksum = lzo_adler32(0,NULL,0); uncompressed_checksum = lzo_adler32(uncompressed_checksum,in,in_len); /* * Step 6a: compress from `in' to `out' with LZO1X-999 */ #ifdef USE_LZO1X for (lazy = 0; lazy <= max_try_lazy; lazy++) { r = lzo1x_999_compress_internal(in,in_len,out,&out_len,wrkmem, NULL, 0, 0, lazy, big, big, big, big, flags); if (r != LZO_E_OK) { /* this should NEVER happen */ printf("internal error - compression failed: %d\n", r); exit(1); } printf("LZO1X-999: lazy =%2d: %8lu -> %8lu\n", lazy, (long) in_len, (long) out_len); if (out_len < best_len) { best_len = out_len; best_lazy = lazy; best_compress = 1; /* LZO1X-999 */ } } #endif /* USE_LZO1X */ /* * Step 6b: compress from `in' to `out' with LZO1Y-999 */ #ifdef USE_LZO1Y for (lazy = 0; lazy <= max_try_lazy; lazy++) { r = lzo1y_999_compress_internal(in,in_len,out,&out_len,wrkmem, NULL, 0, 0, lazy, big, big, big, big, flags); if (r != LZO_E_OK) { /* this should NEVER happen */ printf("internal error - compression failed: %d\n", r); exit(1); } printf("LZO1Y-999: lazy =%2d: %8lu -> %8lu\n", lazy, (long) in_len, (long) out_len); if (out_len < best_len) { best_len = out_len; best_lazy = lazy; best_compress = 2; /* LZO1Y-999 */ } } #endif /* USE_LZO1Y */ /* * Step 7: check if compressible */ if (best_len >= in_len) { printf("This file contains incompressible data.\n"); return 0; } /* * Step 8: compress data again using the best compressor found */ if (best_compress == 1) r = lzo1x_999_compress_internal(in,in_len,out,&out_len,wrkmem, NULL, 0, 0, best_lazy, big, big, big, big, flags); else if (best_compress == 2) r = lzo1y_999_compress_internal(in,in_len,out,&out_len,wrkmem, NULL, 0, 0, best_lazy, big, big, big, big, flags); else r = -100; assert(r == LZO_E_OK); assert(out_len == best_len); /* * Step 9: optimize compressed data (compressed data is in `out' buffer) */ #if 1 /* Optimization does not require any data in the buffer that will * hold the uncompressed data. To prove this, we clear the buffer. */ lzo_memset(in,0,in_len); #endif orig_len = in_len; if (best_compress == 1) r = lzo1x_optimize(out,out_len,in,&orig_len,NULL); else if (best_compress == 2) r = lzo1y_optimize(out,out_len,in,&orig_len,NULL); else r = -100; if (r != LZO_E_OK || orig_len != in_len) { /* this should NEVER happen */ printf("internal error - optimization failed: %d\n", r); exit(1); } /* * Step 10: compute a checksum of the compressed data */ compressed_checksum = lzo_adler32(0,NULL,0); compressed_checksum = lzo_adler32(compressed_checksum,out,out_len); /* * Step 11: write compressed data to a file */ printf("%s: %s: %ld -> %ld, checksum 0x%08lx 0x%08lx\n", progname, in_name, (long) in_len, (long) out_len, (long) uncompressed_checksum, (long) compressed_checksum); if (out_name && out_name[0]) { printf("%s: writing to file %s\n", progname, out_name); f = fopen(out_name,"wb"); if (f == NULL) { printf("%s: cannot open output file %s\n", progname, out_name); exit(1); } if (lzo_fwrite(f,out,out_len) != out_len || fclose(f) != 0) { printf("%s: write error !!\n", progname); exit(1); } } /* * Step 12: verify decompression */ #ifdef PARANOID orig_len = in_len; if (best_compress == 1) r = lzo1x_decompress(out,out_len,in,&orig_len,NULL); else if (best_compress == 2) r = lzo1y_decompress(out,out_len,in,&orig_len,NULL); else r = -100; if (r != LZO_E_OK || orig_len != in_len) { /* this should NEVER happen */ printf("internal error - decompression failed: %d\n", r); exit(1); } if (uncompressed_checksum != lzo_adler32(lzo_adler32(0,NULL,0),in,in_len)) { /* this should NEVER happen */ printf("internal error - decompression data error\n"); exit(1); } /* Now you could also verify decompression under similar conditions as in * your application, e.g. overlapping assembler decompression etc. */ #endif lzo_free(in); lzo_free(out); lzo_free(wrkmem); return 0; }
lzo_bool lzo_compress(file_t *fip, file_t *fop, const header_t *h) { int r = LZO_E_OK; lzo_byte * const b1 = block1.mem; lzo_byte * const b2 = block2.mem; lzo_uint32 src_len = 0, dst_len = 0; lzo_uint32 c_adler32 = ADLER32_INIT_VALUE, d_adler32 = ADLER32_INIT_VALUE; lzo_uint32 c_crc32 = CRC32_INIT_VALUE, d_crc32 = CRC32_INIT_VALUE; lzo_int l; lzo_bool ok = 1; for (;;) { /* read a block */ l = read_buf(fip, b1, block_size); src_len = (l > 0 ? l : 0); /* write uncompressed block size */ write32(fop,src_len); /* exit if last block */ if (src_len == 0) break; /* compute checksum of uncompressed block */ if (h->flags & F_ADLER32_D) d_adler32 = lzo_adler32(ADLER32_INIT_VALUE,b1,src_len); if (h->flags & F_CRC32_D) d_crc32 = lzo_crc32(CRC32_INIT_VALUE,b1,src_len); x_filter(b1,src_len,h); /* compress */ if (h->method == M_LZO1X_1) r = lzo1x_1_compress(b1, src_len, b2, &dst_len, wrkmem.mem); #if defined(USE_LZO1X_1_15) else if (h->method == M_LZO1X_1_15) r = lzo1x_1_15_compress(b1, src_len, b2, &dst_len, wrkmem.mem); #endif #if defined(USE_LZO1X_999) else if (h->method == M_LZO1X_999) r = lzo1x_999_compress_level(b1, src_len, b2, &dst_len, wrkmem.mem, NULL, 0, 0, h->level); #endif else fatal(fip,"Internal error"); #if 0 fprintf(stderr, "%ld %ld %ld\n", (long)src_len, (long)dst_len, (long)block2.size); #endif assert(dst_len <= block2.size); if (r != LZO_E_OK) fatal(fip,"Internal error - compression failed"); /* optimize */ if (opt_optimize && dst_len < src_len) { lzo_uint32 new_len = src_len; r = lzo1x_optimize(b2, dst_len, b1, &new_len, NULL); if (r != LZO_E_OK || new_len != src_len) fatal(fip,"Internal error - optimization failed"); } /* write compressed block size */ if (dst_len < src_len) write32(fop,dst_len); else write32(fop,src_len); /* write checksum of uncompressed block */ if (h->flags & F_ADLER32_D) write32(fop,d_adler32); if (h->flags & F_CRC32_D) write32(fop,d_crc32); /* write checksum of compressed block */ if (dst_len < src_len && (h->flags & F_ADLER32_C)) { c_adler32 = lzo_adler32(ADLER32_INIT_VALUE,b2,dst_len); write32(fop,c_adler32); } if (dst_len < src_len && (h->flags & F_CRC32_C)) { c_crc32 = lzo_crc32(CRC32_INIT_VALUE,b2,dst_len); write32(fop,c_crc32); } /* write compressed block data */ if (dst_len < src_len) write_buf(fop,b2,dst_len); else write_buf(fop,b1,src_len); } return ok; }