static length_t compress_packet(uint8_t *dest, const uint8_t *source, length_t len, int level) {
if(level == 0) {
memcpy(dest, source, len);
return len;
} else if(level == 10) {
#ifdef HAVE_LZO
lzo_uint lzolen = MAXSIZE;
lzo1x_1_compress(source, len, dest, &lzolen, lzo_wrkmem);
return lzolen;
#else
return -1;
#endif
} else if(level < 10) {
#ifdef HAVE_ZLIB
unsigned long destlen = MAXSIZE;
if(compress2(dest, &destlen, source, len, level) == Z_OK)
return destlen;
else
#endif
return -1;
} else {
#ifdef HAVE_LZO
lzo_uint lzolen = MAXSIZE;
lzo1x_999_compress(source, len, dest, &lzolen, lzo_wrkmem);
return lzolen;
#else
return -1;
#endif
}
return -1;
}
static int lzo_compress(void *strm, void *d, void *s, int size, int block_size,
int *error)
{
int res;
lzo_uint outlen;
struct lzo_stream *stream = strm;
res = lzo1x_999_compress(s, size, stream->out, &outlen, stream->wrkmem);
if(res != LZO_E_OK)
goto failed;
if(outlen >= size)
/*
* Output buffer overflow. Return out of buffer space
*/
return 0;
/*
* Success, return the compressed size.
*/
memcpy(d, stream->out, outlen);
return outlen;
failed:
/*
* All other errors return failure, with the compressor
* specific error code in *error
*/
*error = res;
return -1;
}
static int do_flush_ocstream(struct ocstream *ocs)
{
#if WITH_LZO
int retval;
lzo_uint csize;
unsigned short int *optr = ocs->obuf;
if (!ocs->used)
return 0;
retval = lzo1x_999_compress(ocs->ibuf, ocs->used,
(unsigned char*)(optr + 2), &csize,
ocs->cbuf);
if (LZO_E_OK != retval)
goto fail_lzo;
optr[0] = csize;
optr[1] = ocs->used;
csize += 2 * sizeof(*optr);
if (write_ostream(ocs->impl, ocs->obuf, csize) < csize)
goto fail_io;
ocs->used = 0;
return 0;
fail_io:
return -1;
fail_lzo:
return -2;
#else
if (write_ostream(ocs->impl, ocs->ibuf, ocs->used) < ocs->used)
return -1;
ocs->used = 0;
return 0;
#endif
}
static int lzo_compress(void *in_buf, size_t in_len, void *out_buf,
size_t *out_len)
{
lzo_uint len;
int ret;
len = *out_len;
ret = lzo1x_999_compress(in_buf, in_len, out_buf, &len, lzo_mem);
*out_len = len;
if (ret != LZO_E_OK) {
errcnt += 1;
return -1;
}
return 0;
}
/*
* Note about LZO compression.
*
* We want to use the _999_ compression routine which gives better compression
* rates at the expense of time. Decompression time is unaffected. We might as
* well use the standard lzo library routines for this but they will overflow
* the destination buffer since they don't check the destination size.
*
* We therefore compress to a temporary buffer and copy if it will fit.
*
*/
static int jffs2_lzo_cmpr(unsigned char *data_in, unsigned char *cpage_out,
uint32_t *sourcelen, uint32_t *dstlen, void *model)
{
uint32_t compress_size;
int ret;
ret = lzo1x_999_compress(data_in, *sourcelen, lzo_compress_buf, &compress_size, lzo_mem);
if (ret != LZO_E_OK)
return -1;
if (compress_size > *dstlen)
return -1;
memcpy(cpage_out, lzo_compress_buf, compress_size);
*dstlen = compress_size;
return 0;
}
int64_t lzbench_lzo1x_compress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t level, size_t, char* workmem)
{
lzo_uint lzo_complen = 0;
int res;
if (!workmem)
return 0;
switch (level)
{
default:
case 1: res = lzo1x_1_compress((uint8_t*)inbuf, insize, (uint8_t*)outbuf, &lzo_complen, (void*)workmem); break;
case 11: res = lzo1x_1_11_compress((uint8_t*)inbuf, insize, (uint8_t*)outbuf, &lzo_complen, (void*)workmem); break;
case 12: res = lzo1x_1_12_compress((uint8_t*)inbuf, insize, (uint8_t*)outbuf, &lzo_complen, (void*)workmem); break;
case 15: res = lzo1x_1_15_compress((uint8_t*)inbuf, insize, (uint8_t*)outbuf, &lzo_complen, (void*)workmem); break;
case 999: res = lzo1x_999_compress((uint8_t*)inbuf, insize, (uint8_t*)outbuf, &lzo_complen, (void*)workmem); break;
}
if (res != LZO_E_OK) return 0;
return lzo_complen;
}
static void storage__append_block_simple(struct storage__file* c, unsigned char* buf, unsigned char hash) {
assert(c->opened_for_writing!=0);
int inside_block_group_offset = c->current_block % c->block_group_size;
if (inside_block_group_offset==0) {
off_t data_file_offset = ftello(c->data_file);
c->current_index_entry->base_offset = data_file_offset;
}
lzo_uint len = CHUNK;
if (!c->best_compression) {
char tmp[LZO1X_1_MEM_COMPRESS];
lzo1x_1_compress(buf, c->block_size, c->outbuf, &len, &tmp);
} else {
char tmp[LZO1X_999_MEM_COMPRESS];
lzo1x_999_compress(buf, c->block_size, c->outbuf, &len, &tmp);
}
if (len >= c->block_size*63/64) {
// this block looks uncompressible or poorly compressible
int written = fwrite(buf, 1, c->block_size, c->data_file);
assert(written == c->block_size);
len = -0x7FFF; // 80 01 on disk
++c->writestat_uncompressible;
} else {
int written = fwrite(c->outbuf, 1, len, c->data_file);
assert(written == len);
++c->writestat_compressed;
}
c->current_index_entry->offsets[inside_block_group_offset] = len;
fputc(hash, c->hash_file);
++c->current_block;
if (inside_block_group_offset == c->block_group_size-1) {
storage__flush_index_entry(c);
}
}
static VALUE lzoruby_compress(int argc, const VALUE *argv, VALUE self) {
const lzo_bytep in;
lzo_bytep out;
lzo_voidp wrkmem = NULL;
lzo_uint in_len;
lzo_uint out_len;
lzo_uint new_len;
VALUE v_in, v_out, v_level;
int level, err;
rb_scan_args(argc, argv, "11", &v_in, &v_level);
Check_Type(v_in, T_STRING);
level = NIL_P(v_level) ? 1 : NUM2INT(v_level);
in = RSTRING_PTR(v_in);
in_len = RSTRING_LEN(v_in);
out_len = in_len + in_len / 64 + 16 + 3;
out = xmalloc(out_len);
new_len = out_len;
wrkmem = xmalloc((level == 1) ? LZO1X_1_MEM_COMPRESS : LZO1X_999_MEM_COMPRESS);
if (level == 1) {
err = lzo1x_1_compress(in, in_len, out, &new_len, wrkmem);
} else {
err = lzo1x_999_compress(in, in_len, out, &new_len, wrkmem);
}
xfree(wrkmem);
if (err != LZO_E_OK || new_len > out_len) {
xfree(out);
rb_raise(LZO_Error, "Error %d while compressing data", err);
}
v_out = rb_str_new(out, new_len);
xfree(out);
return v_out;
}
void
Compress(const std::vector<T>& data, BufferT& out)
{
ssize_t countBS = sizeof(T)*data.size();
lzo_uint outLen=(countBS + countBS / 16 + 64 + 3); // from simple.c in LZO distrib
// allocate as much memory as we need
out.resize(outLen);
int32_t r = lzo1x_999_compress(
reinterpret_cast<const lzo_bytep>(&data.front()),
countBS,
&out.front(),
&outLen,
&lzoTemp[0]);
if (r != LZO_E_OK) {
Util::fire_exception("lzo compression failed");
}
// trim to really used size
out.resize(outLen);
}
int __lzo_cdecl_main main(int argc, char *argv[])
{
int r;
lzo_bytep in;
lzo_uint in_len;
lzo_bytep out;
lzo_uint out_bufsize;
lzo_uint out_len = 0;
lzo_voidp wrkmem;
lzo_uint wrkmem_size;
lzo_uint best_len;
int best_compress = -1;
lzo_uint orig_len;
lzo_uint32_t uncompressed_checksum;
lzo_uint32_t compressed_checksum;
FILE *fp;
const char *in_name = NULL;
const char *out_name = NULL;
long l;
lzo_wildargv(&argc, &argv);
printf("\nLZO real-time data compression library (v%s, %s).\n",
lzo_version_string(), lzo_version_date());
printf("Copyright (C) 1996-2017 Markus Franz Xaver Johannes Oberhumer\nAll Rights Reserved.\n\n");
progname = argv[0];
if (argc < 2 || argc > 3)
{
printf("usage: %s file [output-file]\n", progname);
exit(1);
}
in_name = argv[1];
if (argc > 2) out_name = argv[2];
/*
* Step 1: initialize the LZO library
*/
if (lzo_init() != LZO_E_OK)
{
printf("internal error - lzo_init() failed !!!\n");
printf("(this usually indicates a compiler bug - try recompiling\nwithout optimizations, and enable '-DLZO_DEBUG' for diagnostics)\n");
exit(1);
}
/*
* Step 2: allocate the work-memory
*/
wrkmem_size = 1;
#ifdef USE_LZO1X
wrkmem_size = (LZO1X_999_MEM_COMPRESS > wrkmem_size) ? LZO1X_999_MEM_COMPRESS : wrkmem_size;
#endif
#ifdef USE_LZO1Y
wrkmem_size = (LZO1Y_999_MEM_COMPRESS > wrkmem_size) ? LZO1Y_999_MEM_COMPRESS : wrkmem_size;
#endif
wrkmem = (lzo_voidp) xmalloc(wrkmem_size);
if (wrkmem == NULL)
{
printf("%s: out of memory\n", progname);
exit(1);
}
/*
* Step 3: open the input file
*/
fp = fopen(in_name,"rb");
if (fp == NULL)
{
printf("%s: cannot open file %s\n", progname, in_name);
exit(1);
}
fseek(fp, 0, SEEK_END);
l = ftell(fp);
fseek(fp, 0, SEEK_SET);
if (l <= 0)
{
printf("%s: %s: empty file\n", progname, in_name);
fclose(fp); fp = NULL;
exit(1);
}
in_len = (lzo_uint) l;
out_bufsize = in_len + in_len / 16 + 64 + 3;
best_len = in_len;
/*
* Step 4: allocate compression buffers and read the file
*/
in = (lzo_bytep) xmalloc(in_len);
out = (lzo_bytep) xmalloc(out_bufsize);
if (in == NULL || out == NULL)
{
printf("%s: out of memory\n", progname);
exit(1);
}
in_len = (lzo_uint) lzo_fread(fp, in, in_len);
printf("%s: loaded file %s: %ld bytes\n", progname, in_name, (long) in_len);
fclose(fp); fp = NULL;
/*
* Step 5: compute a checksum of the uncompressed data
*/
uncompressed_checksum = lzo_adler32(0,NULL,0);
uncompressed_checksum = lzo_adler32(uncompressed_checksum,in,in_len);
/*
* Step 6a: compress from 'in' to 'out' with LZO1X-999
*/
#ifdef USE_LZO1X
out_len = out_bufsize;
r = lzo1x_999_compress(in,in_len,out,&out_len,wrkmem);
if (r != LZO_E_OK)
{
/* this should NEVER happen */
printf("internal error - compression failed: %d\n", r);
exit(1);
}
printf("LZO1X-999: %8lu -> %8lu\n", (unsigned long) in_len, (unsigned long) out_len);
if (out_len < best_len)
{
best_len = out_len;
best_compress = 1; /* LZO1X-999 */
}
#endif /* USE_LZO1X */
/*
* Step 6b: compress from 'in' to 'out' with LZO1Y-999
*/
#ifdef USE_LZO1Y
out_len = out_bufsize;
r = lzo1y_999_compress(in,in_len,out,&out_len,wrkmem);
if (r != LZO_E_OK)
{
/* this should NEVER happen */
printf("internal error - compression failed: %d\n", r);
exit(1);
}
printf("LZO1Y-999: %8lu -> %8lu\n", (unsigned long) in_len, (unsigned long) out_len);
if (out_len < best_len)
{
best_len = out_len;
best_compress = 2; /* LZO1Y-999 */
}
#endif /* USE_LZO1Y */
/*
* Step 7: check if compressible
*/
if (best_len >= in_len)
{
printf("This file contains incompressible data.\n");
return 0;
}
/*
* Step 8: compress data again using the best compressor found
*/
out_len = out_bufsize;
if (best_compress == 1)
r = lzo1x_999_compress(in,in_len,out,&out_len,wrkmem);
else if (best_compress == 2)
r = lzo1y_999_compress(in,in_len,out,&out_len,wrkmem);
else
r = -100;
assert(r == LZO_E_OK);
assert(out_len == best_len);
/*
* Step 9: optimize compressed data (compressed data is in 'out' buffer)
*/
#if 1
/* Optimization does not require any data in the buffer that will
* hold the uncompressed data. To prove this, we clear the buffer.
*/
lzo_memset(in,0,in_len);
#endif
orig_len = in_len;
r = -100;
#ifdef USE_LZO1X
if (best_compress == 1)
r = lzo1x_optimize(out,out_len,in,&orig_len,NULL);
#endif
#ifdef USE_LZO1Y
if (best_compress == 2)
r = lzo1y_optimize(out,out_len,in,&orig_len,NULL);
#endif
if (r != LZO_E_OK || orig_len != in_len)
{
/* this should NEVER happen */
printf("internal error - optimization failed: %d\n", r);
exit(1);
}
/*
* Step 10: compute a checksum of the compressed data
*/
compressed_checksum = lzo_adler32(0,NULL,0);
compressed_checksum = lzo_adler32(compressed_checksum,out,out_len);
/*
* Step 11: write compressed data to a file
*/
printf("%s: %s: %ld -> %ld, checksum 0x%08lx 0x%08lx\n",
progname, in_name, (long) in_len, (long) out_len,
(long) uncompressed_checksum, (long) compressed_checksum);
if (out_name && out_name[0])
{
printf("%s: writing to file %s\n", progname, out_name);
fp = fopen(out_name,"wb");
if (fp == NULL)
{
printf("%s: cannot open output file %s\n", progname, out_name);
exit(1);
}
if (lzo_fwrite(fp, out, out_len) != out_len || fclose(fp) != 0)
{
printf("%s: write error !!\n", progname);
exit(1);
}
}
/*
* Step 12: verify decompression
*/
#ifdef PARANOID
lzo_memset(in,0,in_len); /* paranoia - clear output buffer */
orig_len = in_len;
r = -100;
#ifdef USE_LZO1X
if (best_compress == 1)
r = lzo1x_decompress_safe(out,out_len,in,&orig_len,NULL);
#endif
#ifdef USE_LZO1Y
if (best_compress == 2)
r = lzo1y_decompress_safe(out,out_len,in,&orig_len,NULL);
#endif
if (r != LZO_E_OK || orig_len != in_len)
{
/* this should NEVER happen */
printf("internal error - decompression failed: %d\n", r);
exit(1);
}
if (uncompressed_checksum != lzo_adler32(lzo_adler32(0,NULL,0),in,in_len))
{
/* this should NEVER happen */
printf("internal error - decompression data error\n");
exit(1);
}
/* Now you could also verify decompression under similar conditions as in
* your application, e.g. overlapping assembler decompression etc.
*/
#endif
lzo_free(in);
lzo_free(out);
lzo_free(wrkmem);
return 0;
}
void xrCompressor::CompressOne(LPCSTR path)
{
filesTOTAL ++;
if (testSKIP(path))
{
filesSKIP ++;
printf (" - a SKIP");
Msg ("%-80s - SKIP",path);
return;
}
string_path fn;
strconcat (sizeof(fn), fn, target_name.c_str(), "\\", path);
if (::GetFileAttributes(fn)==u32(-1))
{
filesSKIP ++;
printf (" - CAN'T OPEN");
Msg ("%-80s - CAN'T OPEN",path);
return;
}
IReader* src = FS.r_open (fn);
if (0==src)
{
filesSKIP ++;
printf (" - CAN'T OPEN");
Msg ("%-80s - CAN'T OPEN",path);
return;
}
bytesSRC += src->length ();
u32 c_crc32 = crc32 (src->pointer(),src->length());
u32 c_ptr = 0;
u32 c_size_real = 0;
u32 c_size_compressed = 0;
u32 a_tests = 0;
ALIAS* A = testALIAS (src,c_crc32,a_tests);
printf ("%3da ",a_tests);
if(A)
{
filesALIAS ++;
printf ("ALIAS");
Msg ("%-80s - ALIAS (%s)",path,A->path);
// Alias found
c_ptr = A->c_ptr;
c_size_real = A->c_size_real;
c_size_compressed = A->c_size_compressed;
} else
{
if (testVFS(path))
{
filesVFS ++;
// Write into BaseFS
c_ptr = fs_pack_writer->tell ();
c_size_real = src->length();
c_size_compressed = src->length();
fs_pack_writer->w (src->pointer(),c_size_real);
printf ("VFS");
Msg ("%-80s - VFS",path);
} else
{ //if(testVFS(path))
// Compress into BaseFS
c_ptr = fs_pack_writer->tell();
c_size_real = src->length();
if (0!=c_size_real)
{
u32 c_size_max = rtc_csize (src->length());
u8* c_data = xr_alloc<u8> (c_size_max);
t_compress.Begin ();
c_size_compressed = c_size_max;
if (bFast)
{
R_ASSERT(LZO_E_OK == lzo1x_1_compress ((u8*)src->pointer(),c_size_real,c_data,&c_size_compressed,c_heap));
}else
{
R_ASSERT(LZO_E_OK == lzo1x_999_compress ((u8*)src->pointer(),c_size_real,c_data,&c_size_compressed,c_heap));
}
t_compress.End ();
if ((c_size_compressed+16) >= c_size_real)
{
// Failed to compress - revert to VFS
filesVFS ++;
c_size_compressed = c_size_real;
fs_pack_writer->w (src->pointer(),c_size_real);
printf ("VFS (R)");
Msg ("%-80s - VFS (R)",path);
} else
{
// Compressed OK - optimize
if (!bFast)
{
u8* c_out = xr_alloc<u8> (c_size_real);
u32 c_orig = c_size_real;
R_ASSERT (LZO_E_OK == lzo1x_optimize (c_data,c_size_compressed,c_out,&c_orig, NULL));
R_ASSERT (c_orig == c_size_real );
xr_free (c_out);
}//bFast
fs_pack_writer->w (c_data,c_size_compressed);
printf ("%3.1f%%", 100.f*float(c_size_compressed)/float(src->length()));
Msg ("%-80s - OK (%3.1f%%)",path,100.f*float(c_size_compressed)/float(src->length()));
}
// cleanup
xr_free (c_data);
}else
{ //0!=c_size_real
filesVFS ++;
c_size_compressed = c_size_real;
printf ("VFS (R)");
Msg ("%-80s - EMPTY FILE",path);
}
}//test VFS
} //(A)
// Write description
write_file_header (path,c_crc32,c_ptr,c_size_real,c_size_compressed);
if (0==A)
{
// Register for future aliasing
ALIAS R;
R.path = xr_strdup (fn);
R.crc = c_crc32;
R.c_ptr = c_ptr;
R.c_size_real = c_size_real;
R.c_size_compressed = c_size_compressed;
aliases.insert (mk_pair(R.c_size_real,R));
}
FS.r_close (src);
}
/****************************************************************************
compress ISO
****************************************************************************/
int comp_ciso(int level, int no_comp_diff)
{
unsigned long long file_size;
unsigned long long write_pos;
int total_sectors;
int index_size;
int block;
unsigned char buf4[64];
int cmp_size;
int status;
int percent_period;
int percent_cnt;
int align,align_b,align_m;
lzo_voidp wrkmem;
unsigned long lzolen;
file_size = check_file_size(fin);
if(file_size<0)
{
printf("Can't get file size\n");
return 1;
}
/* allocate index block */
index_size = (ciso_total_block + 1 ) * sizeof(unsigned long);
index_buf = malloc(index_size);
crc_buf = malloc(index_size);
block_buf1 = malloc(ciso.block_size*2);
block_buf2 = malloc(ciso.block_size*2);
if( !index_buf || !crc_buf || !block_buf1 || !block_buf2 )
{
printf("Can't allocate memory\n");
return 1;
}
memset(index_buf,0,index_size);
memset(crc_buf,0,index_size);
memset(buf4,0,sizeof(buf4));
if(is_ziso) {
if(lzo_init() != LZO_E_OK) {
printf("lzo_init() failed\n");
return 1;
}
//wrkmem = (lzo_voidp) malloc(LZO1X_1_MEM_COMPRESS);
wrkmem = (lzo_voidp) malloc(LZO1X_999_MEM_COMPRESS);
} else {
/* init zlib */
z.zalloc = Z_NULL;
z.zfree = Z_NULL;
z.opaque = Z_NULL;
}
/* show info */
printf("Compress '%s' to '%s'\n",fname_in,fname_out);
printf("Total File Size %ld bytes\n",ciso.total_bytes);
printf("block size %d bytes\n",ciso.block_size);
printf("index align %d\n",1<<ciso.align);
printf("compress level %d\n",level);
printf("type %s\n", is_ziso ? "ZISO" : "CISO");
/* write header block */
fwrite(&ciso,1,sizeof(ciso),fout);
/* dummy write index block */
fwrite(index_buf,1,index_size,fout);
write_pos = sizeof(ciso) + index_size;
/* compress data */
percent_period = ciso_total_block/100;
percent_cnt = ciso_total_block/100;
align_b = 1<<(ciso.align);
align_m = align_b -1;
for(block = 0;block < ciso_total_block ; block++)
{
if(--percent_cnt<=0)
{
percent_cnt = percent_period;
printf("compress %3d%% avarage rate %3d%%\r"
,block / percent_period
,block==0 ? 0 : 100*write_pos/(block*0x800));
}
if(!is_ziso) {
if (deflateInit2(&z, level , Z_DEFLATED, -15,8,Z_DEFAULT_STRATEGY) != Z_OK)
{
printf("deflateInit : %s\n", (z.msg) ? z.msg : "???");
return 1;
}
}
/* write align */
align = (int)write_pos & align_m;
if(align)
{
align = align_b - align;
if(fwrite(buf4,1,align, fout) != align)
{
printf("block %d : Write error\n",block);
return 1;
}
write_pos += align;
}
/* mark offset index */
index_buf[block] = write_pos>>(ciso.align);
/* read buffer */
z.next_out = block_buf2;
z.avail_out = ciso.block_size*2;
z.next_in = block_buf1;
z.avail_in = fread(block_buf1, 1, ciso.block_size , fin);
if(z.avail_in != ciso.block_size)
{
printf("block=%d : read error\n",block);
return 1;
}
if(is_ziso) {
//status = lzo1x_1_compress(block_buf1, z.avail_in, block_buf2, &lzolen, wrkmem);
status = lzo1x_999_compress(block_buf1, z.avail_in, block_buf2, &lzolen, wrkmem);
//printf("in: %d out: %d\n", z.avail_in, lzolen);
if (status != LZO_E_OK) {
/* this should NEVER happen */
printf("compression failed: lzo1x_1_compress: %d\n", status);
return 1;
}
cmp_size = lzolen;
} else {
/* compress block
status = deflate(&z, Z_FULL_FLUSH);*/
status = deflate(&z, Z_FINISH);
if (status != Z_STREAM_END)
/* if (status != Z_OK) */
{
printf("block %d:deflate : %s[%d]\n", block,(z.msg) ? z.msg : "error",status);
return 1;
}
cmp_size = ciso.block_size*2 - z.avail_out;
}
/* choise plain / compress */
if(cmp_size >= (ciso.block_size - no_comp_diff))
{
cmp_size = ciso.block_size;
memcpy(block_buf2,block_buf1,cmp_size);
/* plain block mark */
index_buf[block] |= 0x80000000;
}
/* write compressed block */
if(fwrite(block_buf2, 1,cmp_size , fout) != cmp_size)
{
printf("block %d : Write error\n",block);
return 1;
}
/* mark next index */
write_pos += cmp_size;
if(!is_ziso) {
/* term zlib */
if (deflateEnd(&z) != Z_OK)
{
printf("deflateEnd : %s\n", (z.msg) ? z.msg : "error");
return 1;
}
}
}
/* last position (total size)*/
index_buf[block] = write_pos>>(ciso.align);
/* write header & index block */
fseek(fout,sizeof(ciso),SEEK_SET);
fwrite(index_buf,1,index_size,fout);
printf("ciso compress completed , total size = %8d bytes , rate %d%%\n"
,(int)write_pos,(int)(write_pos*100/ciso.total_bytes));
return 0;
}
int main(int argc, char *argv[])
{
int r;
lzo_byte *in;
lzo_uint in_len;
lzo_byte *out;
lzo_uint out_len = 0;
lzo_byte *wrkmem;
lzo_uint wrk_len;
lzo_uint best_len;
int best_compress = -1;
lzo_uint orig_len;
lzo_uint32 uncompressed_checksum;
lzo_uint32 compressed_checksum;
FILE *f;
const char *progname = NULL;
const char *in_name = NULL;
const char *out_name = NULL;
long l;
#if defined(__EMX__)
_response(&argc,&argv);
_wildcard(&argc,&argv);
#endif
printf("\nLZO real-time data compression library (v%s, %s).\n",
lzo_version_string(), lzo_version_date());
printf("Copyright (C) 1996-2002 Markus Franz Xaver Johannes Oberhumer\n\n");
progname = argv[0];
if (argc < 2 || argc > 3)
{
printf("usage: %s file [output-file]\n", progname);
exit(1);
}
in_name = argv[1];
out_name = (argc > 2) ? argv[2] : NULL;
/*
* Step 1: initialize the LZO library
*/
if (lzo_init() != LZO_E_OK)
{
printf("lzo_init() failed !!!\n");
exit(1);
}
/*
* Step 2: allocate the work-memory
*/
wrk_len = 1;
#ifdef USE_LZO1X
if (wrk_len < LZO1X_999_MEM_COMPRESS)
wrk_len = LZO1X_999_MEM_COMPRESS;
#endif
#ifdef USE_LZO1Y
if (wrk_len < LZO1Y_999_MEM_COMPRESS)
wrk_len = LZO1Y_999_MEM_COMPRESS;
#endif
wrkmem = (lzo_bytep) lzo_malloc(wrk_len);
if (wrkmem == NULL)
{
printf("%s: out of memory\n", progname);
exit(1);
}
/*
* Step 3: open the input file
*/
f = fopen(in_name,"rb");
if (f == NULL)
{
printf("%s: cannot open file %s\n", progname, in_name);
exit(1);
}
fseek(f,0,SEEK_END);
l = ftell(f);
fseek(f,0,SEEK_SET);
if (l <= 0)
{
printf("%s: %s: empty file\n", progname, in_name);
fclose(f);
exit(1);
}
in_len = (lzo_uint) l;
best_len = in_len;
/*
* Step 4: allocate compression buffers and read the file
*/
in = (lzo_bytep) lzo_malloc(in_len);
out = (lzo_bytep) lzo_malloc(in_len + in_len / 64 + 16 + 3);
if (in == NULL || out == NULL)
{
printf("%s: out of memory\n", progname);
exit(1);
}
in_len = lzo_fread(f,in,in_len);
printf("%s: loaded file %s: %ld bytes\n", progname, in_name, (long) in_len);
fclose(f);
/*
* Step 5: compute a checksum of the uncompressed data
*/
uncompressed_checksum = lzo_adler32(0,NULL,0);
uncompressed_checksum = lzo_adler32(uncompressed_checksum,in,in_len);
/*
* Step 6a: compress from `in' to `out' with LZO1X-999
*/
#ifdef USE_LZO1X
r = lzo1x_999_compress(in,in_len,out,&out_len,wrkmem);
if (r != LZO_E_OK)
{
/* this should NEVER happen */
printf("internal error - compression failed: %d\n", r);
exit(1);
}
printf("LZO1X-999: %8lu -> %8lu\n", (long) in_len, (long) out_len);
if (out_len < best_len)
{
best_len = out_len;
best_compress = 1; /* LZO1X-999 */
}
#endif /* USE_LZO1X */
/*
* Step 6b: compress from `in' to `out' with LZO1Y-999
*/
#ifdef USE_LZO1Y
r = lzo1y_999_compress(in,in_len,out,&out_len,wrkmem);
if (r != LZO_E_OK)
{
/* this should NEVER happen */
printf("internal error - compression failed: %d\n", r);
exit(1);
}
printf("LZO1Y-999: %8lu -> %8lu\n", (long) in_len, (long) out_len);
if (out_len < best_len)
{
best_len = out_len;
best_compress = 2; /* LZO1Y-999 */
}
#endif /* USE_LZO1Y */
/*
* Step 7: check if compressible
*/
if (best_len >= in_len)
{
printf("This file contains incompressible data.\n");
return 0;
}
/*
* Step 8: compress data again using the best compressor found
*/
if (best_compress == 1)
r = lzo1x_999_compress(in,in_len,out,&out_len,wrkmem);
else if (best_compress == 2)
r = lzo1y_999_compress(in,in_len,out,&out_len,wrkmem);
else
r = -100;
assert(r == LZO_E_OK);
assert(out_len == best_len);
/*
* Step 9: optimize compressed data (compressed data is in `out' buffer)
*/
#if 1
/* Optimization does not require any data in the buffer that will
* hold the uncompressed data. To prove this, we clear the buffer.
*/
lzo_memset(in,0,in_len);
#endif
orig_len = in_len;
if (best_compress == 1)
r = lzo1x_optimize(out,out_len,in,&orig_len,NULL);
else if (best_compress == 2)
r = lzo1y_optimize(out,out_len,in,&orig_len,NULL);
else
r = -100;
if (r != LZO_E_OK || orig_len != in_len)
{
/* this should NEVER happen */
printf("internal error - optimization failed: %d\n", r);
exit(1);
}
/*
* Step 10: compute a checksum of the compressed data
*/
compressed_checksum = lzo_adler32(0,NULL,0);
compressed_checksum = lzo_adler32(compressed_checksum,out,out_len);
/*
* Step 11: write compressed data to a file
*/
printf("%s: %s: %ld -> %ld, checksum 0x%08lx 0x%08lx\n",
progname, in_name, (long) in_len, (long) out_len,
(long) uncompressed_checksum, (long) compressed_checksum);
if (out_name && out_name[0])
{
printf("%s: writing to file %s\n", progname, out_name);
f = fopen(out_name,"wb");
if (f == NULL)
{
printf("%s: cannot open output file %s\n", progname, out_name);
exit(1);
}
if (lzo_fwrite(f,out,out_len) != out_len || fclose(f) != 0)
{
printf("%s: write error !!\n", progname);
exit(1);
}
}
/*
* Step 12: verify decompression
*/
#ifdef PARANOID
orig_len = in_len;
if (best_compress == 1)
r = lzo1x_decompress(out,out_len,in,&orig_len,NULL);
else if (best_compress == 2)
r = lzo1y_decompress(out,out_len,in,&orig_len,NULL);
else
r = -100;
if (r != LZO_E_OK || orig_len != in_len)
{
/* this should NEVER happen */
printf("internal error - decompression failed: %d\n", r);
exit(1);
}
if (uncompressed_checksum != lzo_adler32(lzo_adler32(0,NULL,0),in,in_len))
{
/* this should NEVER happen */
printf("internal error - decompression data error\n");
exit(1);
}
/* Now you could also verify decompression under similar conditions as in
* your application, e.g. overlapping assembler decompression etc.
*/
#endif
lzo_free(in);
lzo_free(out);
lzo_free(wrkmem);
return 0;
}
int main(int argc, char* argv[]) {
if(argc<2 || !strcmp(argv[1], "--help")) {
fprintf(stderr, "Usage:\n"
" fsfs-debug print-index block_size blockgroup_size file.idx [bgstart [bgcount]]\n"
" fsfs-debug comp-stats blockgroup_size file.idx\n"
" fsfs-debug decompress-block file.dat offset compressed_size > output\n"
" fsfs-debug decompress-block2 < input > output\n"
" fsfs-debug compress-block < input > output\n"
" fsfs-debug compress-block2 < input > output\n"
" fsfs-debug calculate-hash < input\n"
" fsfs-debug get-length dir name\n"
" fsfs-debug read-one-block dir name blocknum\n"
);
return 1;
}
if(!strcmp(argv[1], "print-index")) {
int bgsize=1020;
int block_size=4096;
long long int bgstart = 0;
long long int bgcount = -1;
assert(argc>=5 && argc<=7);
sscanf(argv[2], "%d", &block_size);
sscanf(argv[3], "%d", &bgsize);
const char* idxfile = argv[4];
if(argc>=6) sscanf(argv[5], "%lld", &bgstart);
if(argc>=7) sscanf(argv[6], "%lld", &bgcount);
int bglen = bgsize * 2 + 8;
FILE* idx = stdin;
if(strcmp(argv[2], "-")) idx = fopen(idxfile, "rb");
assert(idx!=NULL);
signed short int q;
long long int baseoffset;
if(bgstart!=0) {
int ret = fseek(idx, bgstart*bglen, SEEK_SET);
if(ret) { perror("fseek"); return 2;}
}
while(bgcount!=0) {
int ret = fread(&baseoffset, 1, 8, idx);
if(ret==0)break;
if(ret!=8) {
printf("Trimmed index file\n");
return 2;
}
baseoffset = be64toh(baseoffset);
printf("Block group %lld, base offset: 0x%016llX\n", bgstart, baseoffset);
int i;
int accum = 0;
for(i=0; i<bgsize; ++i) {
int ret = fread(&q, 1, 2, idx);
if(ret!=2) {
printf("Trimmed index file\n");
return 2;
}
q = be16toh(q);
printf("block %lld: ", i + bgstart*bgsize);
if(q==-0x8000) {
printf("zero\n");
} else
if(q==-0x7FFF) {
printf("uncompressed (%d bytes) at 0x%016llX\n", block_size, baseoffset+accum);
accum+=block_size;
} else
if(q==0) {
printf("unallocated\n");
} else
if(q>0 && q<0x4444) {
printf("compressed (%d bytes) at 0x%016llX\n", q, baseoffset+accum);
accum+=q;
} else
if (q<0 && q >= -64) {
printf("reference to %d's dependency\n", (-q)-1);
} else {
printf("probably invalid (%04X)\n", q);
}
}
fflush(stdout);
--bgcount;
++bgstart;
}
return 0;
} else
if(!strcmp(argv[1], "comp-stats")) {
int bgsize=1020;
assert(argc==4);
sscanf(argv[2], "%d", &bgsize);
const char* idxfile = argv[3];
FILE* idx = stdin;
if(strcmp(argv[2], "-")) idx = fopen(idxfile, "rb");
assert(idx!=NULL);
long long int baseoffset;
signed short int q;
unsigned long long int *stats = (unsigned long long int*) malloc(8*32768);
unsigned long long int zeroes = 0;
unsigned long long int invals = 0;
unsigned long long int refs[64];
unsigned long long int total = 0;
unsigned long long int uncompressibles = 0;
memset(&refs, 0, sizeof(refs));
memset(stats, 0, 8*32768);
int i;
int trailing_zero_counter=0;
for(;;) {
int ret = fread(&baseoffset, 1, 8, idx);
if(ret!=8)break;
baseoffset = be64toh(baseoffset);
for(i=0; i<bgsize; ++i) {
int ret = fread(&q, 1, 2, idx);
if(ret!=2) return 2;
q = be16toh(q);
if(q>0) {
++stats[q];
}else
if(q==-0x7FFF) {
++uncompressibles;
}else
if(q==-0x8000) {
++zeroes;
}else
if(q==0) {
++trailing_zero_counter;
}else
if(q<0 && q>=-64) {
++refs[(-q)-1];
}else{
++invals;
}
++total;
}
}
total-=trailing_zero_counter;
long long int running = 0;
printf("total: %lld (100%%) ; 0%% \n", total);
running+=zeroes;
if(zeroes>0)printf("zero: %lld (%g%%) ; %g%%\n", zeroes,
100.0*zeroes/total, 100.0*running/total);
for(i=0; i<64; ++i) {
running+=refs[i];
if(refs[i]>0)printf("refs[%d]: %lld (%g%%) ; %g%%\n", i, refs[i],
100.0*refs[i]/total, 100.0*running/total);
}
for(i=0; i<32768; ++i) {
running+=stats[i];
if(stats[i]>0)printf("compressed[%d]: %lld (%g%%) ; %g%%\n", i, stats[i],
100.0*stats[i]/total, 100.0*running/total);
}
running+=uncompressibles;
if(uncompressibles>0)printf("uncompressible: %lld (%g%%) ; %g%%\n", uncompressibles,
100.0*uncompressibles/total, 100.0*running/total);
running+=invals;
if(invals>0)printf("invalid: %lld (%g%%) ; %g%%\n", invals,
100.0*invals/total, 100.0*running/total);
free(stats);
return 0;
} else
if(!strcmp(argv[1], "decompress-block")) {
assert(argc==5);
const char* datfile = argv[2];
long long int offset = 0;
int size;
sscanf(argv[3], "%lld", &offset);
sscanf(argv[4], "%d", &size);
FILE* dat = fopen(datfile, "rb");
int ret = fseek(dat, offset, SEEK_SET);
assert(ret==0);
assert(size<65536);
unsigned char chunk[65536];
unsigned char chunk2[65536+2048];
ret = fread(&chunk, 1, size, dat);
assert(ret==size);
fclose(dat);
lzo_uint len = 65536+2048;
lzo1x_decompress_safe(chunk, ret, chunk2, &len, NULL);
fwrite(chunk2, 1, len, stdout);
return 0;
} else
if(!strcmp(argv[1], "decompress-block2")) {
unsigned char chunk[65536];
unsigned char chunk2[65536+2048];
int ret = fread(&chunk, 1, 65536, stdin);
lzo_uint len = 65536+2048;
lzo1x_decompress_safe(chunk, ret, chunk2, &len, NULL);
fwrite(chunk2, 1, len, stdout);
return 0;
} else
if(!strcmp(argv[1], "compress-block")) {
unsigned char chunk[65536];
unsigned char chunk2[65536+2048];
int ret = fread(&chunk, 1, 65536, stdin);
char tmp[LZO1X_1_MEM_COMPRESS];
lzo_uint len = 65536+2048;
lzo1x_1_compress(chunk, ret, chunk2, &len, &tmp);
fwrite(chunk2, 1, len, stdout);
return 0;
} else
if(!strcmp(argv[1], "compress-block2")) {
unsigned char chunk[65536];
unsigned char chunk2[65536+2048];
int ret = fread(&chunk, 1, 65536, stdin);
char tmp[LZO1X_999_MEM_COMPRESS];
lzo_uint len = 65536+2048;
lzo1x_999_compress(chunk, ret, chunk2, &len, &tmp);
fwrite(chunk2, 1, len, stdout);
return 0;
} else
if(!strcmp(argv[1], "calculate-hash")) {
unsigned char chunk[65536];
int ret = fread(&chunk, 1, 65536, stdin);
unsigned char c = phash(chunk, ret);
printf("%02x\n", c);
return 0;
} else
if(!strcmp(argv[1], "get-length")) {
assert(argc==4);
const char* dirname = argv[2];
const char* basename = argv[3];
int block_len = storage__get_block_size2(dirname, basename);
long long int number_of_blocks = storage__get_number_of_blocks2(dirname, basename);
printf("%lld\n", number_of_blocks*block_len);
return 0;
} else
if(!strcmp(argv[1], "read-one-block")) {
assert(argc==5);
const char* dirname = argv[2];
const char* basename = argv[3];
long long int blocknum;
sscanf(argv[4], "%lld", &blocknum);
struct storage__file* f = storage__open(dirname, basename);
int block_len = storage__get_block_size(f);
unsigned char buf[65536];
storage__read_block(f, buf, blocknum);
fwrite(buf, 1, block_len, stdout);
return 0;
} else {
fprintf(stderr, "Unknown command %s\n", argv[1]);
return 1;
}
}
int64_t lzbench_lzo_compress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t level, size_t workmem, size_t)
{
lzo_uint lzo_complen = 0;
int res;
switch (level)
{
default:
case 1:
res = lzo1b_1_compress((uint8_t*)inbuf, insize, (uint8_t*)outbuf, &lzo_complen, (void*)workmem);
break;
case 9:
res = lzo1b_9_compress((uint8_t*)inbuf, insize, (uint8_t*)outbuf, &lzo_complen, (void*)workmem);
break;
case 99:
res = lzo1b_99_compress((uint8_t*)inbuf, insize, (uint8_t*)outbuf, &lzo_complen, (void*)workmem);
break;
case 999:
res = lzo1b_999_compress((uint8_t*)inbuf, insize, (uint8_t*)outbuf, &lzo_complen, (void*)workmem);
break;
case 1001:
res = lzo1c_1_compress((uint8_t*)inbuf, insize, (uint8_t*)outbuf, &lzo_complen, (void*)workmem);
break;
case 1009:
res = lzo1c_9_compress((uint8_t*)inbuf, insize, (uint8_t*)outbuf, &lzo_complen, (void*)workmem);
break;
case 1099:
res = lzo1c_99_compress((uint8_t*)inbuf, insize, (uint8_t*)outbuf, &lzo_complen, (void*)workmem);
break;
case 1999:
res = lzo1c_999_compress((uint8_t*)inbuf, insize, (uint8_t*)outbuf, &lzo_complen, (void*)workmem);
break;
case 2001:
res = lzo1f_1_compress((uint8_t*)inbuf, insize, (uint8_t*)outbuf, &lzo_complen, (void*)workmem);
break;
case 2999:
res = lzo1f_999_compress((uint8_t*)inbuf, insize, (uint8_t*)outbuf, &lzo_complen, (void*)workmem);
break;
case 3001:
res = lzo1x_1_compress((uint8_t*)inbuf, insize, (uint8_t*)outbuf, &lzo_complen, (void*)workmem);
break;
case 3999:
res = lzo1x_999_compress((uint8_t*)inbuf, insize, (uint8_t*)outbuf, &lzo_complen, (void*)workmem);
break;
case 4001:
res = lzo1y_1_compress((uint8_t*)inbuf, insize, (uint8_t*)outbuf, &lzo_complen, (void*)workmem);
break;
case 4999:
res = lzo1y_999_compress((uint8_t*)inbuf, insize, (uint8_t*)outbuf, &lzo_complen, (void*)workmem);
break;
case 5999:
res = lzo1z_999_compress((uint8_t*)inbuf, insize, (uint8_t*)outbuf, &lzo_complen, (void*)workmem);
break;
case 6999:
res = lzo2a_999_compress((uint8_t*)inbuf, insize, (uint8_t*)outbuf, &lzo_complen, (void*)workmem);
break;
}
if (res != LZO_E_OK) return 0;
return lzo_complen;
}
bool xboxIMGCompression::Compress( CFile *input, CFile *output )
{
// Make sure we have LZO.
this->InitializeLZO();
if ( !this->isUsingLZO )
{
// We cannot continue if LZO failed to initialize.
return false;
}
// TODO: maybe add write-count verification?
// Write the magic.
output->WriteUInt( 0x67A3A1CE );
// Remember this position in the output file.
fsOffsetNumber_t genericHeaderOffset = output->TellNative();
// Write a dummy generic header.
compressionHeader mainHeader;
mainHeader.blockSize = 0; // we will fill this out later.
mainHeader.checksum = 0; // TODO: how to calculate this field?
output->WriteStruct( mainHeader );
// Prepare the LZO compression environment.
bool lzoSuccess = false;
size_t lzoWorkBufferSize = LZO1X_999_MEM_COMPRESS;
void *lzoCompressionWorkMemory = malloc( lzoWorkBufferSize );
// Allocate block buffers.
simpleWorkBuffer uncompressedFileData;
uncompressedFileData.MinimumSize( this->compressionMaximumBlockSize );
// We use the decompression buffer as compression buffer.
simpleWorkBuffer compressionBuffer = this->decompressBuffer;
bool hasCompressionBufferChanged = false;
// Make sure we have got something in the compression buffer.
// Since there is no safe compression, we must use the stuff that Oberhummer uses...
// His library is bad. We cannot ensure that our stuff does not crash. :/
uint32 requiredCompressionBufferSize = uncompressedFileData.GetSize() + uncompressedFileData.GetSize() / 16 + 64 + 3;
bool hasInitializedCompressBuffer = compressionBuffer.MinimumSize( requiredCompressionBufferSize );
if ( hasInitializedCompressBuffer )
{
hasCompressionBufferChanged = true;
}
if ( lzoCompressionWorkMemory && compressionBuffer.IsReady() && uncompressedFileData.IsReady() )
{
// Do the stuff.
bool compressionSuccess = true;
unsigned long rawChecksum = 0;
checksumCallback_t _checksumCallback = this->_checksumCallback;
if ( _checksumCallback != NULL )
{
// Start with the root checksum.
rawChecksum = _checksumCallback( 0, NULL, 0 );
}
size_t streamSize = 0;
// Process the blocks.
while ( true )
{
// If we have reached the end of the stream, quit.
if ( input->IsEOF() )
{
// Safe exit.
break;
}
// Read from the file stream.
void *uncompressedDataBuffer = uncompressedFileData.GetPointer();
size_t compressionDataSize = input->Read( uncompressedDataBuffer, 1, uncompressedFileData.GetSize() );
// If we could not read anything, we kinda failed.
if ( compressionDataSize == 0 )
{
compressionSuccess = false;
break;
}
// Calculate the checksum of the raw data.
if ( _checksumCallback != NULL )
{
rawChecksum = _checksumCallback( rawChecksum, uncompressedDataBuffer, compressionDataSize );
}
repeatCompression:
// Perform the compression.
void *compressedDataBuffer = compressionBuffer.GetPointer();
lzo_uint realCompressedSize = compressionBuffer.GetSize();
int lzoerr = lzo1x_999_compress(
(const unsigned char*)uncompressedDataBuffer, compressionDataSize,
(unsigned char*)compressedDataBuffer, &realCompressedSize,
lzoCompressionWorkMemory
);
// Process some valid errors.
if ( lzoerr == LZO_E_OUTPUT_OVERRUN )
{
// Increase buffer size.
compressionBuffer.Grow( realCompressedSize );
// Remember to update the decompressBuffer field.
hasCompressionBufferChanged = true;
// Repeat compression.
goto repeatCompression;
}
// Now if we get an error, we are screwed.
if ( lzoerr != LZO_E_OK )
{
compressionSuccess = false;
break;
}
// Write the block into the output stream.
perBlockHeader blockHeader;
blockHeader.compressedSize = realCompressedSize;
blockHeader.uncompressedSize = realCompressedSize; // ???
blockHeader.unk = 4; // ???
output->WriteStruct( blockHeader );
// Now write the compressed data.
output->Write( compressedDataBuffer, 1, realCompressedSize );
// Increase the actual stream size.
streamSize += sizeof( blockHeader ) + realCompressedSize;
}
if ( compressionSuccess )
{
// Update the generic header.
mainHeader.blockSize = streamSize;
mainHeader.checksum = rawChecksum;
// If we succeeded in compressing the file, we succeeded in life :)
lzoSuccess = true;
}
}
// Clean up.
if ( lzoCompressionWorkMemory )
{
free( lzoCompressionWorkMemory );
}
// Update the decompressBuffer.
if ( hasCompressionBufferChanged )
{
this->decompressBuffer = compressionBuffer;
}
// Release to not free the concurrent buffer.
compressionBuffer.Release();
if ( lzoSuccess )
{
// Update the main header.
fsOffsetNumber_t endOffset = output->TellNative();
output->SeekNative( genericHeaderOffset, SEEK_SET );
// Write the header.
output->WriteStruct( mainHeader );
// Seek back to where we left off after compressing.
output->SeekNative( endOffset, SEEK_SET );
}
return lzoSuccess;
}