void lzma_test()
{
#if defined(__FreeBSD__) && !defined(__APPLE__)
_malloc_options = "XARV";
#endif
//char *fn = "/usr/share/dict/web2";
const char *fn = "/etc/motd";
printf("starting up\n");
FILE *f = fopen(fn,"r");
if(!f) err(1,"%s",fn);
struct stat st;
if(fstat(fileno(f),&st)) err(1,"stat");
/* Allocate memory */
size_t buflen = st.st_size;
printf("size=%qd\n",(long long)buflen);
unsigned char *buf = (unsigned char *)malloc(buflen);
if(buf==0) errx(1,"malloc");
if(fread(buf,1,st.st_size,f)!=(size_t)st.st_size) err(1,"read");
/* Allocate memory for the compressed buffer */
size_t cbufsize = (int)(buflen*1.05);
size_t cbuf_actual=0;
unsigned char *cbuf = (unsigned char *)malloc(cbufsize);
#ifdef USE_LZMA
lzma_compress(cbuf,&cbufsize,buf,st.st_size,9);
#endif
printf("cbuf_actual=%d\n",(int)cbuf_actual);
/* Now try to decompress */
size_t outbuf_size = buflen*2;
unsigned char *outbuf = (unsigned char *)malloc(outbuf_size);
#ifdef USE_LZMA
lzma_uncompress(outbuf,&outbuf_size,cbuf,cbufsize);
#endif
printf("cbuf[0]=%d\n",cbuf[0]);
if(memcmp(buf,outbuf,outbuf_size)==0){
printf("Decompression works!\n");
}
}
int main()
{
FILE* f;
f = fopen("test.in","rb");
fseek(f,0,SEEK_END);
unsigned inlen = ftell(f);
fseek(f,0,SEEK_SET);
void* in = VirtualAlloc(NULL,inlen,MEM_COMMIT,PAGE_READWRITE);
void* out = VirtualAlloc(NULL,inlen,MEM_COMMIT,PAGE_READWRITE);
fread(in,1,inlen,f);
fclose(f);
unsigned logdictsize,dictsize;
for (logdictsize=0,dictsize=1;dictsize<inlen && logdictsize<=28;logdictsize++,dictsize<<=1) ;
lzma_set_dict_size(logdictsize);
void* work = VirtualAlloc(NULL,dictsize*19/2+0x509000,MEM_COMMIT,PAGE_READWRITE);
unsigned outlen = lzma_compress(in,out,inlen,work);
printf("%d -> %d\n",inlen,outlen);
f = fopen("test.out","wb");
fwrite(out,1,outlen,f);
fclose(f);
return 0;
}
/* Write a actual data segment to the disk and sign if necessary. */
int af_update_page(AFFILE *af,int64_t pagenum,unsigned char *data,int datalen)
{
char segname_buf[32];
snprintf(segname_buf,sizeof(segname_buf),AF_PAGE,pagenum); // determine segment name
#ifdef USE_AFFSIGS
/* Write out the signature if we have a private key */
if(af->crypto && af->crypto->sign_privkey){
af_sign_seg3(af,segname_buf,0,data,datalen,AF_SIGNATURE_MODE1);
}
#endif
#ifdef HAVE_MD5
/* Write out MD5 if requested */
if(af->write_md5){
unsigned char md5_buf[16];
char md5name_buf[32];
MD5(data,datalen,md5_buf);
snprintf(md5name_buf,sizeof(md5name_buf),AF_PAGE_MD5,pagenum);
af_update_segf(af,md5name_buf,0,md5_buf,sizeof(md5_buf),AF_SIGFLAG_NOSIG); // ignore failure
}
#endif
#ifdef HAVE_SHA1
/* Write out SHA1 if requested */
if(af->write_sha1){
unsigned char sha1_buf[20];
char sha1name_buf[32];
SHA1(data,datalen,sha1_buf);
snprintf(sha1name_buf,sizeof(sha1name_buf),AF_PAGE_SHA1,pagenum);
af_update_segf(af,sha1name_buf,0,sha1_buf,sizeof(sha1_buf),AF_SIGFLAG_NOSIG); // ignore failure
}
#endif
/* Write out SHA256 if requested and if SHA256 is available */
if(af->write_sha256){
unsigned char sha256_buf[32];
if(af_SHA256(data,datalen,sha256_buf)==0){
char sha256name_buf[32];
snprintf(sha256name_buf,sizeof(sha256name_buf),AF_PAGE_SHA256,pagenum);
af_update_segf(af,sha256name_buf,0,sha256_buf,sizeof(sha256_buf),AF_SIGFLAG_NOSIG); // ignore failure
}
}
/* Check for bypass */
if(af->v->write){
int r = (*af->v->write)(af,data,af->image_pagesize * pagenum,datalen);
if(r!=datalen) return -1;
return 0;
}
struct affcallback_info acbi;
int ret = 0;
uint64_t starting_pages_written = af->pages_written;
/* Setup the callback structure */
memset(&acbi,0,sizeof(acbi));
acbi.info_version = 1;
acbi.af = af->parent ? af->parent : af;
acbi.pagenum = pagenum;
acbi.bytes_to_write = datalen;
size_t destLen = af->image_pagesize; // it could be this big.
/* Compress and write the data, if we are allowed to compress */
if(af->compression_type != AF_COMPRESSION_ALG_NONE){
unsigned char *cdata = (unsigned char *)malloc(destLen); // compressed data
unsigned long *ldata = (unsigned long *)cdata; // allows me to reference as a buffer of unsigned longs
if(cdata!=0){ // If data could be allocated
int cres = -1; // compression results
unsigned int flag = 0; // flag for data segment
int dont_compress = 0;
/* Try zero compression first; it's the best algorithm we have */
if(is_buffer_zero(data,datalen)){
acbi.compression_alg = AF_PAGE_COMP_ALG_ZERO;
acbi.compression_level = AF_COMPRESSION_MAX;
if(af->w_callback) { acbi.phase = 1; (*af->w_callback)(&acbi); }
*ldata = htonl(datalen); // store the data length
destLen = 4; // 4 bytes
flag = AF_PAGE_COMPRESSED | AF_PAGE_COMP_ALG_ZERO | AF_PAGE_COMP_MAX;
cres = 0;
acbi.compressed = 1; // it was compressed
if(af->w_callback) {acbi.phase = 2;(*af->w_callback)(&acbi);}
}
#ifdef USE_LZMA
if(cres!=0 && af->compression_type==AF_COMPRESSION_ALG_LZMA){ // try to compress with LZMA
acbi.compression_alg = AF_PAGE_COMP_ALG_LZMA;
acbi.compression_level = 7; // right now, this is the level we use
if(af->w_callback) { acbi.phase = 1; (*af->w_callback)(&acbi); }
cres = lzma_compress(cdata,&destLen,data,datalen,9);
#if 0
switch(cres){
case 0:break; // OKAY
case 1: (*af->error_reporter)("LZMA: Unspecified Error\n");break;
case 2: (*af->error_reporter)("LZMA: Memory Allocating Error\n");break;
case 3: (*af->error_reporter)("LZMA: Output buffer OVERFLOW\n"); break;
default: (*af->error_reporter)("LZMA: Unknown error %d\n",cres);break;
}
#endif
if(cres==0){
flag = AF_PAGE_COMPRESSED | AF_PAGE_COMP_ALG_LZMA;
acbi.compressed = 1;
if(af->w_callback) {acbi.phase = 2;(*af->w_callback)(&acbi);}
}
else {
/* Don't bother reporting LZMA errors; we just won't compress */
dont_compress = 1;
if(af->w_callback) {acbi.phase = 2;(*af->w_callback)(&acbi);}
}
}
#endif
if(cres!=0
&& af->compression_type==AF_COMPRESSION_ALG_ZLIB
&& dont_compress==0){ // try to compress with zlib
acbi.compression_alg = AF_PAGE_COMP_ALG_ZLIB; // only one that we support
acbi.compression_level = af->compression_level;
if(af->w_callback) { acbi.phase = 1; (*af->w_callback)(&acbi); }
cres = compress2((Bytef *)cdata, (uLongf *)&destLen,
(Bytef *)data,datalen, af->compression_level);
if(cres==0){
flag = AF_PAGE_COMPRESSED | AF_PAGE_COMP_ALG_ZLIB;
if(af->compression_level == AF_COMPRESSION_MAX){
flag |= AF_PAGE_COMP_MAX; // useful to know it can't be better
}
}
acbi.compressed = 1; // it was compressed (or not compressed)
if(af->w_callback) {acbi.phase = 2;(*af->w_callback)(&acbi);}
}
if(cres==0 && destLen < af->image_pagesize){
/* Prepare to write out the compressed segment with compression */
if(af->w_callback) {acbi.phase = 3;(*af->w_callback)(&acbi);}
ret = af_update_segf(af,segname_buf,flag,cdata,destLen,AF_SIGFLAG_NOSIG);
acbi.bytes_written = destLen;
if(af->w_callback) {acbi.phase = 4;(*af->w_callback)(&acbi);}
if(ret==0){
af->pages_written++;
af->pages_compressed++;
}
}
free(cdata);
cdata = 0;
}
}
/* If a compressed segment was not written, write it uncompressed */
if(af->pages_written == starting_pages_written){
if(af->w_callback) {acbi.phase = 3;(*af->w_callback)(&acbi);}
ret = af_update_segf(af,segname_buf,0,data,datalen,AF_SIGFLAG_NOSIG);
acbi.bytes_written = datalen;
if(af->w_callback) {acbi.phase = 4;(*af->w_callback)(&acbi);}
if(ret==0){
acbi.bytes_written = datalen; // because that is how much we wrote
af->pages_written++;
}
}
return ret;
}
void *_tc_recencode(const void *ptr, int size, int *sp, void *op){
return lzma_compress(ptr, size, sp);
}
int
adapt_compress(void *src, uint64_t srclen, void *dst,
uint64_t *dstlen, int level, uchar_t chdr, int btype, void *data)
{
struct adapt_data *adat = (struct adapt_data *)(data);
uchar_t *src1 = (uchar_t *)src;
int rv = 0, bsc_type = 0;
if (btype == TYPE_UNKNOWN) {
uint64_t i, tot8b, tag1, tag2, tag3;
double tagcnt, pct_tag;
uchar_t cur_byte, prev_byte;
/*
* Count number of 8-bit binary bytes and XML tags in source.
*/
tot8b = 0;
tag1 = 0;
tag2 = 0;
tag3 = 0;
prev_byte = cur_byte = 0;
for (i = 0; i < srclen; i++) {
cur_byte = src1[i];
tot8b += (cur_byte & 0x80); // This way for possible auto-vectorization
tag1 += (cur_byte == '<');
tag2 += (cur_byte == '>');
tag3 += ((prev_byte == '<') & (cur_byte == '/'));
tag3 += ((prev_byte == '/') & (cur_byte == '>'));
if (cur_byte != ' ')
prev_byte = cur_byte;
}
tot8b /= 0x80;
tagcnt = tag1 + tag2 + tag3;
pct_tag = tagcnt / (double)srclen;
if (adat->adapt_mode == 2 && tot8b > FORTY_PCT(srclen)) {
btype = TYPE_BINARY;
} else if (adat->adapt_mode == 1 && tot8b > FIFTY_PCT(srclen)) {
btype = TYPE_BINARY;
} else {
btype = TYPE_TEXT;
if (tag1 > tag2 - 4 && tag1 < tag2 + 4 && tag3 > (double)tag1 * 0.40 &&
tagcnt > (double)srclen * 0.001)
btype |= TYPE_MARKUP;
}
}
/*
* Use PPMd if some percentage of source is 7-bit textual bytes, otherwise
* use Bzip2 or LZMA. For totally incompressible data we always use LZ4. There
* is no point trying to compress such data, like Jpegs. However some archive headers
* and zero paddings can exist which LZ4 can easily take care of very fast.
*/
#ifdef ENABLE_PC_LIBBSC
bsc_type = is_bsc_type(btype);
#endif
if (is_incompressible(btype)) {
rv = lz4_compress(src, srclen, dst, dstlen, level, chdr, btype, adat->lz4_data);
if (rv < 0)
return (rv);
rv = ADAPT_COMPRESS_LZ4;
lz4_count++;
} else if (adat->adapt_mode == 2 && PC_TYPE(btype) == TYPE_BINARY && !bsc_type) {
rv = lzma_compress(src, srclen, dst, dstlen, level, chdr, btype, adat->lzma_data);
if (rv < 0)
return (rv);
rv = ADAPT_COMPRESS_LZMA;
lzma_count++;
} else if (adat->adapt_mode == 1 && PC_TYPE(btype) == TYPE_BINARY && !bsc_type) {
rv = bzip2_compress(src, srclen, dst, dstlen, level, chdr, btype, NULL);
if (rv < 0)
return (rv);
rv = ADAPT_COMPRESS_BZIP2;
bzip2_count++;
} else {
#ifdef ENABLE_PC_LIBBSC
if (adat->bsc_data && bsc_type) {
rv = libbsc_compress(src, srclen, dst, dstlen, level, chdr, btype, adat->bsc_data);
if (rv < 0)
return (rv);
rv = ADAPT_COMPRESS_BSC;
bsc_count++;
} else {
#endif
rv = ppmd_alloc(adat->ppmd_data);
if (rv < 0)
return (rv);
rv = ppmd_compress(src, srclen, dst, dstlen, level, chdr, btype, adat->ppmd_data);
ppmd_free(adat->ppmd_data);
if (rv < 0)
return (rv);
rv = ADAPT_COMPRESS_PPMD;
ppmd_count++;
#ifdef ENABLE_PC_LIBBSC
}
#endif
}
return (rv);
}
