void lzma_test() { #if defined(__FreeBSD__) && !defined(__APPLE__) _malloc_options = "XARV"; #endif //char *fn = "/usr/share/dict/web2"; const char *fn = "/etc/motd"; printf("starting up\n"); FILE *f = fopen(fn,"r"); if(!f) err(1,"%s",fn); struct stat st; if(fstat(fileno(f),&st)) err(1,"stat"); /* Allocate memory */ size_t buflen = st.st_size; printf("size=%qd\n",(long long)buflen); unsigned char *buf = (unsigned char *)malloc(buflen); if(buf==0) errx(1,"malloc"); if(fread(buf,1,st.st_size,f)!=(size_t)st.st_size) err(1,"read"); /* Allocate memory for the compressed buffer */ size_t cbufsize = (int)(buflen*1.05); size_t cbuf_actual=0; unsigned char *cbuf = (unsigned char *)malloc(cbufsize); #ifdef USE_LZMA lzma_compress(cbuf,&cbufsize,buf,st.st_size,9); #endif printf("cbuf_actual=%d\n",(int)cbuf_actual); /* Now try to decompress */ size_t outbuf_size = buflen*2; unsigned char *outbuf = (unsigned char *)malloc(outbuf_size); #ifdef USE_LZMA lzma_uncompress(outbuf,&outbuf_size,cbuf,cbufsize); #endif printf("cbuf[0]=%d\n",cbuf[0]); if(memcmp(buf,outbuf,outbuf_size)==0){ printf("Decompression works!\n"); } }
int main() { FILE* f; f = fopen("test.in","rb"); fseek(f,0,SEEK_END); unsigned inlen = ftell(f); fseek(f,0,SEEK_SET); void* in = VirtualAlloc(NULL,inlen,MEM_COMMIT,PAGE_READWRITE); void* out = VirtualAlloc(NULL,inlen,MEM_COMMIT,PAGE_READWRITE); fread(in,1,inlen,f); fclose(f); unsigned logdictsize,dictsize; for (logdictsize=0,dictsize=1;dictsize<inlen && logdictsize<=28;logdictsize++,dictsize<<=1) ; lzma_set_dict_size(logdictsize); void* work = VirtualAlloc(NULL,dictsize*19/2+0x509000,MEM_COMMIT,PAGE_READWRITE); unsigned outlen = lzma_compress(in,out,inlen,work); printf("%d -> %d\n",inlen,outlen); f = fopen("test.out","wb"); fwrite(out,1,outlen,f); fclose(f); return 0; }
/* Write a actual data segment to the disk and sign if necessary. */ int af_update_page(AFFILE *af,int64_t pagenum,unsigned char *data,int datalen) { char segname_buf[32]; snprintf(segname_buf,sizeof(segname_buf),AF_PAGE,pagenum); // determine segment name #ifdef USE_AFFSIGS /* Write out the signature if we have a private key */ if(af->crypto && af->crypto->sign_privkey){ af_sign_seg3(af,segname_buf,0,data,datalen,AF_SIGNATURE_MODE1); } #endif #ifdef HAVE_MD5 /* Write out MD5 if requested */ if(af->write_md5){ unsigned char md5_buf[16]; char md5name_buf[32]; MD5(data,datalen,md5_buf); snprintf(md5name_buf,sizeof(md5name_buf),AF_PAGE_MD5,pagenum); af_update_segf(af,md5name_buf,0,md5_buf,sizeof(md5_buf),AF_SIGFLAG_NOSIG); // ignore failure } #endif #ifdef HAVE_SHA1 /* Write out SHA1 if requested */ if(af->write_sha1){ unsigned char sha1_buf[20]; char sha1name_buf[32]; SHA1(data,datalen,sha1_buf); snprintf(sha1name_buf,sizeof(sha1name_buf),AF_PAGE_SHA1,pagenum); af_update_segf(af,sha1name_buf,0,sha1_buf,sizeof(sha1_buf),AF_SIGFLAG_NOSIG); // ignore failure } #endif /* Write out SHA256 if requested and if SHA256 is available */ if(af->write_sha256){ unsigned char sha256_buf[32]; if(af_SHA256(data,datalen,sha256_buf)==0){ char sha256name_buf[32]; snprintf(sha256name_buf,sizeof(sha256name_buf),AF_PAGE_SHA256,pagenum); af_update_segf(af,sha256name_buf,0,sha256_buf,sizeof(sha256_buf),AF_SIGFLAG_NOSIG); // ignore failure } } /* Check for bypass */ if(af->v->write){ int r = (*af->v->write)(af,data,af->image_pagesize * pagenum,datalen); if(r!=datalen) return -1; return 0; } struct affcallback_info acbi; int ret = 0; uint64_t starting_pages_written = af->pages_written; /* Setup the callback structure */ memset(&acbi,0,sizeof(acbi)); acbi.info_version = 1; acbi.af = af->parent ? af->parent : af; acbi.pagenum = pagenum; acbi.bytes_to_write = datalen; size_t destLen = af->image_pagesize; // it could be this big. /* Compress and write the data, if we are allowed to compress */ if(af->compression_type != AF_COMPRESSION_ALG_NONE){ unsigned char *cdata = (unsigned char *)malloc(destLen); // compressed data unsigned long *ldata = (unsigned long *)cdata; // allows me to reference as a buffer of unsigned longs if(cdata!=0){ // If data could be allocated int cres = -1; // compression results unsigned int flag = 0; // flag for data segment int dont_compress = 0; /* Try zero compression first; it's the best algorithm we have */ if(is_buffer_zero(data,datalen)){ acbi.compression_alg = AF_PAGE_COMP_ALG_ZERO; acbi.compression_level = AF_COMPRESSION_MAX; if(af->w_callback) { acbi.phase = 1; (*af->w_callback)(&acbi); } *ldata = htonl(datalen); // store the data length destLen = 4; // 4 bytes flag = AF_PAGE_COMPRESSED | AF_PAGE_COMP_ALG_ZERO | AF_PAGE_COMP_MAX; cres = 0; acbi.compressed = 1; // it was compressed if(af->w_callback) {acbi.phase = 2;(*af->w_callback)(&acbi);} } #ifdef USE_LZMA if(cres!=0 && af->compression_type==AF_COMPRESSION_ALG_LZMA){ // try to compress with LZMA acbi.compression_alg = AF_PAGE_COMP_ALG_LZMA; acbi.compression_level = 7; // right now, this is the level we use if(af->w_callback) { acbi.phase = 1; (*af->w_callback)(&acbi); } cres = lzma_compress(cdata,&destLen,data,datalen,9); #if 0 switch(cres){ case 0:break; // OKAY case 1: (*af->error_reporter)("LZMA: Unspecified Error\n");break; case 2: (*af->error_reporter)("LZMA: Memory Allocating Error\n");break; case 3: (*af->error_reporter)("LZMA: Output buffer OVERFLOW\n"); break; default: (*af->error_reporter)("LZMA: Unknown error %d\n",cres);break; } #endif if(cres==0){ flag = AF_PAGE_COMPRESSED | AF_PAGE_COMP_ALG_LZMA; acbi.compressed = 1; if(af->w_callback) {acbi.phase = 2;(*af->w_callback)(&acbi);} } else { /* Don't bother reporting LZMA errors; we just won't compress */ dont_compress = 1; if(af->w_callback) {acbi.phase = 2;(*af->w_callback)(&acbi);} } } #endif if(cres!=0 && af->compression_type==AF_COMPRESSION_ALG_ZLIB && dont_compress==0){ // try to compress with zlib acbi.compression_alg = AF_PAGE_COMP_ALG_ZLIB; // only one that we support acbi.compression_level = af->compression_level; if(af->w_callback) { acbi.phase = 1; (*af->w_callback)(&acbi); } cres = compress2((Bytef *)cdata, (uLongf *)&destLen, (Bytef *)data,datalen, af->compression_level); if(cres==0){ flag = AF_PAGE_COMPRESSED | AF_PAGE_COMP_ALG_ZLIB; if(af->compression_level == AF_COMPRESSION_MAX){ flag |= AF_PAGE_COMP_MAX; // useful to know it can't be better } } acbi.compressed = 1; // it was compressed (or not compressed) if(af->w_callback) {acbi.phase = 2;(*af->w_callback)(&acbi);} } if(cres==0 && destLen < af->image_pagesize){ /* Prepare to write out the compressed segment with compression */ if(af->w_callback) {acbi.phase = 3;(*af->w_callback)(&acbi);} ret = af_update_segf(af,segname_buf,flag,cdata,destLen,AF_SIGFLAG_NOSIG); acbi.bytes_written = destLen; if(af->w_callback) {acbi.phase = 4;(*af->w_callback)(&acbi);} if(ret==0){ af->pages_written++; af->pages_compressed++; } } free(cdata); cdata = 0; } } /* If a compressed segment was not written, write it uncompressed */ if(af->pages_written == starting_pages_written){ if(af->w_callback) {acbi.phase = 3;(*af->w_callback)(&acbi);} ret = af_update_segf(af,segname_buf,0,data,datalen,AF_SIGFLAG_NOSIG); acbi.bytes_written = datalen; if(af->w_callback) {acbi.phase = 4;(*af->w_callback)(&acbi);} if(ret==0){ acbi.bytes_written = datalen; // because that is how much we wrote af->pages_written++; } } return ret; }
void *_tc_recencode(const void *ptr, int size, int *sp, void *op){ return lzma_compress(ptr, size, sp); }
int adapt_compress(void *src, uint64_t srclen, void *dst, uint64_t *dstlen, int level, uchar_t chdr, int btype, void *data) { struct adapt_data *adat = (struct adapt_data *)(data); uchar_t *src1 = (uchar_t *)src; int rv = 0, bsc_type = 0; if (btype == TYPE_UNKNOWN) { uint64_t i, tot8b, tag1, tag2, tag3; double tagcnt, pct_tag; uchar_t cur_byte, prev_byte; /* * Count number of 8-bit binary bytes and XML tags in source. */ tot8b = 0; tag1 = 0; tag2 = 0; tag3 = 0; prev_byte = cur_byte = 0; for (i = 0; i < srclen; i++) { cur_byte = src1[i]; tot8b += (cur_byte & 0x80); // This way for possible auto-vectorization tag1 += (cur_byte == '<'); tag2 += (cur_byte == '>'); tag3 += ((prev_byte == '<') & (cur_byte == '/')); tag3 += ((prev_byte == '/') & (cur_byte == '>')); if (cur_byte != ' ') prev_byte = cur_byte; } tot8b /= 0x80; tagcnt = tag1 + tag2 + tag3; pct_tag = tagcnt / (double)srclen; if (adat->adapt_mode == 2 && tot8b > FORTY_PCT(srclen)) { btype = TYPE_BINARY; } else if (adat->adapt_mode == 1 && tot8b > FIFTY_PCT(srclen)) { btype = TYPE_BINARY; } else { btype = TYPE_TEXT; if (tag1 > tag2 - 4 && tag1 < tag2 + 4 && tag3 > (double)tag1 * 0.40 && tagcnt > (double)srclen * 0.001) btype |= TYPE_MARKUP; } } /* * Use PPMd if some percentage of source is 7-bit textual bytes, otherwise * use Bzip2 or LZMA. For totally incompressible data we always use LZ4. There * is no point trying to compress such data, like Jpegs. However some archive headers * and zero paddings can exist which LZ4 can easily take care of very fast. */ #ifdef ENABLE_PC_LIBBSC bsc_type = is_bsc_type(btype); #endif if (is_incompressible(btype)) { rv = lz4_compress(src, srclen, dst, dstlen, level, chdr, btype, adat->lz4_data); if (rv < 0) return (rv); rv = ADAPT_COMPRESS_LZ4; lz4_count++; } else if (adat->adapt_mode == 2 && PC_TYPE(btype) == TYPE_BINARY && !bsc_type) { rv = lzma_compress(src, srclen, dst, dstlen, level, chdr, btype, adat->lzma_data); if (rv < 0) return (rv); rv = ADAPT_COMPRESS_LZMA; lzma_count++; } else if (adat->adapt_mode == 1 && PC_TYPE(btype) == TYPE_BINARY && !bsc_type) { rv = bzip2_compress(src, srclen, dst, dstlen, level, chdr, btype, NULL); if (rv < 0) return (rv); rv = ADAPT_COMPRESS_BZIP2; bzip2_count++; } else { #ifdef ENABLE_PC_LIBBSC if (adat->bsc_data && bsc_type) { rv = libbsc_compress(src, srclen, dst, dstlen, level, chdr, btype, adat->bsc_data); if (rv < 0) return (rv); rv = ADAPT_COMPRESS_BSC; bsc_count++; } else { #endif rv = ppmd_alloc(adat->ppmd_data); if (rv < 0) return (rv); rv = ppmd_compress(src, srclen, dst, dstlen, level, chdr, btype, adat->ppmd_data); ppmd_free(adat->ppmd_data); if (rv < 0) return (rv); rv = ADAPT_COMPRESS_PPMD; ppmd_count++; #ifdef ENABLE_PC_LIBBSC } #endif } return (rv); }