void Compressor::compress(char* dst, unsigned long dst_len, char* src, unsigned long src_len) { m_compressed = 0; m_processed = 0; unsigned long rv = compressBlock(dst, dst_len, src, src_len); m_compressed = rv; m_compressed_total += rv; m_processed = src_len; m_processed_total += src_len; }
/* * Deals with getting a block ready to send to the compressor */ void prepCompressBlock(int threadNum){ if(nextCol[threadNum]->colNum>=numCols) return;//this column is not actually a valid column (out of range) so return compressBlock(nextCol[threadNum]);//send the next block to the compressor if(nextCol[threadNum]->status==LAST_BLOCK || nextCol[threadNum]->status==FIRST_LAST){//if this happened to be the last one fclose(nextCol[threadNum]->colFile);//we can close the file if(nextCol[threadNum]->next->colNum<numCols){//and if the next one is in range, open a file for it nextCol[threadNum]->next->colFile = fopen(getCompressedFileName(nextCol[threadNum]->next->colNum),"wb"); } } else{//not the last file so keep writing to the same file nextCol[threadNum]->next->colFile = nextCol[threadNum]->colFile; } struct blockSeg *temp = nextCol[threadNum];//temp pointer nextCol[threadNum] = nextCol[threadNum]->next;//update the top of the queue free(temp);//don't need the block we just compressed }
// flush input buffer into output buffer BgzfDeflateZStreamBaseFlushInfo flush( BgzfDeflateInputBufferBase & in, BgzfDeflateOutputBufferBase & out, bool const fullflush ) { try { uint64_t const uncompressedsize = in.pc-in.pa; uint64_t const payloadsize = compressBlock(in.pa,uncompressedsize,out.outbuf.begin()); fillHeaderFooter(in.pa,out.outbuf.begin(),payloadsize,uncompressedsize); in.pc = in.pa; return BgzfDeflateZStreamBaseFlushInfo(uncompressedsize,getBgzfHeaderSize()+getBgzfFooterSize()+payloadsize); // return getBgzfHeaderSize()+getBgzfFooterSize()+payloadsize; } catch(...) { return flushBound(in,out,fullflush); } }
/* * Compresses one column (colNum = col) */ void compressColumn(int col, int threadNum){ FILE *ptr = fopen(getReadingFileName(col),"rb");//open the uncompressed file int read=blockWords;//will hold how many words have been successfully read segs[threadNum]->colNum = col;//save colNumber of this block segs[threadNum]->status=READ_FIRST;//we know it's the first one segs[threadNum]->colFile = fopen(getCompressedFileName(col),"wb");//open the respective file for writing while(read==blockWords){//while we still can read read = fread(segs[threadNum]->toCompress, sizeof(word_32), blockWords, ptr);//transfer over all the words segs[threadNum]->size = read;//and save how many words there are in there compressBlock(segs[threadNum]);//compress it segs[threadNum]->status = VALID;//not at the beginning anymore if(read<blockWords){//if we reached the end of the file fwrite(&(segs[threadNum]->curr), sizeof(word_32),1,segs[threadNum]->colFile);//write the last word fclose(segs[threadNum]->colFile);//close the compressed file break;//and leave } } fclose(ptr);//close the uncompressed column data file compressNext(&threadNum);//try to compress more }
BgzfDeflateZStreamBaseFlushInfo flushBound( BgzfDeflateInputBufferBase & in, BgzfDeflateOutputBufferBase & out, bool const fullflush ) { // full flush, compress block in two halves if ( fullflush ) { uint64_t const toflush = in.pc-in.pa; uint64_t const flush0 = (toflush+1)/2; uint64_t const flush1 = toflush-flush0; /* compress first half of data */ uint64_t const payload0 = compressBlock(in.pa,flush0,out.outbuf.begin()); fillHeaderFooter(in.pa,out.outbuf.begin(),payload0,flush0); /* compress second half of data */ setupHeader(out.outbuf.begin()+getBgzfHeaderSize()+payload0+getBgzfFooterSize()); uint64_t const payload1 = compressBlock(in.pa+flush0,flush1,out.outbuf.begin()+getBgzfHeaderSize()+payload0+getBgzfFooterSize()); fillHeaderFooter(in.pa+flush0,out.outbuf.begin()+getBgzfHeaderSize()+payload0+getBgzfFooterSize(),payload1,flush1); assert ( 2*getBgzfHeaderSize()+2*getBgzfFooterSize()+payload0+payload1 <= out.outbuf.size() ); in.pc = in.pa; return BgzfDeflateZStreamBaseFlushInfo( flush0, getBgzfHeaderSize()+getBgzfFooterSize()+payload0, flush1, getBgzfHeaderSize()+getBgzfFooterSize()+payload1 ); } else { unsigned int const toflush = std::min(static_cast<unsigned int>(in.pc-in.pa),deflbound); unsigned int const unflushed = (in.pc-in.pa)-toflush; /* * write out compressed data */ uint64_t const payloadsize = compressBlock(in.pa,toflush,out.outbuf.begin()); fillHeaderFooter(in.pa,out.outbuf.begin(),payloadsize,toflush); #if 0 /* * copy rest of uncompressed data to front of buffer */ if ( unflushed ) memmove(in.pa,in.pc-unflushed,unflushed); // set new output pointer in.pc = in.pa + unflushed; #endif /* number number of bytes in output buffer */ // return getBgzfHeaderSize()+getBgzfFooterSize()+payloadsize; return BgzfDeflateZStreamBaseFlushInfo( toflush, getBgzfHeaderSize()+getBgzfFooterSize()+payloadsize, in.pa, // moveto in.pc-unflushed, // movefrom unflushed // movesize ); } }