void
Compressor::compress(char* dst, unsigned long dst_len, char* src, unsigned long src_len)
{
m_compressed = 0;
m_processed = 0;
unsigned long rv = compressBlock(dst, dst_len, src, src_len);
m_compressed = rv;
m_compressed_total += rv;
m_processed = src_len;
m_processed_total += src_len;
}
/*
* Deals with getting a block ready to send to the compressor
*/
void prepCompressBlock(int threadNum){
if(nextCol[threadNum]->colNum>=numCols) return;//this column is not actually a valid column (out of range) so return
compressBlock(nextCol[threadNum]);//send the next block to the compressor
if(nextCol[threadNum]->status==LAST_BLOCK || nextCol[threadNum]->status==FIRST_LAST){//if this happened to be the last one
fclose(nextCol[threadNum]->colFile);//we can close the file
if(nextCol[threadNum]->next->colNum<numCols){//and if the next one is in range, open a file for it
nextCol[threadNum]->next->colFile = fopen(getCompressedFileName(nextCol[threadNum]->next->colNum),"wb");
}
}
else{//not the last file so keep writing to the same file
nextCol[threadNum]->next->colFile = nextCol[threadNum]->colFile;
}
struct blockSeg *temp = nextCol[threadNum];//temp pointer
nextCol[threadNum] = nextCol[threadNum]->next;//update the top of the queue
free(temp);//don't need the block we just compressed
}
// flush input buffer into output buffer
BgzfDeflateZStreamBaseFlushInfo flush(
BgzfDeflateInputBufferBase & in,
BgzfDeflateOutputBufferBase & out,
bool const fullflush
)
{
try
{
uint64_t const uncompressedsize = in.pc-in.pa;
uint64_t const payloadsize = compressBlock(in.pa,uncompressedsize,out.outbuf.begin());
fillHeaderFooter(in.pa,out.outbuf.begin(),payloadsize,uncompressedsize);
in.pc = in.pa;
return BgzfDeflateZStreamBaseFlushInfo(uncompressedsize,getBgzfHeaderSize()+getBgzfFooterSize()+payloadsize);
// return getBgzfHeaderSize()+getBgzfFooterSize()+payloadsize;
}
catch(...)
{
return flushBound(in,out,fullflush);
}
}
/*
* Compresses one column (colNum = col)
*/
void compressColumn(int col, int threadNum){
FILE *ptr = fopen(getReadingFileName(col),"rb");//open the uncompressed file
int read=blockWords;//will hold how many words have been successfully read
segs[threadNum]->colNum = col;//save colNumber of this block
segs[threadNum]->status=READ_FIRST;//we know it's the first one
segs[threadNum]->colFile = fopen(getCompressedFileName(col),"wb");//open the respective file for writing
while(read==blockWords){//while we still can read
read = fread(segs[threadNum]->toCompress, sizeof(word_32), blockWords, ptr);//transfer over all the words
segs[threadNum]->size = read;//and save how many words there are in there
compressBlock(segs[threadNum]);//compress it
segs[threadNum]->status = VALID;//not at the beginning anymore
if(read<blockWords){//if we reached the end of the file
fwrite(&(segs[threadNum]->curr), sizeof(word_32),1,segs[threadNum]->colFile);//write the last word
fclose(segs[threadNum]->colFile);//close the compressed file
break;//and leave
}
}
fclose(ptr);//close the uncompressed column data file
compressNext(&threadNum);//try to compress more
}
BgzfDeflateZStreamBaseFlushInfo flushBound(
BgzfDeflateInputBufferBase & in,
BgzfDeflateOutputBufferBase & out,
bool const fullflush
)
{
// full flush, compress block in two halves
if ( fullflush )
{
uint64_t const toflush = in.pc-in.pa;
uint64_t const flush0 = (toflush+1)/2;
uint64_t const flush1 = toflush-flush0;
/* compress first half of data */
uint64_t const payload0 = compressBlock(in.pa,flush0,out.outbuf.begin());
fillHeaderFooter(in.pa,out.outbuf.begin(),payload0,flush0);
/* compress second half of data */
setupHeader(out.outbuf.begin()+getBgzfHeaderSize()+payload0+getBgzfFooterSize());
uint64_t const payload1 = compressBlock(in.pa+flush0,flush1,out.outbuf.begin()+getBgzfHeaderSize()+payload0+getBgzfFooterSize());
fillHeaderFooter(in.pa+flush0,out.outbuf.begin()+getBgzfHeaderSize()+payload0+getBgzfFooterSize(),payload1,flush1);
assert ( 2*getBgzfHeaderSize()+2*getBgzfFooterSize()+payload0+payload1 <= out.outbuf.size() );
in.pc = in.pa;
return
BgzfDeflateZStreamBaseFlushInfo(
flush0,
getBgzfHeaderSize()+getBgzfFooterSize()+payload0,
flush1,
getBgzfHeaderSize()+getBgzfFooterSize()+payload1
);
}
else
{
unsigned int const toflush = std::min(static_cast<unsigned int>(in.pc-in.pa),deflbound);
unsigned int const unflushed = (in.pc-in.pa)-toflush;
/*
* write out compressed data
*/
uint64_t const payloadsize = compressBlock(in.pa,toflush,out.outbuf.begin());
fillHeaderFooter(in.pa,out.outbuf.begin(),payloadsize,toflush);
#if 0
/*
* copy rest of uncompressed data to front of buffer
*/
if ( unflushed )
memmove(in.pa,in.pc-unflushed,unflushed);
// set new output pointer
in.pc = in.pa + unflushed;
#endif
/* number number of bytes in output buffer */
// return getBgzfHeaderSize()+getBgzfFooterSize()+payloadsize;
return BgzfDeflateZStreamBaseFlushInfo(
toflush,
getBgzfHeaderSize()+getBgzfFooterSize()+payloadsize,
in.pa, // moveto
in.pc-unflushed, // movefrom
unflushed // movesize
);
}
}
