Ejemplo n.º 1
0
			static void concatenate(std::vector<std::string> const & infilenames, std::string const & outfilename, bool const removeinput = false)
			{
				uint64_t const n = ::libmaus2::gamma::GammaRLDecoder::getLength(infilenames);
				unsigned int const albits = infilenames.size() ? ::libmaus2::gamma::GammaRLDecoder::getAlBits(infilenames[0]) : 0;
				
				::libmaus2::aio::OutputStreamInstance COS(outfilename);
				::libmaus2::aio::SynchronousGenericOutput<uint64_t> SGO(COS,64);
				SGO.put(n);
				SGO.put(albits);
				SGO.flush();
				uint64_t const headerlen = 2*sizeof(uint64_t);
				
				std::vector < ::libmaus2::huffman::IndexEntry > index;
				uint64_t ioff = headerlen;
				
				for ( uint64_t i = 0; i < infilenames.size(); ++i )
				{
					uint64_t const indexpos = ::libmaus2::huffman::IndexLoaderBase::getIndexPos(infilenames[i]);
					uint64_t const datalen = indexpos-headerlen;
					
					// copy data
					::libmaus2::aio::InputStreamInstance CIS(infilenames[i]);
					CIS.seekg(headerlen);
					::libmaus2::util::GetFileSize::copy(CIS,COS,datalen);
					
					// add entries to index
					::libmaus2::huffman::IndexLoaderSequential indexdata(infilenames[i]);
					::libmaus2::huffman::IndexEntry ij = indexdata.getNext();
					
					// ::libmaus2::huffman::IndexDecoderData indexdata(infilenames[i]);
					for ( uint64_t j = 0; j < indexdata.numentries; ++j )
					{
						::libmaus2::huffman::IndexEntry ij1 = indexdata.getNext();
						/*
						::libmaus2::huffman::IndexEntry const ij  = indexdata.readEntry(j);
						::libmaus2::huffman::IndexEntry const ij1 = indexdata.readEntry(j+1);						
						*/
						index.push_back(::libmaus2::huffman::IndexEntry((ij.pos - headerlen) + ioff, ij1.kcnt - ij.kcnt, ij1.vcnt - ij.vcnt));
						
						ij = ij1;
					}
					
					// update position pointer
					ioff += datalen;
					
					if ( removeinput )
						libmaus2::aio::FileRemoval::removeFile(infilenames[i]);
				}

				// write index
				::libmaus2::aio::SynchronousGenericOutput<uint8_t> SGO8(COS,64*1024);
				::libmaus2::aio::SynchronousGenericOutput<uint8_t>::iterator_type it(SGO8);
				::libmaus2::bitio::FastWriteBitWriterStream8Std FWBWS(it);
				writeIndex(index,FWBWS,ioff);

				FWBWS.flush();
				SGO8.flush();
				COS.flush();
			}
Ejemplo n.º 2
0
			GammaRLEncoder(std::string const & filename, unsigned int const ralbits, uint64_t const n, uint64_t const rblocksize, uint64_t const rbufsize = 64*1024)
			: 
			  blocksize(rblocksize),
			  COS(filename), SGO(COS,rbufsize), GE(SGO), 
			  A(blocksize), pa(A.begin()), pc(pa), pe(A.end()), 
			  cursym(0), curcnt(0), indexwritten(false), albits(ralbits)
			{
				SGO.put(n);
				SGO.put(albits);
			}
Ejemplo n.º 3
0
			void flush()
			{
				if ( curcnt )
				{
					*(pc++) = ptype(cursym,curcnt);
					implicitFlush();
					curcnt = 0;
				}

				SGO.flush();

				uint64_t const indexpos = SGO.getWrittenBytes();
				writeIndex(indexpos);				
			}
Ejemplo n.º 4
0
			void writeIndex(uint64_t const indexpos)
			{
				if ( ! indexwritten )
				{
					::libmaus::aio::SynchronousGenericOutput<uint8_t> SGO(COS,64*1024);
					::libmaus::aio::SynchronousGenericOutput<uint8_t>::iterator_type it(SGO);
					::libmaus::bitio::FastWriteBitWriterStream8Std FWBWS(it);
					writeIndex(index,FWBWS,indexpos);

					FWBWS.flush();
					SGO.flush();
					COS.flush();
					
					indexwritten = true;
				}
			}
Ejemplo n.º 5
0
			void implicitFlush()
			{
				uint64_t const bs = pc-pa;
				
				if ( bs )
				{
					uint64_t acc = 0;
					uint64_t const pos = SGO.getWrittenBytes();

					GE.encode(bs);
					for ( uint64_t i = 0; i < bs; ++i )
					{
						acc += pa[i].second;
						GE.encodeWord(pa[i].first,albits);
						GE.encode(pa[i].second);
					}
					GE.flush();
					
					::libmaus::huffman::IndexEntry const entry(pos,bs,acc);
					index.push_back(entry);
					
					pc = pa;
				}
			}