int bam12auxmerge(::libmaus2::util::ArgInfo const & arginfo) { if ( isatty(STDIN_FILENO) ) { ::libmaus2::exception::LibMausException se; se.getStream() << "Refusing to read binary data from terminal, please redirect standard input to pipe or file." << std::endl; se.finish(); throw se; } if ( isatty(STDOUT_FILENO) ) { ::libmaus2::exception::LibMausException se; se.getStream() << "Refusing write binary data to terminal, please redirect standard output to pipe or file." << std::endl; se.finish(); throw se; } std::string const prefilename = arginfo.getRestArg<std::string>(0); libmaus2::bambam::BamDecoder bampredec(prefilename); int const level = libmaus2::bambam::BamBlockWriterBaseFactory::checkCompressionLevel(arginfo.getValue<int>("level",getDefaultLevel())); int const verbose = arginfo.getValue<int>("verbose",getDefaultVerbose()); int const ranksplit = arginfo.getValue<int>("ranksplit",getDefaultRankSplit()); int const rankstrip = arginfo.getValue<int>("rankstrip",getDefaultRankSplit()); int const clipreinsert = arginfo.getValue<int>("clipreinsert",getDefaultClipReinsert()); int const zztoname = arginfo.getValue<int>("zztoname",getDefaultZZToName()); int const sanity = arginfo.getValue<int>("sanity",getDefaultSanity()); uint64_t const mod = arginfo.getValue<int>("mod",getDefaultMod()); uint64_t const bmod = libmaus2::math::nextTwoPow(mod); uint64_t const bmask = bmod-1; libmaus2::autoarray::AutoArray<char> Aread; ::libmaus2::bambam::BamDecoder bamdec(std::cin,false); ::libmaus2::bambam::BamHeader const & header = bamdec.getHeader(); ::libmaus2::bambam::BamHeader const & preheader = bampredec.getHeader(); std::string const headertext(header.text); std::string const preheadertext(libmaus2::bambam::HeaderLine::removeSequenceLines(preheader.text)); libmaus2::bambam::ProgramHeaderLineSet headerlines(headertext); libmaus2::bambam::ProgramHeaderLineSet preheaderlines(preheadertext); std::vector<libmaus2::bambam::HeaderLine> allheaderlines = libmaus2::bambam::HeaderLine::extractLines(headertext); std::string const lastid = preheaderlines.getLastIdInChain(); std::stack < std::pair<uint64_t,std::string> > pgtodo; for ( uint64_t i = 0; i < headerlines.roots.size(); ++i ) pgtodo.push(std::pair<uint64_t,std::string>(headerlines.roots[i],lastid)); std::string upheadtext = preheadertext; while ( pgtodo.size() ) { uint64_t const hid = pgtodo.top().first; std::string const PP = pgtodo.top().second; pgtodo.pop(); libmaus2::bambam::HeaderLine const & line = headerlines.lines[hid]; // ID, PP, PN, CL, VN std::string ID = (line.M.find("ID") != line.M.end()) ? line.M.find("ID")->second : ""; std::string const PN = (line.M.find("PN") != line.M.end()) ? line.M.find("PN")->second : ""; std::string const CL = (line.M.find("CL") != line.M.end()) ? line.M.find("CL")->second : ""; std::string const VN = (line.M.find("VN") != line.M.end()) ? line.M.find("VN")->second : ""; upheadtext = ::libmaus2::bambam::ProgramHeaderLineSet::addProgramLineRef( upheadtext, ID, PN, CL, PP, VN ); if ( headerlines.edges.find(hid) != headerlines.edges.end() ) { std::vector<uint64_t> const & children = headerlines.edges.find(hid)->second; for ( uint64_t j = 0; j < children.size(); ++j ) pgtodo.push(std::pair<uint64_t,std::string>(children[j],ID)); } } /* copy SQ lines */ std::ostringstream sqconcstr; sqconcstr << upheadtext; for ( uint64_t i = 0; i < allheaderlines.size(); ++i ) if ( allheaderlines[i].type == "SQ" ) sqconcstr << allheaderlines[i].line << "\n"; upheadtext = sqconcstr.str(); ::libmaus2::bambam::BamHeader uphead(upheadtext); uphead.changeSortOrder("unknown"); /* * start index/md5 callbacks */ std::string const tmpfilenamebase = arginfo.getValue<std::string>("tmpfile",arginfo.getDefaultTmpFileName()); std::string const tmpfileindex = tmpfilenamebase + "_index"; ::libmaus2::util::TempFileRemovalContainer::addTempFile(tmpfileindex); std::string md5filename; std::string indexfilename; std::vector< ::libmaus2::lz::BgzfDeflateOutputCallback * > cbs; ::libmaus2::lz::BgzfDeflateOutputCallbackMD5::unique_ptr_type Pmd5cb; if ( arginfo.getValue<unsigned int>("md5",getDefaultMD5()) ) { if ( arginfo.hasArg("md5filename") && arginfo.getUnparsedValue("md5filename","") != "" ) md5filename = arginfo.getUnparsedValue("md5filename",""); else std::cerr << "[V] no filename for md5 given, not creating hash" << std::endl; if ( md5filename.size() ) { ::libmaus2::lz::BgzfDeflateOutputCallbackMD5::unique_ptr_type Tmd5cb(new ::libmaus2::lz::BgzfDeflateOutputCallbackMD5); Pmd5cb = UNIQUE_PTR_MOVE(Tmd5cb); cbs.push_back(Pmd5cb.get()); } } libmaus2::bambam::BgzfDeflateOutputCallbackBamIndex::unique_ptr_type Pindex; if ( arginfo.getValue<unsigned int>("index",getDefaultIndex()) ) { if ( arginfo.hasArg("indexfilename") && arginfo.getUnparsedValue("indexfilename","") != "" ) indexfilename = arginfo.getUnparsedValue("indexfilename",""); else std::cerr << "[V] no filename for index given, not creating index" << std::endl; if ( indexfilename.size() ) { libmaus2::bambam::BgzfDeflateOutputCallbackBamIndex::unique_ptr_type Tindex(new libmaus2::bambam::BgzfDeflateOutputCallbackBamIndex(tmpfileindex)); Pindex = UNIQUE_PTR_MOVE(Tindex); cbs.push_back(Pindex.get()); } } std::vector< ::libmaus2::lz::BgzfDeflateOutputCallback * > * Pcbs = 0; if ( cbs.size() ) Pcbs = &cbs; /* * end md5/index callbacks */ ::libmaus2::bambam::BamWriter::unique_ptr_type writer(new ::libmaus2::bambam::BamWriter(std::cout,uphead,level,Pcbs)); ::libmaus2::bambam::BamAlignment & algn = bamdec.getAlignment(); ::libmaus2::bambam::BamAlignment & prealgn = bampredec.getAlignment(); int64_t curid = -1; libmaus2::autoarray::AutoArray< std::pair<uint8_t,uint8_t> > auxpre; libmaus2::autoarray::AutoArray< std::pair<uint8_t,uint8_t> > auxnew; libmaus2::bambam::BamAuxFilterVector auxfilter; // helpers for clipReinsert libmaus2::autoarray::AutoArray < std::pair<uint8_t,uint8_t> > auxtags; libmaus2::autoarray::AutoArray<libmaus2::bambam::cigar_operation> cigop; std::stack < libmaus2::bambam::cigar_operation > hardstack; libmaus2::bambam::BamAlignment::D_array_type Tcigar; libmaus2::bambam::BamAuxFilterVector bafv; libmaus2::bambam::BamAuxFilterVector auxfilterout; auxfilterout.set('q','s'); auxfilterout.set('q','q'); // helpers for zztoname libmaus2::bambam::BamAuxFilterVector zzbafv; zzbafv.set('z','z'); // tag filters for secondary/supplementary reads libmaus2::bambam::BamAuxFilterVector auxfiltersec; auxfiltersec.set('q','s'); auxfiltersec.set('q','q'); auxfiltersec.set('a','s'); auxfiltersec.set('a','h'); auxfiltersec.set('a','a'); auxfiltersec.set('a','f'); auxfiltersec.set('a','r'); auxfiltersec.set('a','3'); // loop over aligned BAM file while ( bamdec.readAlignment() ) { if ( ranksplit ) split12(algn); // extract rank char const * name = algn.getName(); char const * u1 = name; bool ok = true; uint64_t rank = 0; while ( *u1 && *u1 != '_' ) { rank *= 10; rank += (*u1-'0'); ok = ok && isdigit(*u1); ++u1; } // unable to find rank? write out as is and continue if ( ! ok ) { algn.serialise(writer->getStream()); continue; } // loop over unaligned BAM file while ( curid != static_cast<int64_t>(rank) ) { bool const a_ok = bampredec.readAlignment(); if ( ! a_ok ) { libmaus2::exception::LibMausException se; se.getStream() << "Found unexpected EOF on file " << prefilename << std::endl; se.finish(); throw se; } assert ( a_ok ); ++curid; if ( verbose && (! (curid & bmask)) ) std::cerr << "[V] " << (curid / bmod) << std::endl; } if ( verbose > 1 ) std::cerr << "Merging:\n" << algn.formatAlignment(header) << "\n" << prealgn.formatAlignment(preheader) << std::endl; uint64_t pretagnum = prealgn.enumerateAuxTags(auxpre); uint64_t newtagnum = algn.enumerateAuxTags(auxnew); // do some sanity checking if ( sanity ) { // first do a name check char const * prename = prealgn.getName(); u1++; // put on the first letter of readname if ( verbose > 1 ) std::cerr << "Sanity: comparing " << name << " and " << prename << std::endl; if ( !is_suffix(prename, u1) ) // names do not match { libmaus2::exception::LibMausException se; se.getStream() << "Sanity check failed on read names, found " << name << " and " << prename << std::endl; se.finish(); throw se; } // now the names match so try the flags if ( !(algn.isPaired() == prealgn.isPaired() && algn.isRead1() == prealgn.isRead1() && algn.isRead2() == prealgn.isRead2()) ) { libmaus2::exception::LibMausException se; se.getStream() << "Sanity check failed on flags, " << std::endl << "Aligned " << name << " paired " << algn.isPaired() << " first " << algn.isRead1() << " last " << algn.isRead2() << std::endl << "Unaligned " << prename << " paired " << prealgn.isPaired() << " first " << prealgn.isRead1() << " last " << prealgn.isRead2() << std::endl; se.finish(); throw se; } if ( verbose > 1 ) std::cerr << "Sanity check on flags: " << std::endl << "Aligned " << name << " paired " << algn.isPaired() << " first " << algn.isRead1() << " last " << algn.isRead2() << std::endl << "Unaligned " << prename << " paired " << prealgn.isPaired() << " first " << prealgn.isRead1() << " last " << prealgn.isRead2() << std::endl; } std::sort(auxpre.begin(),auxpre.begin()+pretagnum); std::sort(auxnew.begin(),auxnew.begin()+newtagnum); if ( verbose > 1 ) std::cerr << "pretagnum=" << pretagnum << " newtagnum=" << newtagnum << std::endl; std::pair<uint8_t,uint8_t> * prec = auxpre.begin(); std::pair<uint8_t,uint8_t> * pree = prec + pretagnum; std::pair<uint8_t,uint8_t> * preo = prec; std::pair<uint8_t,uint8_t> * newc = auxnew.begin(); std::pair<uint8_t,uint8_t> * newe = newc + newtagnum; std::pair<uint8_t,uint8_t> * newo = newc; while ( prec != pree && newc != newe ) { // pre which is not in new if ( *prec < *newc ) { *(preo++) = *(prec++); } // tag in both, drop pre else if ( *prec == *newc ) { *(newo++) = *(newc++); prec++; } // new not in pre else { *(newo++) = *(newc++); } } while ( prec != pree ) *(preo++) = *(prec++); while ( newc != newe ) *(newo++) = *(newc++); pretagnum = preo-auxpre.begin(); newtagnum = newo-auxnew.begin(); for ( uint64_t i = 0; i < pretagnum; ++i ) auxfilter.set(auxpre[i].first,auxpre[i].second); algn.copyAuxTags(prealgn, auxfilter); for ( uint64_t i = 0; i < pretagnum; ++i ) auxfilter.clear(auxpre[i].first,auxpre[i].second); if ( verbose > 1 ) { std::cerr << "pretagnum=" << pretagnum << " newtagnum=" << newtagnum << std::endl; std::cerr << "result: " << algn.formatAlignment(header) << std::endl; } if ( algn.isSecondary() || algn.isSupplementary() ) { // adding adapter clip data to secondary/supplementary reads // can lead to incorrect clip reinserts so remove these tags algn.filterOutAux(auxfiltersec); } // copy QC fail flag from original file to aligner output if ( prealgn.isQCFail() ) algn.putFlags( algn.getFlags() | libmaus2::bambam::BamFlagBase::LIBMAUS2_BAMBAM_FQCFAIL ); if ( rankstrip ) strip12(algn); if ( clipreinsert ) clipReinsert(algn,auxtags,bafv,cigop,Tcigar,hardstack,auxfilterout); if ( zztoname ) zzToRank(algn,zzbafv); algn.serialise(writer->getStream()); } writer.reset(); if ( Pmd5cb ) { Pmd5cb->saveDigestAsFile(md5filename); } if ( Pindex ) { Pindex->flush(std::string(indexfilename)); } return EXIT_SUCCESS; }
int bam12split(::libmaus::util::ArgInfo const & arginfo) { ::libmaus::util::TempFileRemovalContainer::setup(); if ( isatty(STDIN_FILENO) ) { ::libmaus::exception::LibMausException se; se.getStream() << "Refusing to read binary data from terminal, please redirect standard input to pipe or file." << std::endl; se.finish(); throw se; } if ( isatty(STDOUT_FILENO) ) { ::libmaus::exception::LibMausException se; se.getStream() << "Refusing write binary data to terminal, please redirect standard output to pipe or file." << std::endl; se.finish(); throw se; } if ( arginfo.hasArg("keep") && arginfo.hasArg("remove") ) { ::libmaus::exception::LibMausException se; se.getStream() << "The keep and remove keys are mutually exclusive." << std::endl; se.finish(); throw se; } int const level = libmaus::bambam::BamBlockWriterBaseFactory::checkCompressionLevel(arginfo.getValue<int>("level",getDefaultLevel())); int const verbose = arginfo.getValue<int>("verbose",getDefaultVerbose()); ::libmaus::bambam::BamDecoder dec(std::cin,false); ::libmaus::bambam::BamHeader const & header = dec.getHeader(); std::string const headertext(header.text); // add PG line to header std::string const upheadtext = ::libmaus::bambam::ProgramHeaderLineSet::addProgramLine( headertext, "bam12split", // ID "bam12split", // PN arginfo.commandline, // CL ::libmaus::bambam::ProgramHeaderLineSet(headertext).getLastIdInChain(), // PP std::string(PACKAGE_VERSION) // VN ); // construct new header libmaus::bambam::BamHeader uphead(upheadtext); uphead.changeSortOrder("unknown"); /* * start index/md5 callbacks */ std::string const tmpfilenamebase = arginfo.getValue<std::string>("tmpfile",arginfo.getDefaultTmpFileName()); std::string const tmpfileindex = tmpfilenamebase + "_index"; ::libmaus::util::TempFileRemovalContainer::addTempFile(tmpfileindex); std::string md5filename; std::string indexfilename; std::vector< ::libmaus::lz::BgzfDeflateOutputCallback * > cbs; ::libmaus::lz::BgzfDeflateOutputCallbackMD5::unique_ptr_type Pmd5cb; if ( arginfo.getValue<unsigned int>("md5",getDefaultMD5()) ) { if ( arginfo.hasArg("md5filename") && arginfo.getUnparsedValue("md5filename","") != "" ) md5filename = arginfo.getUnparsedValue("md5filename",""); else std::cerr << "[V] no filename for md5 given, not creating hash" << std::endl; if ( md5filename.size() ) { ::libmaus::lz::BgzfDeflateOutputCallbackMD5::unique_ptr_type Tmd5cb(new ::libmaus::lz::BgzfDeflateOutputCallbackMD5); Pmd5cb = UNIQUE_PTR_MOVE(Tmd5cb); cbs.push_back(Pmd5cb.get()); } } libmaus::bambam::BgzfDeflateOutputCallbackBamIndex::unique_ptr_type Pindex; if ( arginfo.getValue<unsigned int>("index",getDefaultIndex()) ) { if ( arginfo.hasArg("indexfilename") && arginfo.getUnparsedValue("indexfilename","") != "" ) indexfilename = arginfo.getUnparsedValue("indexfilename",""); else std::cerr << "[V] no filename for index given, not creating index" << std::endl; if ( indexfilename.size() ) { libmaus::bambam::BgzfDeflateOutputCallbackBamIndex::unique_ptr_type Tindex(new libmaus::bambam::BgzfDeflateOutputCallbackBamIndex(tmpfileindex)); Pindex = UNIQUE_PTR_MOVE(Tindex); cbs.push_back(Pindex.get()); } } std::vector< ::libmaus::lz::BgzfDeflateOutputCallback * > * Pcbs = 0; if ( cbs.size() ) Pcbs = &cbs; /* * end md5/index callbacks */ ::libmaus::bambam::BamWriter::unique_ptr_type writer(new ::libmaus::bambam::BamWriter(std::cout,uphead,level,Pcbs)); libmaus::bambam::BamAlignment & algn = dec.getAlignment(); uint64_t c = 0; while ( dec.readAlignment() ) { bool const ok = split12(algn); if ( ok ) algn.serialise(writer->getStream()); if ( verbose && (++c & (1024*1024-1)) == 0 ) std::cerr << "[V] " << c/(1024*1024) << std::endl; } writer.reset(); if ( Pmd5cb ) { Pmd5cb->saveDigestAsFile(md5filename); } if ( Pindex ) { Pindex->flush(std::string(indexfilename)); } return EXIT_SUCCESS; }