int codegen_jschema(struct_data *defs, int ndefs,
char *prefix, char *suffix, FILE *out) {
int i;
int n;
struct_data *filtered_defs[ndefs];
for (i = n = 0; i < ndefs; i++) {
if (is_prefix(defs[i].info.name,prefix) && is_suffix(defs[i].info.name,suffix)) {
filtered_defs[n] = &defs[i];
n++;
}
}
fprintf(out,"[\n");
for (i = 0; i < n; i++) {
fprintf(out,"{");
fprintf(out,"\"name\" : \"%s\",",filtered_defs[i]->info.name);
codegen_jschema_internal(filtered_defs[i],defs,out);
fprintf(out,"}");
if (i != (n - 1)) {
fprintf(out,",");
}
fprintf(out,"\n");
}
fprintf(out,"]\n");
return 0;
}
/*
* Remove .p2p suffix and convert to lowercase.
*/
int query_sanitize( char buf[], size_t buflen, const char query[] ) {
size_t len;
size_t i;
len = strlen( query );
/* Remove .p2p suffix */
if( is_suffix( query, gconf->query_tld ) ) {
len -= strlen( gconf->query_tld );
}
/* Buffer too small */
if( (len+1) >= buflen ) {
return 1;
}
/* Convert to lower case */
for( i = 0; i < len; ++i ) {
buf[i] = tolower( (unsigned char) query[i] );
}
buf[len] = '\0';
return 0;
}
my_string intern_filename(my_string to, const char *from)
{
#ifndef VMS
{
uint length;
char buff[FN_REFLEN];
if (from == to)
{ /* Dirname may destroy from */
strmov(buff,from);
from=buff;
}
length=dirname_part(to,from); /* Copy dirname & fix chars */
(void) strcat(to,from+length);
return (to);
}
#else /* VMS */
/* change 'dev:[lib]xxx' to 'dev:lib/xxx' */
/* change 'dev:xxx' to 'dev:xxx' */
/* change 'dev:x/y/[.lib]' to 'dev:x/y/lib/ */
/* change '[.lib]' to './lib/' */
/* change '[x.y]' or '[x.][y]' or '[x][.y]' to '/x/y/' */
/* change '[000000.x] or [x.000000]' to '/x/' */
int par_length,root_length;
my_string pos,from_pos,to_pos,end_pos;
char buff[FN_REFLEN];
convert_dirname(buff,from,NullS); /* change '<>' to '[]' */
from_pos=buff;
if ((pos=strrchr(from_pos,FN_DEVCHAR))) /* Skipp device part */
{
pos++;
to_pos=strnmov(to,from_pos,(size_s) (pos-from_pos));
from_pos=pos;
}
else
to_pos=to;
root_length=strlen(FN_C_ROOT_DIR);
if ((pos = strchr(from_pos,FN_C_BEFORE_DIR)) &&
(end_pos = strrchr(pos+1,FN_C_AFTER_DIR)))
{
to_pos=strnmov(to_pos,from_pos,(size_s) (pos-from_pos));
/* Copy all between ':' and '[' */
from_pos=pos+1;
if (strinstr(from_pos,FN_C_ROOT_DIR) == 1 &&
(from_pos[root_length] == FN_C_DIR_SEP ||
from_pos[root_length] == FN_C_AFTER_DIR))
{
from_pos+=root_length+1;
}
else if (*from_pos == FN_C_DIR_SEP)
*(to_pos++) = FN_CURLIB; /* Set ./ first */
*(to_pos++) = FN_LIBCHAR;
par_length=strlen(FN_C_PARENT_DIR);
pos=to_pos;
for (; from_pos <= end_pos ; from_pos++)
{
switch (*from_pos) {
case FN_C_DIR_SEP:
case FN_C_AFTER_DIR:
if (pos != to_pos)
{
if ((int) (to_pos-pos) == root_length &&
is_suffix(pos,FN_C_ROOT_DIR))
to_pos=pos; /* remove root-pos */
else
{
*(to_pos++)=FN_LIBCHAR; /* Find lib */
pos=to_pos;
}
}
break;
case FN_C_BEFORE_DIR:
break;
case '-': /* *(FN_C_PARENT_DIR): */
if (to_pos[-1] == FN_LIBCHAR &&
strncmp(from_pos,FN_C_PARENT_DIR,par_length) == 0)
{ /* Change '-' to '..' */
to_pos=strmov(to_pos,FN_PARENTDIR);
*(to_pos++)=FN_LIBCHAR;
pos=to_pos;
from_pos+=par_length-1;
break;
}
/* Fall through */
default:
*(to_pos++)= *from_pos;
break;
}
}
}
(void) strmov(to_pos,from_pos);
return (to);
#endif /* VMS */
} /* intern_filename */
void
_bfd_elf_strtab_finalize (struct elf_strtab_hash *tab)
{
struct elf_strtab_hash_entry **array, **a, *e;
bfd_size_type size, amt;
/* GCC 2.91.66 (egcs-1.1.2) on i386 miscompiles this function when i is
a 64-bit bfd_size_type: a 64-bit target or --enable-64-bit-bfd.
Besides, indexing with a long long wouldn't give anything but extra
cycles. */
size_t i;
/* Sort the strings by suffix and length. */
amt = tab->size * sizeof (struct elf_strtab_hash_entry *);
array = bfd_malloc (amt);
if (array == NULL)
goto alloc_failure;
for (i = 1, a = array; i < tab->size; ++i)
{
e = tab->array[i];
if (e->refcount)
{
*a++ = e;
/* Adjust the length to not include the zero terminator. */
e->len -= 1;
}
else
e->len = 0;
}
size = a - array;
if (size != 0)
{
qsort (array, size, sizeof (struct elf_strtab_hash_entry *), strrevcmp);
/* Loop over the sorted array and merge suffixes. Start from the
end because we want eg.
s1 -> "d"
s2 -> "bcd"
s3 -> "abcd"
to end up as
s3 -> "abcd"
s2 _____^
s1 _______^
ie. we don't want s1 pointing into the old s2. */
e = *--a;
e->len += 1;
while (--a >= array)
{
struct elf_strtab_hash_entry *cmp = *a;
cmp->len += 1;
if (is_suffix (e, cmp))
{
cmp->u.suffix = e;
cmp->len = -cmp->len;
}
else
e = cmp;
}
}
alloc_failure:
if (array)
free (array);
/* Assign positions to the strings we want to keep. */
size = 1;
for (i = 1; i < tab->size; ++i)
{
e = tab->array[i];
if (e->refcount && e->len > 0)
{
e->u.index = size;
size += e->len;
}
}
tab->sec_size = size;
/* Adjust the rest. */
for (i = 1; i < tab->size; ++i)
{
e = tab->array[i];
if (e->refcount && e->len < 0)
e->u.index = e->u.suffix->u.index + (e->u.suffix->len + e->len);
}
}
void
_bfd_elf_strtab_finalize (struct elf_strtab_hash *tab)
{
struct elf_strtab_hash_entry **array, **a, *e;
bfd_size_type amt, sec_size;
size_t size, i;
/* Sort the strings by suffix and length. */
amt = tab->size;
amt *= sizeof (struct elf_strtab_hash_entry *);
array = (struct elf_strtab_hash_entry **) bfd_malloc (amt);
if (array == NULL)
goto alloc_failure;
for (i = 1, a = array; i < tab->size; ++i)
{
e = tab->array[i];
if (e->refcount)
{
*a++ = e;
/* Adjust the length to not include the zero terminator. */
e->len -= 1;
}
else
e->len = 0;
}
size = a - array;
if (size != 0)
{
qsort (array, size, sizeof (struct elf_strtab_hash_entry *), strrevcmp);
/* Loop over the sorted array and merge suffixes. Start from the
end because we want eg.
s1 -> "d"
s2 -> "bcd"
s3 -> "abcd"
to end up as
s3 -> "abcd"
s2 _____^
s1 _______^
ie. we don't want s1 pointing into the old s2. */
e = *--a;
e->len += 1;
while (--a >= array)
{
struct elf_strtab_hash_entry *cmp = *a;
cmp->len += 1;
if (is_suffix (e, cmp))
{
cmp->u.suffix = e;
cmp->len = -cmp->len;
}
else
e = cmp;
}
}
alloc_failure:
if (array)
free (array);
/* Assign positions to the strings we want to keep. */
sec_size = 1;
for (i = 1; i < tab->size; ++i)
{
e = tab->array[i];
if (e->refcount && e->len > 0)
{
e->u.index = sec_size;
sec_size += e->len;
}
}
tab->sec_size = sec_size;
/* Adjust the rest. */
for (i = 1; i < tab->size; ++i)
{
e = tab->array[i];
if (e->refcount && e->len < 0)
e->u.index = e->u.suffix->u.index + (e->u.suffix->len + e->len);
}
}
int bam12auxmerge(::libmaus2::util::ArgInfo const & arginfo)
{
if ( isatty(STDIN_FILENO) )
{
::libmaus2::exception::LibMausException se;
se.getStream() << "Refusing to read binary data from terminal, please redirect standard input to pipe or file." << std::endl;
se.finish();
throw se;
}
if ( isatty(STDOUT_FILENO) )
{
::libmaus2::exception::LibMausException se;
se.getStream() << "Refusing write binary data to terminal, please redirect standard output to pipe or file." << std::endl;
se.finish();
throw se;
}
std::string const prefilename = arginfo.getRestArg<std::string>(0);
libmaus2::bambam::BamDecoder bampredec(prefilename);
int const level = libmaus2::bambam::BamBlockWriterBaseFactory::checkCompressionLevel(arginfo.getValue<int>("level",getDefaultLevel()));
int const verbose = arginfo.getValue<int>("verbose",getDefaultVerbose());
int const ranksplit = arginfo.getValue<int>("ranksplit",getDefaultRankSplit());
int const rankstrip = arginfo.getValue<int>("rankstrip",getDefaultRankSplit());
int const clipreinsert = arginfo.getValue<int>("clipreinsert",getDefaultClipReinsert());
int const zztoname = arginfo.getValue<int>("zztoname",getDefaultZZToName());
int const sanity = arginfo.getValue<int>("sanity",getDefaultSanity());
uint64_t const mod = arginfo.getValue<int>("mod",getDefaultMod());
uint64_t const bmod = libmaus2::math::nextTwoPow(mod);
uint64_t const bmask = bmod-1;
libmaus2::autoarray::AutoArray<char> Aread;
::libmaus2::bambam::BamDecoder bamdec(std::cin,false);
::libmaus2::bambam::BamHeader const & header = bamdec.getHeader();
::libmaus2::bambam::BamHeader const & preheader = bampredec.getHeader();
std::string const headertext(header.text);
std::string const preheadertext(libmaus2::bambam::HeaderLine::removeSequenceLines(preheader.text));
libmaus2::bambam::ProgramHeaderLineSet headerlines(headertext);
libmaus2::bambam::ProgramHeaderLineSet preheaderlines(preheadertext);
std::vector<libmaus2::bambam::HeaderLine> allheaderlines = libmaus2::bambam::HeaderLine::extractLines(headertext);
std::string const lastid = preheaderlines.getLastIdInChain();
std::stack < std::pair<uint64_t,std::string> > pgtodo;
for ( uint64_t i = 0; i < headerlines.roots.size(); ++i )
pgtodo.push(std::pair<uint64_t,std::string>(headerlines.roots[i],lastid));
std::string upheadtext = preheadertext;
while ( pgtodo.size() )
{
uint64_t const hid = pgtodo.top().first;
std::string const PP = pgtodo.top().second;
pgtodo.pop();
libmaus2::bambam::HeaderLine const & line = headerlines.lines[hid];
// ID, PP, PN, CL, VN
std::string ID = (line.M.find("ID") != line.M.end()) ? line.M.find("ID")->second : "";
std::string const PN = (line.M.find("PN") != line.M.end()) ? line.M.find("PN")->second : "";
std::string const CL = (line.M.find("CL") != line.M.end()) ? line.M.find("CL")->second : "";
std::string const VN = (line.M.find("VN") != line.M.end()) ? line.M.find("VN")->second : "";
upheadtext = ::libmaus2::bambam::ProgramHeaderLineSet::addProgramLineRef(
upheadtext,
ID,
PN,
CL,
PP,
VN
);
if ( headerlines.edges.find(hid) != headerlines.edges.end() )
{
std::vector<uint64_t> const & children = headerlines.edges.find(hid)->second;
for ( uint64_t j = 0; j < children.size(); ++j )
pgtodo.push(std::pair<uint64_t,std::string>(children[j],ID));
}
}
/* copy SQ lines */
std::ostringstream sqconcstr;
sqconcstr << upheadtext;
for ( uint64_t i = 0; i < allheaderlines.size(); ++i )
if ( allheaderlines[i].type == "SQ" )
sqconcstr << allheaderlines[i].line << "\n";
upheadtext = sqconcstr.str();
::libmaus2::bambam::BamHeader uphead(upheadtext);
uphead.changeSortOrder("unknown");
/*
* start index/md5 callbacks
*/
std::string const tmpfilenamebase = arginfo.getValue<std::string>("tmpfile",arginfo.getDefaultTmpFileName());
std::string const tmpfileindex = tmpfilenamebase + "_index";
::libmaus2::util::TempFileRemovalContainer::addTempFile(tmpfileindex);
std::string md5filename;
std::string indexfilename;
std::vector< ::libmaus2::lz::BgzfDeflateOutputCallback * > cbs;
::libmaus2::lz::BgzfDeflateOutputCallbackMD5::unique_ptr_type Pmd5cb;
if ( arginfo.getValue<unsigned int>("md5",getDefaultMD5()) )
{
if ( arginfo.hasArg("md5filename") && arginfo.getUnparsedValue("md5filename","") != "" )
md5filename = arginfo.getUnparsedValue("md5filename","");
else
std::cerr << "[V] no filename for md5 given, not creating hash" << std::endl;
if ( md5filename.size() )
{
::libmaus2::lz::BgzfDeflateOutputCallbackMD5::unique_ptr_type Tmd5cb(new ::libmaus2::lz::BgzfDeflateOutputCallbackMD5);
Pmd5cb = UNIQUE_PTR_MOVE(Tmd5cb);
cbs.push_back(Pmd5cb.get());
}
}
libmaus2::bambam::BgzfDeflateOutputCallbackBamIndex::unique_ptr_type Pindex;
if ( arginfo.getValue<unsigned int>("index",getDefaultIndex()) )
{
if ( arginfo.hasArg("indexfilename") && arginfo.getUnparsedValue("indexfilename","") != "" )
indexfilename = arginfo.getUnparsedValue("indexfilename","");
else
std::cerr << "[V] no filename for index given, not creating index" << std::endl;
if ( indexfilename.size() )
{
libmaus2::bambam::BgzfDeflateOutputCallbackBamIndex::unique_ptr_type Tindex(new libmaus2::bambam::BgzfDeflateOutputCallbackBamIndex(tmpfileindex));
Pindex = UNIQUE_PTR_MOVE(Tindex);
cbs.push_back(Pindex.get());
}
}
std::vector< ::libmaus2::lz::BgzfDeflateOutputCallback * > * Pcbs = 0;
if ( cbs.size() )
Pcbs = &cbs;
/*
* end md5/index callbacks
*/
::libmaus2::bambam::BamWriter::unique_ptr_type writer(new ::libmaus2::bambam::BamWriter(std::cout,uphead,level,Pcbs));
::libmaus2::bambam::BamAlignment & algn = bamdec.getAlignment();
::libmaus2::bambam::BamAlignment & prealgn = bampredec.getAlignment();
int64_t curid = -1;
libmaus2::autoarray::AutoArray< std::pair<uint8_t,uint8_t> > auxpre;
libmaus2::autoarray::AutoArray< std::pair<uint8_t,uint8_t> > auxnew;
libmaus2::bambam::BamAuxFilterVector auxfilter;
// helpers for clipReinsert
libmaus2::autoarray::AutoArray < std::pair<uint8_t,uint8_t> > auxtags;
libmaus2::autoarray::AutoArray<libmaus2::bambam::cigar_operation> cigop;
std::stack < libmaus2::bambam::cigar_operation > hardstack;
libmaus2::bambam::BamAlignment::D_array_type Tcigar;
libmaus2::bambam::BamAuxFilterVector bafv;
libmaus2::bambam::BamAuxFilterVector auxfilterout;
auxfilterout.set('q','s');
auxfilterout.set('q','q');
// helpers for zztoname
libmaus2::bambam::BamAuxFilterVector zzbafv;
zzbafv.set('z','z');
// tag filters for secondary/supplementary reads
libmaus2::bambam::BamAuxFilterVector auxfiltersec;
auxfiltersec.set('q','s');
auxfiltersec.set('q','q');
auxfiltersec.set('a','s');
auxfiltersec.set('a','h');
auxfiltersec.set('a','a');
auxfiltersec.set('a','f');
auxfiltersec.set('a','r');
auxfiltersec.set('a','3');
// loop over aligned BAM file
while ( bamdec.readAlignment() )
{
if ( ranksplit )
split12(algn);
// extract rank
char const * name = algn.getName();
char const * u1 = name;
bool ok = true;
uint64_t rank = 0;
while ( *u1 && *u1 != '_' )
{
rank *= 10;
rank += (*u1-'0');
ok = ok && isdigit(*u1);
++u1;
}
// unable to find rank? write out as is and continue
if ( ! ok )
{
algn.serialise(writer->getStream());
continue;
}
// loop over unaligned BAM file
while ( curid != static_cast<int64_t>(rank) )
{
bool const a_ok = bampredec.readAlignment();
if ( ! a_ok )
{
libmaus2::exception::LibMausException se;
se.getStream() << "Found unexpected EOF on file " << prefilename << std::endl;
se.finish();
throw se;
}
assert ( a_ok );
++curid;
if ( verbose && (! (curid & bmask)) )
std::cerr << "[V] " << (curid / bmod) << std::endl;
}
if ( verbose > 1 )
std::cerr << "Merging:\n" << algn.formatAlignment(header) << "\n" << prealgn.formatAlignment(preheader) << std::endl;
uint64_t pretagnum = prealgn.enumerateAuxTags(auxpre);
uint64_t newtagnum = algn.enumerateAuxTags(auxnew);
// do some sanity checking
if ( sanity )
{
// first do a name check
char const * prename = prealgn.getName();
u1++; // put on the first letter of readname
if ( verbose > 1 )
std::cerr << "Sanity: comparing " << name << " and " << prename << std::endl;
if ( !is_suffix(prename, u1) ) // names do not match
{
libmaus2::exception::LibMausException se;
se.getStream() << "Sanity check failed on read names, found " << name << " and " << prename << std::endl;
se.finish();
throw se;
}
// now the names match so try the flags
if ( !(algn.isPaired() == prealgn.isPaired() &&
algn.isRead1() == prealgn.isRead1() &&
algn.isRead2() == prealgn.isRead2()) )
{
libmaus2::exception::LibMausException se;
se.getStream() << "Sanity check failed on flags, " << std::endl
<< "Aligned " << name << " paired " << algn.isPaired() << " first " << algn.isRead1() << " last " << algn.isRead2() << std::endl
<< "Unaligned " << prename << " paired " << prealgn.isPaired() << " first " << prealgn.isRead1() << " last " << prealgn.isRead2() << std::endl;
se.finish();
throw se;
}
if ( verbose > 1 )
std::cerr << "Sanity check on flags: " << std::endl
<< "Aligned " << name << " paired " << algn.isPaired() << " first " << algn.isRead1() << " last " << algn.isRead2() << std::endl
<< "Unaligned " << prename << " paired " << prealgn.isPaired() << " first " << prealgn.isRead1() << " last " << prealgn.isRead2() << std::endl;
}
std::sort(auxpre.begin(),auxpre.begin()+pretagnum);
std::sort(auxnew.begin(),auxnew.begin()+newtagnum);
if ( verbose > 1 )
std::cerr << "pretagnum=" << pretagnum << " newtagnum=" << newtagnum << std::endl;
std::pair<uint8_t,uint8_t> * prec = auxpre.begin();
std::pair<uint8_t,uint8_t> * pree = prec + pretagnum;
std::pair<uint8_t,uint8_t> * preo = prec;
std::pair<uint8_t,uint8_t> * newc = auxnew.begin();
std::pair<uint8_t,uint8_t> * newe = newc + newtagnum;
std::pair<uint8_t,uint8_t> * newo = newc;
while ( prec != pree && newc != newe )
{
// pre which is not in new
if ( *prec < *newc )
{
*(preo++) = *(prec++);
}
// tag in both, drop pre
else if ( *prec == *newc )
{
*(newo++) = *(newc++);
prec++;
}
// new not in pre
else
{
*(newo++) = *(newc++);
}
}
while ( prec != pree )
*(preo++) = *(prec++);
while ( newc != newe )
*(newo++) = *(newc++);
pretagnum = preo-auxpre.begin();
newtagnum = newo-auxnew.begin();
for ( uint64_t i = 0; i < pretagnum; ++i )
auxfilter.set(auxpre[i].first,auxpre[i].second);
algn.copyAuxTags(prealgn, auxfilter);
for ( uint64_t i = 0; i < pretagnum; ++i )
auxfilter.clear(auxpre[i].first,auxpre[i].second);
if ( verbose > 1 )
{
std::cerr << "pretagnum=" << pretagnum << " newtagnum=" << newtagnum << std::endl;
std::cerr << "result: " << algn.formatAlignment(header) << std::endl;
}
if ( algn.isSecondary() || algn.isSupplementary() )
{
// adding adapter clip data to secondary/supplementary reads
// can lead to incorrect clip reinserts so remove these tags
algn.filterOutAux(auxfiltersec);
}
// copy QC fail flag from original file to aligner output
if ( prealgn.isQCFail() )
algn.putFlags( algn.getFlags() | libmaus2::bambam::BamFlagBase::LIBMAUS2_BAMBAM_FQCFAIL );
if ( rankstrip )
strip12(algn);
if ( clipreinsert )
clipReinsert(algn,auxtags,bafv,cigop,Tcigar,hardstack,auxfilterout);
if ( zztoname )
zzToRank(algn,zzbafv);
algn.serialise(writer->getStream());
}
writer.reset();
if ( Pmd5cb )
{
Pmd5cb->saveDigestAsFile(md5filename);
}
if ( Pindex )
{
Pindex->flush(std::string(indexfilename));
}
return EXIT_SUCCESS;
}
static void create_file_names(void)
{
register int len;
register char *defines_suffix;
register char *prefix;
prefix = NULL;
defines_suffix = DEFINES_SUFFIX;
/* compute the file_prefix from the user provided output_file_name */
if (output_file_name != 0)
{
defines_suffix = ".h";
if (!(prefix = is_suffix(output_file_name, ".cpp")))
prefix = is_suffix(output_file_name, "");
}
if (prefix != NULL)
{
len = prefix - output_file_name;
file_prefix = (char *)MALLOC(len + 1);
if (file_prefix == 0)
no_space();
strncpy(file_prefix, output_file_name, len)[len] = 0;
}
else
len = strlen(file_prefix);
/* if "-o filename" was not given */
if (output_file_name == 0)
{
oflag = 1;
CREATE_FILE_NAME(output_file_name, OUTPUT_SUFFIX);
}
if (rflag)
{
CREATE_FILE_NAME(code_file_name, CODE_SUFFIX);
}
else
code_file_name = output_file_name;
if (dflag)
{
CREATE_FILE_NAME(defines_file_name, defines_suffix);
}
if (vflag)
{
CREATE_FILE_NAME(verbose_file_name, VERBOSE_SUFFIX);
}
if (gflag)
{
CREATE_FILE_NAME(graph_file_name, GRAPH_SUFFIX);
}
if (prefix != NULL)
{
FREE(file_prefix);
}
create_temp_name(&text_file_name, output_file_name, ".byacc.text");
create_temp_name(&union_file_name, output_file_name, ".byacc.union");
create_temp_name(&action_file_name, output_file_name, ".byacc.action");
}