int main(int argc, char **argv)
{
AjPList idlist;
AjPList* fieldList = NULL;
AjBool systemsort;
AjBool cleanup;
ajint blastv = 0;
char dbtype = '\0';
ajuint maxindex;
ajuint maxidlen = 0;
ajuint maxlen;
AjPStr version = NULL;
AjPStr seqtype = NULL;
AjPFile elistfile = NULL;
AjPFile* alistfile = NULL;
AjPStr dbname = NULL;
AjPStr release = NULL;
AjPStr datestr = NULL;
AjPStr sortopt = NULL;
void **entryIds = NULL;
AjBool usesrc = AJTRUE;
AjPStr directory;
AjPStr indexdir;
AjPStr filename;
AjPStr exclude;
AjPStr curfilename = NULL;
AjPStr idformat = NULL;
EmbPEntry entry;
PBlastDb db = NULL;
ajuint idCount = 0;
ajuint idDone;
AjPList listTestFiles = NULL;
void ** testFiles = NULL;
ajuint nfiles;
ajuint ifile;
ajuint jfile;
ajuint filesize;
short recsize;
ajuint maxfilelen = 20;
char date[4] =
{
0,0,0,0
};
AjPStr tmpfname = NULL;
AjPStr* fields = NULL;
AjPFile entFile = NULL;
AjPStr* divfiles = NULL;
ajint* maxFieldLen = NULL;
ajuint ifield = 0;
ajuint nfields = 0;
AjPFile logfile = NULL;
ajuint* countField = NULL;
ajuint* fieldTot = NULL;
ajuint idCountFile = 0;
ajuint i = 0;
embInit("dbiblast", argc, argv);
idformat = ajStrNewC("NCBI");
fields = ajAcdGetList("fields");
directory = ajAcdGetDirectoryName("directory");
indexdir = ajAcdGetOutdirName("indexoutdir");
filename = ajAcdGetString("filenames");
exclude = ajAcdGetString("exclude");
dbname = ajAcdGetString("dbname");
release = ajAcdGetString("release");
datestr = ajAcdGetString("date");
systemsort = ajAcdGetBoolean("systemsort");
cleanup = ajAcdGetBoolean("cleanup");
sortopt = ajAcdGetString("sortoptions");
maxindex = ajAcdGetInt("maxindex");
version = ajAcdGetListSingle("blastversion");
seqtype = ajAcdGetListSingle("seqtype");
usesrc = ajAcdGetBoolean("sourcefile");
logfile = ajAcdGetOutfile("outfile");
while(fields[nfields]) /* array ends with a NULL */
nfields++;
if(nfields)
{
AJCNEW(maxFieldLen, nfields);
AJCNEW0(countField, nfields);
AJCNEW0(fieldTot, nfields);
for(ifield=0; ifield < nfields; ifield++)
maxFieldLen[ifield] = (ajint) maxindex * -1;
if(systemsort)
AJCNEW(alistfile, nfields);
else
{
AJCNEW(fieldList, nfields);
for(ifield=0; ifield < nfields; ifield++)
fieldList[ifield] = ajListNew();
}
}
if(ajStrMatchC(datestr, "00/00/00"))
ajFmtPrintS(&datestr, "%D", ajTimeRefTodayFmt("dbindex"));
ajStrRemoveWhite(&dbname); /* used for temp filenames */
embDbiDateSet(datestr, date);
idlist = ajListNew();
if(ajUtilGetBigendian())
readReverse = ajFalse;
else
readReverse = ajTrue;
ajStrToInt(version, &blastv);
dbtype = ajStrGetCharFirst(seqtype);
ajDebug("reading '%S/%S'\n", directory, filename);
ajDebug("writing '%S/'\n", indexdir);
listTestFiles = embDbiFileListExc(directory, filename, exclude);
ajListSort(listTestFiles, ajStrVcmp);
nfiles = ajListToarray(listTestFiles, &testFiles);
if(!nfiles)
ajDie("No input files in '%S' matched filename '%S'",
directory, filename);
embDbiLogHeader(logfile, dbname, release, datestr,
indexdir, maxindex);
embDbiLogFields(logfile, fields, nfields);
embDbiLogSource(logfile, directory, filename, exclude,
(AjPStr*) testFiles, nfiles);
embDbiLogCmdline(logfile);
AJCNEW0(divfiles, nfiles);
/*
** process each input file, one at a time
*/
jfile = 0;
for(ifile=0; ifile < nfiles; ifile++)
{
curfilename = (AjPStr) testFiles[ifile];
if(!dbiblast_blastopenlib(curfilename,
usesrc, blastv, dbtype, &db))
continue; /* could be the wrong file type with "*.*" */
ajDebug("processing filename '%S' ...\n", curfilename);
ajDebug("processing file '%S' ...\n", db->TFile->Name);
ajStrAssignS(&divfiles[jfile], db->TFile->Name);
ajFilenameTrimPath(&divfiles[jfile]);
if(ajStrGetLen(divfiles[jfile]) >= maxfilelen)
maxfilelen = ajStrGetLen(divfiles[jfile]) + 1;
if(systemsort) /* elistfile for entries, alist for fields */
elistfile = embDbiSortOpen(alistfile, jfile,
dbname, fields, nfields);
idCountFile = 0;
for(i=0;i<nfields;i++)
countField[i] = 0;
while((entry=dbiblast_nextblastentry(db, jfile,
idformat, systemsort,
fields,
maxFieldLen,
&maxidlen, countField,
elistfile, alistfile)))
{
idCountFile++;
if(!systemsort) /* save the entry data in lists */
{
embDbiMemEntry(idlist, fieldList, nfields, entry, jfile);
}
}
idCount += idCountFile;
if(systemsort)
{
embDbiSortClose(&elistfile, alistfile, nfields);
/* lost the entry, so can't free it :-) */
}
embDbiLogFile(logfile, curfilename, idCountFile, fields,
countField, nfields);
dbiblast_dbfree(&db);
jfile++;
}
nfiles = jfile;
/*
** write the division.lkp file
*/
embDbiWriteDivision(indexdir, dbname, release, date,
maxfilelen, nfiles, divfiles, NULL);
/*
** Write the entryname.idx index
*/
ajStrAssignC(&tmpfname, "entrynam.idx");
entFile = ajFileNewOutNamePathS(tmpfname, indexdir);
recsize = maxidlen+10;
filesize = 300 + (idCount*(ajint)recsize);
embDbiHeader(entFile, filesize, idCount, recsize, dbname, release, date);
if(systemsort)
idDone = embDbiSortWriteEntry(entFile, maxidlen,
dbname, nfiles, cleanup, sortopt);
else /* save entries in entryIds array */
{
idDone = embDbiMemWriteEntry(entFile, maxidlen,
idlist, &entryIds);
if(idDone != idCount)
ajFatal("Duplicates not allowed for in-memory processing");
}
embDbiHeaderSize(entFile, 300+(idDone*(ajint)recsize), idDone);
ajFileClose(&entFile);
/*
** Write the fields index files
*/
for(ifield=0; ifield < nfields; ifield++)
{
if(maxindex)
maxlen = maxindex;
else
{
if(maxFieldLen[ifield] >= 0)
maxlen = maxFieldLen[ifield];
else
maxlen = - maxFieldLen[ifield];
}
if(systemsort)
fieldTot[ifield] = embDbiSortWriteFields(dbname, release,
date, indexdir,
fields[ifield], maxlen,
nfiles, idCount,
cleanup, sortopt);
else
fieldTot[ifield] = embDbiMemWriteFields(dbname, release,
date, indexdir,
fields[ifield], maxlen,
fieldList[ifield],
entryIds);
}
embDbiLogFinal(logfile,maxindex, maxFieldLen, fields, fieldTot,
nfields, nfiles, idDone, idCount);
if(systemsort)
embDbiRmEntryFile(dbname, cleanup);
ajListMap(idlist, embDbiEntryDelMap, NULL);
ajListFree(&idlist);
AJFREE(entryIds);
ajStrDelarray(&fields);
for(i=0;i<nfields;i++)
{
if(systemsort)
{
ajFileClose(&alistfile[i]);
}
else
{
ajListMap(fieldList[i], embDbiFieldDelMap, NULL);
ajListFree(&fieldList[i]);
}
}
AJFREE(alistfile);
AJFREE(fieldList);
ajStrDel(&version);
ajStrDel(&seqtype);
ajFileClose(&elistfile);
for(i=0;i<nfiles;i++)
{
ajStrDel(&divfiles[i]);
}
AJFREE(countField);
AJFREE(fieldTot);
ajStrDel(&dbname);
ajStrDel(&release);
ajStrDel(&datestr);
ajStrDel(&sortopt);
ajStrDel(&directory);
ajStrDel(&indexdir);
ajStrDel(&filename);
ajStrDel(&exclude);
ajStrDel(&idformat);
ajStrDel(&tmpfname);
AJFREE(maxFieldLen);
ajFileClose(&logfile);
ajListstrFreeData(&listTestFiles);
ajStrDel(&t);
ajStrDel(&id);
ajStrDel(&acc);
ajStrDel(&hline);
ajStrDel(&tmpdes);
ajStrDel(&tmpfd);
ajStrDel(&tmpgi);
ajStrDel(&tmpdb);
ajStrDel(&tmpac);
ajStrDel(&tmpsv);
ajRegFree(&wrdexp);
embDbiEntryDel(&dbiblastEntry);
if(fdl)
{
for(i=0; i < nfields; i++)
ajListFree(&fdl[i]);
AJFREE(fdl);
}
for(i=0;i<nfiles;i++)
{
ajStrDel(&divfiles[i]);
}
AJFREE(divfiles);
AJFREE(testFiles);
embExit();
return 0;
}
int main(int argc, char **argv)
{
AjPList idlist;
AjPList* fieldList = NULL;
AjBool systemsort;
AjBool cleanup;
ajuint maxindex;
ajuint maxidlen = 0;
ajuint maxlen;
AjPFile elistfile = NULL;
AjPFile* alistfile = NULL;
AjPStr dbname = NULL;
AjPStr release = NULL;
AjPStr datestr = NULL;
AjPStr sortopt = NULL;
void **entryIds = NULL;
AjPStr directory;
AjPStr indexdir;
AjPStr filename;
AjPStr exclude;
AjPStr curfilename = NULL;
AjPFile libr=NULL;
AjPStr idformat = NULL;
EmbPEntry entry;
ajuint idtype = 0;
ajuint idCount = 0;
ajuint idDone;
AjPList listInputFiles = NULL;
void ** inputFiles = NULL;
ajuint nfiles;
ajuint ifile;
ajuint filesize;
short recsize;
ajuint maxfilelen = 20;
char date[4] =
{
0,0,0,0
};
AjPStr tmpfname = NULL;
AjPStr* fields = NULL;
AjPFile entFile = NULL;
AjPStr* divfiles = NULL;
AjPRegexp regIdExp = NULL;
ajint* maxFieldLen = NULL;
ajuint ifield = 0;
ajuint nfields = 0;
AjPFile logfile = NULL;
ajuint* countField = NULL;
ajuint* fieldTot = NULL;
ajuint idCountFile = 0;
ajuint i;
embInit("dbifasta", argc, argv);
idformat = ajAcdGetListSingle("idformat");
fields = ajAcdGetList("fields");
directory = ajAcdGetDirectoryName("directory");
indexdir = ajAcdGetOutdirName("indexoutdir");
filename = ajAcdGetString("filenames");
exclude = ajAcdGetString("exclude");
dbname = ajAcdGetString("dbname");
release = ajAcdGetString("release");
datestr = ajAcdGetString("date");
systemsort = ajAcdGetBoolean("systemsort");
cleanup = ajAcdGetBoolean("cleanup");
sortopt = ajAcdGetString("sortoptions");
maxindex = ajAcdGetInt("maxindex");
logfile = ajAcdGetOutfile("outfile");
while(fields[nfields]) /* array ends with a NULL */
nfields++;
if(nfields)
{
AJCNEW(maxFieldLen, nfields);
AJCNEW0(countField, nfields);
AJCNEW0(fieldTot, nfields);
for(ifield=0; ifield < nfields; ifield++)
maxFieldLen[ifield] = (ajint)maxindex * -1;
if(systemsort)
AJCNEW(alistfile, nfields);
else
{
AJCNEW(fieldList, nfields);
for(ifield=0; ifield < nfields; ifield++)
fieldList[ifield] = ajListNew();
}
}
if(ajStrMatchC(datestr, "00/00/00"))
ajFmtPrintS(&datestr, "%D", ajTimeRefTodayFmt("dbindex"));
ajStrRemoveWhite(&dbname); /* used for temp filenames */
embDbiDateSet(datestr, date);
idlist = ajListNew();
regIdExp = dbifasta_getExpr(idformat, &idtype);
ajDebug("reading '%S/%S'\n", directory, filename);
ajDebug("writing '%S/'\n", indexdir);
listInputFiles = embDbiFileListExc(directory, filename, exclude);
ajListSort(listInputFiles, &ajStrVcmp);
nfiles = (ajuint) ajListToarray(listInputFiles, &inputFiles);
if(!nfiles)
ajDie("No input files in '%S' matched filename '%S'",
directory, filename);
embDbiLogHeader(logfile, dbname, release, datestr,
indexdir, maxindex);
embDbiLogFields(logfile, fields, nfields);
embDbiLogSource(logfile, directory, filename, exclude,
(AjPStr*) inputFiles, nfiles);
embDbiLogCmdline(logfile);
AJCNEW0(divfiles, nfiles);
/*
** process each input file, one at a time
*/
for(ifile=0; ifile < nfiles; ifile++)
{
ajStrAssignS(&curfilename,(AjPStr) inputFiles[ifile]);
embDbiFlatOpenlib(curfilename, &libr);
ajFilenameTrimPath(&curfilename);
if(ajStrGetLen(curfilename) >= maxfilelen)
maxfilelen = ajStrGetLen(curfilename) + 1;
ajDebug("processing filename '%S' ...\n", curfilename);
ajDebug("processing file '%F' ...\n", libr);
ajStrAssignS(&divfiles[ifile], curfilename);
if(systemsort) /* elistfile for entries, alist for fields */
elistfile = embDbiSortOpen(alistfile, ifile,
dbname, fields, nfields);
idCountFile = 0;
for(i=0;i<nfields;i++)
countField[i] = 0;
while((entry=dbifasta_NextFlatEntry(libr, ifile,
regIdExp, idtype,
systemsort, fields,
maxFieldLen, &maxidlen,
countField, elistfile,
alistfile)))
{
idCountFile++;
if(!systemsort) /* save the entry data in lists */
embDbiMemEntry(idlist, fieldList, nfields,
entry, ifile);
entry = NULL;
}
idCount += idCountFile;
if(systemsort)
{
embDbiSortClose(&elistfile, alistfile, nfields);
AJFREE(entry);
}
else
{
embDbiEntryDel(&dbifastaGEntry);
}
embDbiLogFile(logfile, curfilename, idCountFile, fields,
countField, nfields);
}
/* write the division.lkp file */
embDbiWriteDivision(indexdir, dbname, release, date,
maxfilelen, nfiles, divfiles, NULL);
/* Write the entryname.idx index */
ajStrAssignC(&tmpfname, "entrynam.idx");
entFile = ajFileNewOutNamePathS(tmpfname, indexdir);
recsize = maxidlen+10;
filesize = 300 + (idCount*(ajint)recsize);
embDbiHeader(entFile, filesize, idCount, recsize, dbname, release, date);
if(systemsort)
idDone = embDbiSortWriteEntry(entFile, maxidlen,
dbname, nfiles, cleanup, sortopt);
else /* save entries in entryIds array */
{
idDone = embDbiMemWriteEntry(entFile, maxidlen,
idlist, &entryIds);
if(idDone != idCount)
ajFatal("Duplicates not allowed for in-memory processing");
}
embDbiHeaderSize(entFile, 300+(idDone*(ajint)recsize), idDone);
ajFileClose(&entFile);
/* Write the fields index files */
for(ifield=0; ifield < nfields; ifield++)
{
if(maxindex)
maxlen = maxindex;
else
{
if(maxFieldLen[ifield] >= 0)
maxlen = maxFieldLen[ifield];
else
maxlen = - maxFieldLen[ifield];
}
if(systemsort)
fieldTot[ifield] = embDbiSortWriteFields(dbname, release,
date, indexdir,
fields[ifield], maxlen,
nfiles, idCount,
cleanup, sortopt);
else
fieldTot[ifield] = embDbiMemWriteFields(dbname, release,
date, indexdir,
fields[ifield], maxlen,
fieldList[ifield],
entryIds);
}
embDbiLogFinal(logfile,maxindex, maxFieldLen, fields, fieldTot,
nfields, nfiles, idDone, idCount);
if(systemsort)
embDbiRmEntryFile(dbname, cleanup);
ajStrDel(&idformat);
ajStrDelarray(&fields);
ajStrDel(&filename);
ajStrDel(&exclude);
ajStrDel(&dbname);
ajStrDel(&release);
ajStrDel(&datestr);
ajStrDel(&sortopt);
ajStrDel(&directory);
ajStrDel(&indexdir);
ajStrDel(&tmpfname);
ajFileClose(&libr);
ajFileClose(&logfile);
for(i=0;i<nfields;i++)
{
if(systemsort)
{
ajFileClose(&alistfile[i]);
}
else
{
ajListMap(fieldList[i], &embDbiFieldDelMap, NULL);
ajListFree(&fieldList[i]);
}
}
AJFREE(alistfile);
AJFREE(fieldList);
AJFREE(maxFieldLen);
AJFREE(countField);
AJFREE(fieldTot);
for(i=0;i<nfiles;i++)
{
ajStrDel(&divfiles[i]);
}
AJFREE(divfiles);
AJFREE(inputFiles);
embDbiEntryDel(&dbifastaGEntry);
ajStrDel(&dbifastaGRline);
ajStrDel(&dbifastaGTmpId);
if(dbifastaGFdl)
{
for(i=0; i < nfields; i++)
ajListFree(&dbifastaGFdl[i]);
AJFREE(dbifastaGFdl);
}
ajListMap(idlist, &embDbiEntryDelMap, NULL);
ajListFree(&idlist);
ajListstrFreeData(&listInputFiles);
AJFREE(entryIds);
ajRegFree(&dbifastaGIdexp);
ajRegFree(&dbifastaGWrdexp);
ajRegFree(®IdExp);
ajStrDel(&dbifastaGTmpAc);
ajStrDel(&dbifastaGTmpSv);
ajStrDel(&dbifastaGTmpGi);
ajStrDel(&dbifastaGTmpDb);
ajStrDel(&dbifastaGTmpDes);
ajStrDel(&dbifastaGTmpFd);
ajStrDel(&curfilename);
embExit();
return 0;
}