static int
do_by_sequences(ESL_GENCODE *gcode, ESL_GENCODE_WORKSTATE *wrk, ESL_SQFILE *sqfp)
{
ESL_SQ *sq = esl_sq_CreateDigital(gcode->nt_abc);
int status;
while (( status = esl_sqio_Read(sqfp, sq )) == eslOK)
{
if (sq->n < 3) continue;
if (wrk->do_watson) {
esl_gencode_ProcessStart(gcode, wrk, sq);
esl_gencode_ProcessPiece(gcode, wrk, sq);
esl_gencode_ProcessEnd(wrk, sq);
}
if (wrk->do_crick) {
esl_sq_ReverseComplement(sq);
esl_gencode_ProcessStart(gcode, wrk, sq);
esl_gencode_ProcessPiece(gcode, wrk, sq);
esl_gencode_ProcessEnd(wrk, sq);
}
esl_sq_Reuse(sq);
}
if (status == eslEFORMAT) esl_fatal("Parse failed (sequence file %s)\n%s\n",
sqfp->filename, sqfp->get_error(sqfp));
else if (status != eslEOF) esl_fatal("Unexpected error %d reading sequence file %s",
status, sqfp->filename);
esl_sq_Destroy(sq);
return eslOK;
}
/* Open the source sequence database for negative subseqs;
* upon return, cfg->dbfp is open (digital, SSI indexed);
* cfg->db_maxL and cfg->db_nseq are set.
*/
static int
process_dbfile(struct cfg_s *cfg, char *dbfile, int dbfmt)
{
ESL_SQ *sq = esl_sq_CreateDigital(cfg->abc);
int status;
/* Open the sequence file in digital mode */
status = esl_sqfile_OpenDigital(cfg->abc, dbfile, dbfmt, NULL, &(cfg->dbfp));
if (status == eslENOTFOUND) esl_fatal("No such file %s", dbfile);
else if (status == eslEFORMAT) esl_fatal("Format of seqfile %s unrecognized.", dbfile);
else if (status == eslEINVAL) esl_fatal("Can't autodetect stdin or .gz.");
else if (status != eslOK) esl_fatal("Open failed, code %d.", status);
/* Read info on each sequence */
cfg->db_nseq = 0;
cfg->db_maxL = 0;
while ((status = esl_sqio_ReadInfo(cfg->dbfp, sq)) == eslOK) {
cfg->db_maxL = ESL_MAX(sq->L, cfg->db_maxL);
cfg->db_nseq++;
esl_sq_Reuse(sq);
}
if (status != eslEOF) esl_fatal("Something went wrong with reading the seq db");
/* Open SSI index */
if (esl_sqfile_OpenSSI(cfg->dbfp, NULL) != eslOK) esl_fatal("Failed to open SSI index file");
if (cfg->dbfp->data.ascii.ssi->nprimary != cfg->db_nseq) esl_fatal("oops, nprimary != nseq");
esl_sq_Destroy(sq);
return eslOK;
}
int
main(int argc, char **argv)
{
ESL_GETOPTS *go = p7_CreateDefaultApp(options, 1, argc, argv, banner, usage);
ESL_RANDOMNESS *rng = esl_randomness_CreateFast(esl_opt_GetInteger(go, "-s"));
char *hmmfile = esl_opt_GetArg(go, 1);
int L = esl_opt_GetInteger(go, "-L");
int N = esl_opt_GetInteger(go, "-N");
ESL_ALPHABET *abc = NULL;
P7_HMMFILE *hfp = NULL;
P7_HMM *hmm = NULL;
P7_BG *bg = NULL;
P7_PROFILE *gm = NULL;
P7_TRACE *tr = p7_trace_Create();
ESL_SQ *sq = NULL;
char errbuf[eslERRBUFSIZE];
int i;
int status;
status = p7_hmmfile_OpenE(hmmfile, NULL, &hfp, errbuf);
if (status == eslENOTFOUND) p7_Fail("File existence/permissions problem in trying to open HMM file %s.\n%s\n", hmmfile, errbuf);
else if (status == eslEFORMAT) p7_Fail("File format problem in trying to open HMM file %s.\n%s\n", hmmfile, errbuf);
else if (status != eslOK) p7_Fail("Unexpected error %d in opening HMM file %s.\n%s\n", status, hmmfile, errbuf);
status = p7_hmmfile_Read(hfp, &abc, &hmm);
if (status == eslEFORMAT) p7_Fail("Bad file format in HMM file %s:\n%s\n", hfp->fname, hfp->errbuf);
else if (status == eslEINCOMPAT) p7_Fail("HMM in %s is not in the expected %s alphabet\n", hfp->fname, esl_abc_DecodeType(abc->type));
else if (status == eslEOF) p7_Fail("Empty HMM file %s? No HMM data found.\n", hfp->fname);
else if (status != eslOK) p7_Fail("Unexpected error in reading HMMs from %s\n", hfp->fname);
p7_hmmfile_Close(hfp);
bg = p7_bg_Create(abc); p7_bg_SetLength(bg, L);
gm = p7_profile_Create(hmm->M, abc); p7_ProfileConfig(hmm, bg, gm, L, p7_LOCAL);
sq = esl_sq_CreateDigital(abc);
for (i = 0; i < N; i++)
{
p7_ProfileEmit(rng, hmm, gm, bg, sq, tr);
esl_sq_FormatName(sq, "%s-sample%d", hmm->name, i);
esl_sqio_Write(stdout, sq, eslSQFILE_FASTA, FALSE);
if (p7_trace_Validate(tr, abc, sq->dsq, errbuf) != eslOK) esl_fatal(errbuf);
esl_sq_Reuse(sq);
p7_trace_Reuse(tr);
}
esl_sq_Destroy(sq);
p7_trace_Destroy(tr);
p7_profile_Destroy(gm);
p7_bg_Destroy(bg);
p7_hmm_Destroy(hmm);
esl_alphabet_Destroy(abc);
esl_randomness_Destroy(rng);
esl_getopts_Destroy(go);
return 0;
}
static int
do_by_windows(ESL_GENCODE *gcode, ESL_GENCODE_WORKSTATE *wrk, ESL_SQFILE *sqfp)
{
ESL_SQ *sq = esl_sq_CreateDigital(gcode->nt_abc);
int windowsize = 4092; // can be any value, but a multiple of 3 makes most sense. windowsize can be +/-; + means reading top strand; - means bottom strand.
int contextsize = 2; // contextsize (adjacent window overlap) must be 2, or translation won't work properly.
int wstatus;
ESL_DASSERT1(( windowsize % 3 == 0 ));
while (( wstatus = esl_sqio_ReadWindow(sqfp, contextsize, windowsize, sq)) != eslEOF)
{
if (wstatus == eslEOD)
{
if ( (windowsize > 0 && wrk->do_watson) || (windowsize < 0 && wrk->do_crick))
esl_gencode_ProcessEnd(wrk, sq);
if (windowsize > 0 && ! wrk->do_crick) { esl_sq_Reuse(sq); continue; } // Don't switch to revcomp if we don't need do. Allows -W --watson to work on nonrewindable streams
if (windowsize < 0) esl_sq_Reuse(sq); // Do not Reuse the sq on the switch from watson to crick; ReadWindow needs sq->L
windowsize = -windowsize; // switch to other strand.
continue;
}
else if (wstatus == eslEFORMAT) esl_fatal("Parsing failed in sequence file %s:\n%s", sqfp->filename, esl_sqfile_GetErrorBuf(sqfp));
else if (wstatus == eslEINVAL) esl_fatal("Invalid residue(s) found in sequence file %s\n%s", sqfp->filename, esl_sqfile_GetErrorBuf(sqfp));
else if (wstatus != eslOK) esl_fatal("Unexpected error %d reading sequence file %s", wstatus, sqfp->filename);
/* If we're the first window in this input DNA sequence
* (or the first window in its revcomp), then initialize.
* sq->C is the actual context overlap; 0=1st window; 2 (i.e. C)= subsequent.
*/
if (sq->C == 0)
{
if (sq->n < 3) continue; // DNA sequence too short; skip it, don't even bother to revcomp, go to next sequence.
if ( (windowsize > 0 && wrk->do_watson) || (windowsize < 0 && wrk->do_crick))
esl_gencode_ProcessStart(gcode, wrk, sq);
}
if ( (windowsize > 0 && wrk->do_watson) || (windowsize < 0 && wrk->do_crick))
esl_gencode_ProcessPiece(gcode, wrk, sq);
}
esl_sq_Destroy(sq);
return eslOK;
}
/* onefetch():
* Given one <key> (a seq name or accession), retrieve the corresponding sequence.
* In SSI mode, we can do this quickly by positioning the file, then regurgitating
* every line until the end-of-record marker; we don't even have to parse.
* Without an SSI index, we have to parse the file sequentially 'til we find
* the one we're after.
*/
static void
onefetch(ESL_GETOPTS *go, FILE *ofp, char *key, ESL_SQFILE *sqfp)
{
ESL_SQ *sq = esl_sq_Create();
int do_revcomp = esl_opt_GetBoolean(go, "-r");
char *newname = esl_opt_GetString(go, "-n");
int status;
/* Try to position the file at the desired sequence with SSI. */
if (sqfp->data.ascii.ssi != NULL)
{
status = esl_sqfile_PositionByKey(sqfp, key);
if (status == eslENOTFOUND) esl_fatal("seq %s not found in SSI index for file %s\n", key, sqfp->filename);
else if (status == eslEFORMAT) esl_fatal("Failed to parse SSI index for %s\n", sqfp->filename);
else if (status != eslOK) esl_fatal("Failed to look up location of seq %s in SSI index of file %s\n", key, sqfp->filename);
status = esl_sqio_Read(sqfp, sq);
if (status == eslEFORMAT) esl_fatal("Parse failed (sequence file %s):\n%s\n",
sqfp->filename, esl_sqfile_GetErrorBuf(sqfp));
else if (status == eslEOF) esl_fatal("Unexpected EOF reading sequence file %s",
status, sqfp->filename);
else if (status != eslOK) esl_fatal("Unexpected error %d reading sequence file %s",
status, sqfp->filename);
if (strcmp(key, sq->name) != 0 && strcmp(key, sq->acc) != 0)
esl_fatal("whoa, internal error; found the wrong sequence %s, not %s", sq->name, key);
}
else
{ /* Else, we have to read the whole damn file sequentially until we find the seq */
while ((status = esl_sqio_Read(sqfp, sq)) != eslEOF) {
if (status == eslEFORMAT) esl_fatal("Parse failed (sequence file %s):\n%s\n",
sqfp->filename, esl_sqfile_GetErrorBuf(sqfp));
else if (status != eslOK) esl_fatal("Unexpected error %d reading sequence file %s",
status, sqfp->filename);
if (strcmp(key, sq->name) == 0 || strcmp(key, sq->acc) == 0) break;
esl_sq_Reuse(sq);
}
if (status == eslEOF) esl_fatal("Failed to find sequence %s in file %s\n", key, sqfp->filename);
}
if (do_revcomp == FALSE && newname == NULL && ! esl_sqio_IsAlignment(sqfp->format))
{ /* If we're not manipulating the sequence in any way, and it's not from an alignment file, we can Echo() it. */
if (esl_sqio_Echo(sqfp, sq, ofp) != eslOK) esl_fatal("Echo failed: %s\n", esl_sqfile_GetErrorBuf(sqfp));
}
else
{ /* Otherwise we Write() the parsed version. */
if (do_revcomp && esl_sq_ReverseComplement(sq) != eslOK) esl_fatal("Failed to reverse complement %s; is it a protein?\n", sq->name);
if (newname != NULL) esl_sq_SetName(sq, newname);
esl_sqio_Write(ofp, sq, eslSQFILE_FASTA, FALSE);
}
esl_sq_Destroy(sq);
}
static void
emit_sequences(ESL_GETOPTS *go, FILE *ofp, int outfmt, ESL_RANDOMNESS *r, P7_HMM *hmm)
{
ESL_SQ *sq = NULL;
P7_TRACE *tr = NULL;
P7_BG *bg = NULL;
P7_PROFILE *gm = NULL;
int do_profile = esl_opt_GetBoolean(go, "-p");
int N = esl_opt_GetInteger(go, "-N");
int L = esl_opt_GetInteger(go, "-L");
int mode = p7_LOCAL;
int nseq;
int status;
if (esl_opt_GetBoolean(go, "--local")) mode = p7_LOCAL;
else if (esl_opt_GetBoolean(go, "--unilocal")) mode = p7_UNILOCAL;
else if (esl_opt_GetBoolean(go, "--glocal")) mode = p7_GLOCAL;
else if (esl_opt_GetBoolean(go, "--uniglocal")) mode = p7_UNIGLOCAL;
if ((sq = esl_sq_CreateDigital(hmm->abc)) == NULL) esl_fatal("failed to allocate sequence");
if ((tr = p7_trace_Create()) == NULL) esl_fatal("failed to allocate trace");
if ((bg = p7_bg_Create(hmm->abc)) == NULL) esl_fatal("failed to create null model");
if ((gm = p7_profile_Create(hmm->M, hmm->abc)) == NULL) esl_fatal("failed to create profile");
if (p7_ProfileConfig(hmm, bg, gm, L, mode) != eslOK) esl_fatal("failed to configure profile");
if (p7_bg_SetLength(bg, L) != eslOK) esl_fatal("failed to reconfig null model length");
if (p7_hmm_Validate (hmm, NULL, 0.0001) != eslOK) esl_fatal("whoops, HMM is bad!");
if (p7_profile_Validate(gm, NULL, 0.0001) != eslOK) esl_fatal("whoops, profile is bad!");
for (nseq = 1; nseq <= N; nseq++)
{
if (do_profile) status = p7_ProfileEmit(r, hmm, gm, bg, sq, tr);
else status = p7_CoreEmit (r, hmm, sq, tr);
if (status) esl_fatal("Failed to emit sequence\n");
status = esl_sq_FormatName(sq, "%s-sample%d", hmm->name, nseq);
if (status) esl_fatal("Failed to set sequence name\n");
status = esl_sqio_Write(ofp, sq, outfmt, FALSE);
if (status != eslOK) esl_fatal("Failed to write sequence\n");
p7_trace_Reuse(tr);
esl_sq_Reuse(sq);
}
esl_sq_Destroy(sq);
p7_trace_Destroy(tr);
p7_bg_Destroy(bg);
p7_profile_Destroy(gm);
return;
}
int
main(int argc, char **argv)
{
ESL_GETOPTS *go = esl_getopts_CreateDefaultApp(options, 1, argc, argv, banner, usage);
ESL_ALPHABET *abc = esl_alphabet_Create(eslAMINO);
ESL_SQ *sq = esl_sq_CreateDigital(abc);
ESL_SQFILE *sqfp = NULL;
ESL_HMM *hmm = create_test_hmm(abc);
ESL_HMM *bg = create_null_hmm(abc);
ESL_HMX *hmx = esl_hmx_Create(400, hmm->M);
int format = eslSQFILE_UNKNOWN;
char *seqfile = esl_opt_GetArg(go, 1);
float fwdsc, nullsc;
int status;
status = esl_sqfile_OpenDigital(abc, seqfile, format, NULL, &sqfp);
if (status == eslENOTFOUND) esl_fatal("No such file.");
else if (status == eslEFORMAT) esl_fatal("Format unrecognized.");
else if (status != eslOK) esl_fatal("Open failed, code %d.", status);
while ((status = esl_sqio_Read(sqfp, sq)) == eslOK)
{
esl_hmx_GrowTo(hmx, sq->n, hmm->M);
esl_hmm_Forward(sq->dsq, sq->n, hmm, hmx, &fwdsc);
esl_hmm_Forward(sq->dsq, sq->n, bg, hmx, &nullsc);
printf("%-16s %5d %11.4f %8.4f %11.4f %8.4f %11.4f %8.4f\n", sq->name, (int) sq->n,
fwdsc * eslCONST_LOG2R, (fwdsc * eslCONST_LOG2R) / sq->n,
nullsc * eslCONST_LOG2R, (nullsc * eslCONST_LOG2R) / sq->n,
(fwdsc - nullsc) * eslCONST_LOG2R, (fwdsc-nullsc) * eslCONST_LOG2R / sq->n);
esl_sq_Reuse(sq);
}
if (status == eslEFORMAT) esl_fatal("Parse failed (sequence file %s)\n%s\n",
sqfp->filename, sqfp->get_error(sqfp));
else if (status != eslEOF) esl_fatal("Unexpected error %d reading sequence file %s",
status, sqfp->filename);
esl_sqfile_Close(sqfp);
esl_sq_Destroy(sq);
esl_hmm_Destroy(hmm);
esl_hmm_Destroy(bg);
esl_hmx_Destroy(hmx);
esl_alphabet_Destroy(abc);
esl_getopts_Destroy(go);
return 0;
}
static int
synthesize_negatives(ESL_GETOPTS *go, struct cfg_s *cfg, int nneg)
{
ESL_SQ *sq = esl_sq_CreateDigital(cfg->abc);
int a;
int i;
int L1,L2,L3,d1n,d2n;
for (i = 0; i < nneg; i++)
{
/* Select a random test seq, to use its same segments */
a = esl_rnd_Roll(cfg->r, cfg->ntest);
L1 = cfg->test_lens[a].L1;
L2 = cfg->test_lens[a].L2;
L3 = cfg->test_lens[a].L3;
d1n = cfg->test_lens[a].d1n;
d2n = cfg->test_lens[a].d2n;
esl_sq_GrowTo(sq, cfg->test_lens[a].L);
esl_sq_FormatName(sq, "decoy%d", i+1);
esl_sq_FormatDesc(sq, "L=%d in segments: %d/%d/%d/%d/%d", cfg->test_lens[a].L, L1, d1n, L2, d2n, L3);
sq->n = cfg->test_lens[a].L;
fprintf(cfg->negsummfp, "%-15s %5d %5d %5d %5d %5d %5d",
sq->name, (int) sq->n,
L1, d1n, L2, d2n, L3);
sq->dsq[0] = sq->dsq[cfg->test_lens[a].L+1] = eslDSQ_SENTINEL;
set_random_segment(go, cfg, cfg->negsummfp, sq->dsq+1, L1);
set_random_segment(go, cfg, cfg->negsummfp, sq->dsq+1+L1, d1n);
set_random_segment(go, cfg, cfg->negsummfp, sq->dsq+1+L1+d1n, L2);
set_random_segment(go, cfg, cfg->negsummfp, sq->dsq+1+L1+d1n+L2, d2n);
set_random_segment(go, cfg, cfg->negsummfp, sq->dsq+1+L1+d1n+L2+d2n, L3);
fprintf(cfg->negsummfp, "\n");
esl_sqio_Write(cfg->out_seqfp, sq, eslSQFILE_FASTA, FALSE);
esl_sq_Reuse(sq);
}
esl_sq_Destroy(sq);
return eslOK;
}
int
main(int argc, char **argv)
{
ESL_GETOPTS *go = esl_getopts_CreateDefaultApp(options, 2, argc, argv, banner, usage);
char *hmmfile = esl_opt_GetArg(go, 1);
char *seqfile = esl_opt_GetArg(go, 2);
ESL_ALPHABET *abc = NULL;
P7_HMMFILE *hfp = NULL;
P7_HMM *hmm = NULL;
P7_BG *bg = NULL;
P7_PROFILE *gm = NULL;
P7_OPROFILE *om = NULL;
P7_GMX *gx = NULL;
P7_OMX *fwd = NULL;
P7_OMX *bck = NULL;
ESL_SQ *sq = NULL;
ESL_SQFILE *sqfp = NULL;
int format = eslSQFILE_UNKNOWN;
float fraw, braw, nullsc, fsc;
float gfraw, gbraw, gfsc;
double P, gP;
int status;
/* Read in one HMM */
if (p7_hmmfile_Open(hmmfile, NULL, &hfp) != eslOK) p7_Fail("Failed to open HMM file %s", hmmfile);
if (p7_hmmfile_Read(hfp, &abc, &hmm) != eslOK) p7_Fail("Failed to read HMM");
/* Open sequence file for reading */
sq = esl_sq_CreateDigital(abc);
status = esl_sqfile_Open(seqfile, format, NULL, &sqfp);
if (status == eslENOTFOUND) p7_Fail("No such file.");
else if (status == eslEFORMAT) p7_Fail("Format unrecognized.");
else if (status == eslEINVAL) p7_Fail("Can't autodetect stdin or .gz.");
else if (status != eslOK) p7_Fail("Open failed, code %d.", status);
/* create default null model, then create and optimize profile */
bg = p7_bg_Create(abc);
p7_bg_SetLength(bg, sq->n);
gm = p7_profile_Create(hmm->M, abc);
p7_ProfileConfig(hmm, bg, gm, sq->n, p7_UNILOCAL);
om = p7_oprofile_Create(gm->M, abc);
p7_oprofile_Convert(gm, om);
/* p7_oprofile_Dump(stdout, om); */
/* allocate DP matrices for O(M+L) parsers */
fwd = p7_omx_Create(gm->M, 0, sq->n);
bck = p7_omx_Create(gm->M, 0, sq->n);
gx = p7_gmx_Create(gm->M, sq->n);
/* allocate DP matrices for O(ML) fills */
/* fwd = p7_omx_Create(gm->M, sq->n, sq->n); */
/* bck = p7_omx_Create(gm->M, sq->n, sq->n); */
/* p7_omx_SetDumpMode(stdout, fwd, TRUE); */ /* makes the fast DP algorithms dump their matrices */
/* p7_omx_SetDumpMode(stdout, bck, TRUE); */
while ((status = esl_sqio_Read(sqfp, sq)) == eslOK)
{
p7_oprofile_ReconfigLength(om, sq->n);
p7_ReconfigLength(gm, sq->n);
p7_bg_SetLength(bg, sq->n);
p7_omx_GrowTo(fwd, om->M, 0, sq->n);
p7_omx_GrowTo(bck, om->M, 0, sq->n);
p7_gmx_GrowTo(gx, gm->M, sq->n);
p7_bg_NullOne (bg, sq->dsq, sq->n, &nullsc);
p7_ForwardParser (sq->dsq, sq->n, om, fwd, &fraw);
p7_BackwardParser(sq->dsq, sq->n, om, fwd, bck, &braw);
/* p7_Forward (sq->dsq, sq->n, om, fwd, &fsc); printf("forward: %.2f nats\n", fsc); */
/* p7_Backward(sq->dsq, sq->n, om, fwd, bck, &bsc); printf("backward: %.2f nats\n", bsc); */
/* Comparison to other F/B implementations */
p7_GForward (sq->dsq, sq->n, gm, gx, &gfraw);
p7_GBackward (sq->dsq, sq->n, gm, gx, &gbraw);
/* p7_gmx_Dump(stdout, gx); */
fsc = (fraw-nullsc) / eslCONST_LOG2;
gfsc = (gfraw-nullsc) / eslCONST_LOG2;
P = esl_exp_surv(fsc, om->evparam[p7_FTAU], om->evparam[p7_FLAMBDA]);
gP = esl_exp_surv(gfsc, gm->evparam[p7_FTAU], gm->evparam[p7_FLAMBDA]);
if (esl_opt_GetBoolean(go, "-1"))
{
printf("%-30s\t%-20s\t%9.2g\t%6.1f\t%9.2g\t%6.1f\n", sq->name, hmm->name, P, fsc, gP, gfsc);
}
else if (esl_opt_GetBoolean(go, "-P"))
{ /* output suitable for direct use in profmark benchmark postprocessors: */
printf("%g\t%.2f\t%s\t%s\n", P, fsc, sq->name, hmm->name);
}
else
{
printf("target sequence: %s\n", sq->name);
printf("fwd filter raw score: %.2f nats\n", fraw);
printf("bck filter raw score: %.2f nats\n", braw);
printf("null score: %.2f nats\n", nullsc);
printf("per-seq score: %.2f bits\n", fsc);
printf("P-value: %g\n", P);
printf("GForward raw score: %.2f nats\n", gfraw);
printf("GBackward raw score: %.2f nats\n", gbraw);
printf("GForward seq score: %.2f bits\n", gfsc);
printf("GForward P-value: %g\n", gP);
}
esl_sq_Reuse(sq);
}
/* cleanup */
esl_sq_Destroy(sq);
esl_sqfile_Close(sqfp);
p7_omx_Destroy(bck);
p7_omx_Destroy(fwd);
p7_gmx_Destroy(gx);
p7_oprofile_Destroy(om);
p7_profile_Destroy(gm);
p7_bg_Destroy(bg);
p7_hmm_Destroy(hmm);
p7_hmmfile_Close(hfp);
esl_alphabet_Destroy(abc);
esl_getopts_Destroy(go);
return 0;
}
int main(int argc, char **argv)
{
ESL_GETOPTS *go = esl_getopts_CreateDefaultApp(options, 2, argc, argv, banner, usage);
char *hmmfile = esl_opt_GetArg(go, 1);
char *seqfile = esl_opt_GetArg(go, 2);
ESL_STOPWATCH *w = esl_stopwatch_Create();
ESL_RANDOMNESS*r = esl_randomness_Create(esl_opt_GetInteger(go, "-s"));
ESL_ALPHABET*abc = NULL;
P7_HMMFILE *hfp = NULL;
P7_HMM *hmm = NULL;
P7_BG *bg = NULL;
P7_PROFILE *gm1, *gm2;
int L = esl_opt_GetInteger(go, "-L");
int N = esl_opt_GetInteger(go, "-N") / SSE16_NVALS;
int MaxPart = esl_opt_GetInteger(go, "-M");
int NROUNDS = esl_opt_GetInteger(go, "-R");
int check = esl_opt_GetBoolean(go, "-c");
__m128 resdata[10];
int i, j;
float *sc1 = (float*) resdata;
ESL_SQFILE *sqfp = NULL;
DATA_COPS16 *dcops;
struct timeb tbstart, tbend;
int sumlengths = 0;
float* results = NULL;
srand(time(NULL));
if (p7_hmmfile_Open(hmmfile, NULL, &hfp) != eslOK) p7_Fail("Failed to open HMM file %s", hmmfile);
if (p7_hmmfile_Read(hfp, &abc, &hmm) != eslOK) p7_Fail("Failed to read HMM");
bg = p7_bg_Create(abc);
p7_bg_SetLength(bg, L);
gm1 = p7_profile_Create(hmm->M, abc);
gm2 = p7_profile_Create(hmm->M, abc);
p7_ProfileConfig(hmm, bg, gm1, L, p7_UNILOCAL);
p7_ProfileConfig(hmm, bg, gm2, L, p7_UNILOCAL);
dcops = p7_ViterbiCOPSw_Create(gm1);
p7_ViterbiCOPSW_Setup(dcops, L+100, MaxPart); // use max L
dcops->L = L;
int dbsize = SSE16_NVALS*N;
SEQ **seqsdb= calloc(dbsize+1, sizeof(SEQ*));
int equallength = 1;
if (esl_sqfile_OpenDigital(abc, seqfile, eslSQFILE_FASTA, NULL, &sqfp) == eslOK)
{ // Use Sequence file
ESL_SQ* sq = esl_sq_CreateDigital(abc);
int maxseqs, len=0;
if (esl_opt_IsDefault(go, "-N")) // N not specified in cmdline
maxseqs = INT_MAX; // no limit
else
maxseqs = SSE16_NVALS*N; // use cmdline limit
for (j = 0; j < maxseqs && esl_sqio_Read(sqfp, sq) == eslOK; j++)
{
if (equallength && sq->n != len && j > 0)
equallength = 0;
len = sq->n;
if (j > dbsize)
{ seqsdb = realloc(seqsdb, 2*(dbsize+1)*sizeof(SEQ*));
dbsize *= 2;
}
ESL_DSQ* dsq = sq->dsq;
seqsdb[j] = malloc(sizeof(SEQ));
seqsdb[j]->length = len;
seqsdb[j]->seq = malloc((len+4)*sizeof(ESL_DSQ));
memcpy(seqsdb[j]->seq, dsq, len+2);
sumlengths += len;
esl_sq_Reuse(sq);
}
N = j/SSE16_NVALS;
}
else // Not found database. Generate random sequences
for (i = 0; i < N; i++)
for (j = 0; j < SSE16_NVALS; j++)
{
int len = L; // - rand()%1000;
seqsdb[i*SSE16_NVALS+j] = malloc(sizeof(SEQ));
seqsdb[i*SSE16_NVALS+j]->seq = malloc(len+4);
seqsdb[i*SSE16_NVALS+j]->length = len;
esl_rsq_xfIID(r, bg->f, abc->K, len, seqsdb[i*SSE16_NVALS+j]->seq);
sumlengths += len;
}
printf("Viterbi COPS Word with %d threads, model %s. ModelLen: %d, #Segms: %d, SeqL.: %d, #seqs: %d, Partition: %d, #parts: %d\n",
NTHREADS, hmmfile, gm1->M, (int) ceil(gm1->M/SSE16_NVALS), L, SSE16_NVALS*N*NROUNDS, dcops->partition, dcops->Npartitions);
/* // No. of partitions computed without full parallelism ( == no. of threads active while some are idle)
int Niters_part = dcops->Npartitions % NTHREADS;
// No. of Model lines that could be computed but are wasted by idle threads waiting on the end
int Nwasted_threads = dcops->partition * ((NTHREADS-Niters_part) % NTHREADS);
// No. of lines in the last partition that go beyond M. It's wasted comp time by a single thread
int Nwasted_leftover= (dcops->partition - gm1->M % dcops->partition) % dcops->partition;
// Total number of wasted lines
int wastedcomp = Nwasted_threads + Nwasted_leftover;
// Total number of lines computed and waited for
int totalcomp = wastedcomp + gm1->M; // same as: roundtop(gm1->M, dcops->partition * NTHREADS);
printf("Total Comp Lines: %d | Wasted Comp Lines: %d\n", totalcomp, wastedcomp);
*/
if (check) results = (float*) alloc_m128_aligned64((N+1)*2);
ftime(&tbstart);
if (!equallength)
{ // Sort sequences by length
qsort(seqsdb, N*SSE16_NVALS, sizeof(SEQ*), compare_seqs);
}
for (j = 0; j < NROUNDS; j++)
for (i = 0; i < N; i++)
{
// if (i % 1000 == 0) printf("Seq %d\n", i);
p7_ViterbiCOPSw_run(dcops, seqsdb+i*SSE16_NVALS, sc1);
if (check) memcpy(results+i*SSE16_NVALS, sc1, 32); // 32 bytes indeed! SSE16_NVALS floats
}
ftime(&tbend);
double secs = TIMEDIFF(tbstart,tbend);
w->elapsed = w->user = secs;
esl_stopwatch_Display(stdout, w, "# Opt CPU time: ");
double compmillioncells = NROUNDS * (double) sumlengths * (double) hmm->M * 1e-6;
printf("# %.0fM cells in %.1f Mc/s\n", compmillioncells, compmillioncells / secs);
if (check)
{ P7_OPROFILE *om = p7_oprofile_Create(hmm->M, gm1->abc);
p7_oprofile_Convert(gm1, om);
P7_OMX *ox = p7_omx_Create(hmm->M, 0, 0);
printf("Compare results against base version\n");
for (i = 0; i < N; i++)
{
int maxll = 0; float sc2;
for (j = 0; j < SSE16_NVALS; j++)
if (maxll < seqsdb[i*SSE16_NVALS+j]->length)
maxll = seqsdb[i*SSE16_NVALS+j]->length;
p7_oprofile_ReconfigRestLength(om, maxll);
// p7_ReconfigLength(gm2, maxll); // emulate the lock-step inter-sequence reconfigs
for (j = 0; j < SSE16_NVALS; j++)
{
// p7_ReconfigLength(gm2, seqsdb[i*SSE16_NVALS+j]->length);
// p7_Viterbi_unilocal(seqsdb[i*SSE16_NVALS+j]->seq, seqsdb[i*SSE16_NVALS+j]->length, gm2, &sc3);
// p7_Viterbi_unilocal_word(seqsdb[i*SSE16_NVALS+j]->seq, seqsdb[i*SSE16_NVALS+j]->length, gm2, &sc2);
// p7_oprofile_ReconfigLength(om, seqsdb[i*SSE16_NVALS+j]->length);
p7_ViterbiFilter(seqsdb[i*SSE16_NVALS+j]->seq, seqsdb[i*SSE16_NVALS+j]->length, om, ox, &sc2);
sc2 += 1.0; // -2.0nat optimization, Local to Unilocal mode
if (fabs(results[i*SSE16_NVALS+j] - sc2) > 0.0001)
{ printf("Seq %d Len %4d: %f - %f\tdiff: %f\n", i*SSE16_NVALS+j, seqsdb[i*SSE16_NVALS+j]->length,
results[i*SSE16_NVALS+j], sc2, fabs(results[i*SSE16_NVALS+j] - sc2));
}
}
}
}
return 0;
}
int
main(int argc, char **argv)
{
ESL_GETOPTS *go = esl_getopts_CreateDefaultApp(options, 1, argc, argv, banner, usage);
char *hmmfile = esl_opt_GetArg(go, 1);
int N = esl_opt_GetInteger(go, "-N");
ESL_STOPWATCH *w = esl_stopwatch_Create();
ESL_RANDOMNESS *r = esl_randomness_CreateFast(esl_opt_GetInteger(go, "-s"));
ESL_ALPHABET *abc = NULL;
P7_HMMFILE *hfp = NULL;
P7_HMM *hmm = NULL;
P7_BG *bg = NULL;
P7_PROFILE *gm = NULL;
P7_OPROFILE *om = NULL;
P7_TRACE *tr = NULL;
ESL_SQ *sq = NULL;
P7_ALIDISPLAY *ad = NULL;
int i,z;
if (p7_hmmfile_Open(hmmfile, NULL, &hfp) != eslOK) p7_Fail("Failed to open HMM file %s", hmmfile);
if (p7_hmmfile_Read(hfp, &abc, &hmm) != eslOK) p7_Fail("Failed to read HMM");
p7_hmmfile_Close(hfp);
bg = p7_bg_Create(abc);
p7_bg_SetLength(bg, 0);
gm = p7_profile_Create(hmm->M, abc);
p7_ProfileConfig(hmm, bg, gm, 0, p7_UNIGLOCAL); /* that sets N,C,J to generate nothing */
om = p7_oprofile_Create(gm->M, abc);
p7_oprofile_Convert(gm, om);
if (esl_opt_GetBoolean(go, "-p")) tr = p7_trace_CreateWithPP();
else tr = p7_trace_Create();
sq = esl_sq_CreateDigital(abc);
esl_stopwatch_Start(w);
for (i = 0; i < N; i++)
{
p7_ProfileEmit(r, hmm, gm, bg, sq, tr);
esl_sq_SetName(sq, "random");
if (! esl_opt_GetBoolean(go, "-b"))
{
if (esl_opt_GetBoolean(go, "-p"))
for (z = 0; z < tr->N; z++)
if (tr->i[z] > 0) tr->pp[z] = esl_random(r);
ad = p7_alidisplay_Create(tr, 0, om, sq);
p7_alidisplay_Print(stdout, ad, 40, 80, FALSE);
p7_alidisplay_Destroy(ad);
}
p7_trace_Reuse(tr);
esl_sq_Reuse(sq);
}
esl_stopwatch_Stop(w);
esl_stopwatch_Display(stdout, w, "# CPU time: ");
esl_sq_Destroy(sq);
p7_trace_Destroy(tr);
p7_oprofile_Destroy(om);
p7_profile_Destroy(gm);
p7_bg_Destroy(bg);
p7_hmm_Destroy(hmm);
esl_alphabet_Destroy(abc);
esl_randomness_Destroy(r);
esl_stopwatch_Destroy(w);
esl_getopts_Destroy(go);
return 0;
}
/* serial_master()
* The serial version of hmmsearch.
* For each query HMM in <hmmdb> search the database for hits.
*
* A master can only return if it's successful. All errors are handled
* immediately and fatally with p7_Fail().
*/
static int
serial_master(ESL_GETOPTS *go, struct cfg_s *cfg)
{
FILE *ofp = stdout; /* output file for results (default stdout) */
FILE *tblfp = NULL; /* output stream for tabular per-seq (--tblout) */
FILE *dfamtblfp = NULL; /* output stream for tabular Dfam format (--dfamtblout) */
FILE *aliscoresfp = NULL; /* output stream for alignment scores (--aliscoresout) */
// P7_HMM *hmm = NULL; /* one HMM query */
// P7_SCOREDATA *scoredata = NULL;
int seqfmt = eslSQFILE_UNKNOWN; /* format of seqfile */
ESL_SQFILE *sqfp = NULL; /* open seqfile */
P7_HMMFILE *hfp = NULL; /* open HMM database file */
ESL_ALPHABET *abc = NULL; /* sequence alphabet */
P7_OPROFILE *om = NULL; /* target profile */
ESL_STOPWATCH *w = NULL; /* timing */
ESL_SQ *qsq = NULL; /* query sequence */
int nquery = 0;
int textw;
int status = eslOK;
int hstatus = eslOK;
int sstatus = eslOK;
int i;
int ncpus = 0;
int infocnt = 0;
WORKER_INFO *info = NULL;
#ifdef HMMER_THREADS
P7_OM_BLOCK *block = NULL;
ESL_THREADS *threadObj= NULL;
ESL_WORK_QUEUE *queue = NULL;
#endif
char errbuf[eslERRBUFSIZE];
double window_beta = -1.0 ;
int window_length = -1;
if (esl_opt_IsUsed(go, "--w_beta")) { if ( ( window_beta = esl_opt_GetReal(go, "--w_beta") ) < 0 || window_beta > 1 ) esl_fatal("Invalid window-length beta value\n"); }
if (esl_opt_IsUsed(go, "--w_length")) { if (( window_length = esl_opt_GetInteger(go, "--w_length")) < 4 ) esl_fatal("Invalid window length value\n"); }
w = esl_stopwatch_Create();
if (esl_opt_GetBoolean(go, "--notextw")) textw = 0;
else textw = esl_opt_GetInteger(go, "--textw");
/* If caller declared an input format, decode it */
if (esl_opt_IsOn(go, "--qformat")) {
seqfmt = esl_sqio_EncodeFormat(esl_opt_GetString(go, "--qformat"));
if (seqfmt == eslSQFILE_UNKNOWN) p7_Fail("%s is not a recognized input sequence file format\n", esl_opt_GetString(go, "--qformat"));
}
/* validate options if running as a daemon */
// if (esl_opt_IsOn(go, "--daemon")) {
/* running as a daemon, the input format must be type daemon */
// if (seqfmt != eslSQFILE_UNKNOWN && seqfmt != eslSQFILE_DAEMON)
// esl_fatal("Input format %s not supported. Must be daemon\n", esl_opt_GetString(go, "--qformat"));
// seqfmt = eslSQFILE_DAEMON;
// if (strcmp(cfg->seqfile, "-") != 0) esl_fatal("Query sequence file must be '-'\n");
// }
/* Open the target profile database to get the sequence alphabet */
status = p7_hmmfile_OpenE(cfg->hmmfile, p7_HMMDBENV, &hfp, errbuf);
if (status == eslENOTFOUND) p7_Fail("File existence/permissions problem in trying to open HMM file %s.\n%s\n", cfg->hmmfile, errbuf);
else if (status == eslEFORMAT) p7_Fail("File format problem, trying to open HMM file %s.\n%s\n", cfg->hmmfile, errbuf);
else if (status != eslOK) p7_Fail("Unexpected error %d in opening HMM file %s.\n%s\n", status, cfg->hmmfile, errbuf);
if (! hfp->is_pressed) p7_Fail("Failed to open binary auxfiles for %s: use hmmpress first\n", hfp->fname);
hstatus = p7_oprofile_ReadMSV(hfp, &abc, &om);
if (hstatus == eslEFORMAT) p7_Fail("bad format, binary auxfiles, %s:\n%s", cfg->hmmfile, hfp->errbuf);
else if (hstatus == eslEINCOMPAT) p7_Fail("HMM file %s contains different alphabets", cfg->hmmfile);
else if (hstatus != eslOK) p7_Fail("Unexpected error in reading HMMs from %s", cfg->hmmfile);
p7_oprofile_Destroy(om);
p7_hmmfile_Close(hfp);
/* Open the query sequence database */
status = esl_sqfile_OpenDigital(abc, cfg->seqfile, seqfmt, NULL, &sqfp);
if (status == eslENOTFOUND) p7_Fail("Failed to open sequence file %s for reading\n", cfg->seqfile);
else if (status == eslEFORMAT) p7_Fail("Sequence file %s is empty or misformatted\n", cfg->seqfile);
else if (status == eslEINVAL) p7_Fail("Can't autodetect format of a stdin or .gz seqfile");
else if (status != eslOK) p7_Fail("Unexpected error %d opening sequence file %s\n", status, cfg->seqfile);
if (sqfp->format > 100) // breaking the law! That range is reserved for msa, for aligned formats
p7_Fail("%s contains a multiple sequence alignment; expect unaligned sequences, like FASTA\n", cfg->seqfile);
qsq = esl_sq_CreateDigital(abc);
/* Open the results output files */
if (esl_opt_IsOn(go, "-o")) { if ((ofp = fopen(esl_opt_GetString(go, "-o"), "w")) == NULL) esl_fatal("Failed to open output file %s for writing\n", esl_opt_GetString(go, "-o")); }
if (esl_opt_IsOn(go, "--tblout")) { if ((tblfp = fopen(esl_opt_GetString(go, "--tblout"), "w")) == NULL) esl_fatal("Failed to open tabular per-seq output file %s for writing\n", esl_opt_GetString(go, "--tblfp")); }
if (esl_opt_IsOn(go, "--dfamtblout")) { if ((dfamtblfp = fopen(esl_opt_GetString(go, "--dfamtblout"),"w")) == NULL) esl_fatal("Failed to open tabular dfam output file %s for writing\n", esl_opt_GetString(go, "--dfamtblout")); }
if (esl_opt_IsOn(go, "--aliscoresout")) { if ((aliscoresfp = fopen(esl_opt_GetString(go, "--aliscoresout"),"w")) == NULL) esl_fatal("Failed to open alignment scores output file %s for writing\n", esl_opt_GetString(go, "--aliscoresout")); }
output_header(ofp, go, cfg->hmmfile, cfg->seqfile);
#ifdef HMMER_THREADS
/* initialize thread data */
if (esl_opt_IsOn(go, "--cpu")) ncpus = esl_opt_GetInteger(go, "--cpu");
else esl_threads_CPUCount(&ncpus);
if (ncpus > 0)
{
threadObj = esl_threads_Create(&pipeline_thread);
queue = esl_workqueue_Create(ncpus * 2);
}
#endif
infocnt = (ncpus == 0) ? 1 : ncpus;
ESL_ALLOC(info, sizeof(*info) * infocnt);
for (i = 0; i < infocnt; ++i)
{
info[i].bg = p7_bg_Create(abc);
#ifdef HMMER_THREADS
info[i].queue = queue;
#endif
}
#ifdef HMMER_THREADS
for (i = 0; i < ncpus * 2; ++i)
{
block = p7_oprofile_CreateBlock(BLOCK_SIZE);
if (block == NULL) esl_fatal("Failed to allocate sequence block");
status = esl_workqueue_Init(queue, block);
if (status != eslOK) esl_fatal("Failed to add block to work queue");
}
#endif
/* Outside loop: over each query sequence in <seqfile>. */
while ((sstatus = esl_sqio_Read(sqfp, qsq)) == eslOK)
{
if (sstatus == eslEMEM) p7_Fail("Memory allocation error reading sequence file\n", status);
if (sstatus == eslEINCONCEIVABLE) p7_Fail("Unexpected error %d reading sequence file\n", status);
// if (qsq->L > NHMMER_MAX_RESIDUE_COUNT) p7_Fail("Input sequence %s in file %s exceeds maximum length of %d bases.\n", qsq->name, cfg->seqfile, NHMMER_MAX_RESIDUE_COUNT);
nquery++;
esl_stopwatch_Start(w);
/* Open the target profile database */
status = p7_hmmfile_OpenE(cfg->hmmfile, p7_HMMDBENV, &hfp, NULL);
if (status != eslOK) p7_Fail("Unexpected error %d in opening hmm file %s.\n", status, cfg->hmmfile);
#ifdef HMMER_THREADS
/* if we are threaded, create a lock to prevent multiple readers */
if (ncpus > 0)
{
status = p7_hmmfile_CreateLock(hfp);
if (status != eslOK) p7_Fail("Unexpected error %d creating lock\n", status);
}
#endif
if (fprintf(ofp, "Query: %s [L=%ld]\n", qsq->name, (long) qsq->n) < 0) ESL_EXCEPTION_SYS(eslEWRITE, "write failed");
if (qsq->acc[0] != 0 && fprintf(ofp, "Accession: %s\n", qsq->acc) < 0) ESL_EXCEPTION_SYS(eslEWRITE, "write failed");
if (qsq->desc[0] != 0 && fprintf(ofp, "Description: %s\n", qsq->desc) < 0) ESL_EXCEPTION_SYS(eslEWRITE, "write failed");
for (i = 0; i < infocnt; ++i)
{
/* Create processing pipeline and hit list */
info[i].th = p7_tophits_Create();
info[i].pli = p7_pipeline_Create(go, 100, 100, TRUE, p7_SCAN_MODELS); /* M_hint = 100, L_hint = 100 are just dummies for now */
info[i].pli->hfp = hfp; /* for two-stage input, pipeline needs <hfp> */
p7_pli_NewSeq(info[i].pli, qsq);
info[i].qsq = qsq;
if ( esl_opt_IsUsed(go, "--toponly") )
info[i].pli->strand = p7_STRAND_TOPONLY;
else if ( esl_opt_IsUsed(go, "--bottomonly") )
info[i].pli->strand = p7_STRAND_BOTTOMONLY;
else
info[i].pli->strand = p7_STRAND_BOTH;
#ifdef HMMER_THREADS
if (ncpus > 0) esl_threads_AddThread(threadObj, &info[i]);
#endif
}
#ifdef HMMER_THREADS
if (ncpus > 0) hstatus = thread_loop(threadObj, queue, hfp);
else hstatus = serial_loop(info, hfp);
#else
hstatus = serial_loop(info, hfp);
#endif
switch(hstatus)
{
case eslEFORMAT: p7_Fail("bad file format in HMM file %s", cfg->hmmfile); break;
case eslEINCOMPAT: p7_Fail("HMM file %s contains different alphabets", cfg->hmmfile); break;
case eslEOF:
case eslOK: /* do nothing */
break;
default: p7_Fail("Unexpected error in reading HMMs from %s", cfg->hmmfile);
}
/* merge the results of the search results */
for (i = 1; i < infocnt; ++i)
{
p7_tophits_Merge(info[0].th, info[i].th);
p7_pipeline_Merge(info[0].pli, info[i].pli);
p7_pipeline_Destroy(info[i].pli);
p7_tophits_Destroy(info[i].th);
}
/* modify e-value to account for number of models */
for (i = 0; i < info->th->N ; i++)
{
info->th->unsrt[i].lnP += log((float)info->pli->nmodels);
info->th->unsrt[i].dcl[0].lnP = info->th->unsrt[i].lnP;
info->th->unsrt[i].sortkey = -1.0 * info->th->unsrt[i].lnP;
}
/* it's possible to have duplicates based on how viterbi ranges can overlap */
p7_tophits_SortByModelnameAndAlipos(info->th);
p7_tophits_RemoveDuplicates(info->th, info->pli->use_bit_cutoffs);
/* Print results */
p7_tophits_SortBySortkey(info->th);
p7_tophits_Threshold(info->th, info->pli);
//tally up total number of hits and target coverage
info->pli->n_output = info->pli->pos_output = 0;
for (i = 0; i < info->th->N; i++) {
if ( (info->th->hit[i]->flags & p7_IS_REPORTED) || info->th->hit[i]->flags & p7_IS_INCLUDED) {
info->pli->n_output++;
info->pli->pos_output += abs(info->th->hit[i]->dcl[0].jali - info->th->hit[i]->dcl[0].iali) + 1;
}
}
p7_tophits_Targets(ofp, info->th, info->pli, textw); if (fprintf(ofp, "\n\n") < 0) ESL_EXCEPTION_SYS(eslEWRITE, "write failed");
p7_tophits_Domains(ofp, info->th, info->pli, textw); if (fprintf(ofp, "\n\n") < 0) ESL_EXCEPTION_SYS(eslEWRITE, "write failed");
if (tblfp) p7_tophits_TabularTargets(tblfp, qsq->name, qsq->acc, info->th, info->pli, (nquery == 1));
if (dfamtblfp) p7_tophits_TabularXfam(dfamtblfp, qsq->name, NULL, info->th, info->pli);
if (aliscoresfp) p7_tophits_AliScores(aliscoresfp, qsq->name, info->th );
esl_stopwatch_Stop(w);
info->pli->nseqs = 1;
p7_pli_Statistics(ofp, info->pli, w);
if (fprintf(ofp, "//\n") < 0) ESL_EXCEPTION_SYS(eslEWRITE, "write failed");
fflush(ofp);
p7_hmmfile_Close(hfp);
p7_pipeline_Destroy(info->pli);
p7_tophits_Destroy(info->th);
esl_sq_Reuse(qsq);
}
if (sstatus == eslEFORMAT) esl_fatal("Parse failed (sequence file %s):\n%s\n",
sqfp->filename, esl_sqfile_GetErrorBuf(sqfp));
else if (sstatus != eslEOF) esl_fatal("Unexpected error %d reading sequence file %s",
sstatus, sqfp->filename);
/* Terminate outputs - any last words?
*/
if (tblfp) p7_tophits_TabularTail(tblfp, "hmmscan", p7_SCAN_MODELS, cfg->seqfile, cfg->hmmfile, go);
if (ofp) { if (fprintf(ofp, "[ok]\n") < 0) ESL_EXCEPTION_SYS(eslEWRITE, "write failed"); }
/* Cleanup - prepare for successful exit
*/
for (i = 0; i < infocnt; ++i)
p7_bg_Destroy(info[i].bg);
#ifdef HMMER_THREADS
if (ncpus > 0)
{
esl_workqueue_Reset(queue);
while (esl_workqueue_Remove(queue, (void **) &block) == eslOK)
p7_oprofile_DestroyBlock(block);
esl_workqueue_Destroy(queue);
esl_threads_Destroy(threadObj);
}
#endif
free(info);
esl_sq_Destroy(qsq);
esl_stopwatch_Destroy(w);
esl_alphabet_Destroy(abc);
esl_sqfile_Close(sqfp);
if (ofp != stdout) fclose(ofp);
if (tblfp) fclose(tblfp);
if (dfamtblfp) fclose(dfamtblfp);
if (aliscoresfp) fclose(aliscoresfp);
return eslOK;
ERROR:
if (ofp != stdout) fclose(ofp);
if (tblfp) fclose(tblfp);
if (dfamtblfp) fclose(dfamtblfp);
if (aliscoresfp) fclose(aliscoresfp);
return status;
}
/* seq_shuffling()
* SRE, Tue Jan 22 08:35:51 2008 [Market Street Cafe, Leesburg]
*
* Shuffling of input sequences.
*
* Fixed-length (L>0) vs. full-length (L=0) modes handled differently.
* In fixed-length mode:
* <shuff->seq> only needs to be allocated once, for L
* <targ> is an allocated copy of a random subseq of length L
* sequences < L residues long can't be shuffled
* In full-length mode:
* <shuff->seq> is grown to length <sq->n> for each input seq
* <targ> just points to <sq->seq>
*/
static int
seq_shuffling(ESL_GETOPTS *go, ESL_RANDOMNESS *r, FILE *ofp, int outfmt)
{
char *seqfile = esl_opt_GetArg(go, 1);
int infmt = eslSQFILE_UNKNOWN;
ESL_SQFILE *sqfp = NULL;
ESL_SQ *sq = esl_sq_Create();
ESL_SQ *shuff = esl_sq_Create();
char *targ = NULL;
int N = esl_opt_GetInteger(go, "-N");
int L = esl_opt_GetInteger(go, "-L"); /* L>0 means select random fixed-len subseqs */
int kmers = 0;
int i;
int status;
if (esl_opt_GetString(go, "--informat") != NULL) {
infmt = esl_sqio_EncodeFormat(esl_opt_GetString(go, "--informat"));
if (infmt == eslSQFILE_UNKNOWN) esl_fatal("%s is not a valid input sequence file format for --informat");
}
if (esl_opt_IsOn(go, "-k")) kmers = esl_opt_GetInteger(go, "-k");
status = esl_sqfile_Open(seqfile, infmt, NULL, &sqfp);
if (status == eslENOTFOUND) esl_fatal("No such file %s", seqfile);
else if (status == eslEFORMAT) esl_fatal("Format of seqfile %s unrecognized.", seqfile);
else if (status == eslEINVAL) esl_fatal("Can't autodetect stdin or .gz.");
else if (status != eslOK) esl_fatal("Open failed, code %d.", status);
if (L>0) {
esl_sq_GrowTo(shuff, L);
shuff->n = L;
ESL_ALLOC(targ, sizeof(char) * (L+1));
}
while ((status = esl_sqio_Read(sqfp, sq)) == eslOK)
{
if (L == 0) { /* shuffling entire sequence */
esl_sq_GrowTo(shuff, sq->n); /* make sure shuff can hold sq */
shuff->n = sq->n;
targ = sq->seq;
} else {
if (sq->n < L) continue; /* reject seqs < L long */
}
for (i = 0; i < N; i++)
{
if (L > 0) { /* fixed-len mode: copy a random subseq */
int pos = esl_rnd_Roll(r, sq->n - L + 1);
strncpy(targ, sq->seq + pos, L);
targ[L] = '\0';
}
/* Do the requested kind of shuffling */
if (esl_opt_GetBoolean(go, "-m")) esl_rsq_CShuffle (r, targ, shuff->seq); /* monoresidue shuffling */
else if (esl_opt_GetBoolean(go, "-d")) esl_rsq_CShuffleDP (r, targ, shuff->seq); /* diresidue shuffling */
else if (esl_opt_IsOn (go, "-k")) esl_rsq_CShuffleKmers(r, targ, kmers, shuff->seq); /* diresidue shuffling */
else if (esl_opt_GetBoolean(go, "-0")) esl_rsq_CMarkov0 (r, targ, shuff->seq); /* 0th order Markov */
else if (esl_opt_GetBoolean(go, "-1")) esl_rsq_CMarkov1 (r, targ, shuff->seq); /* 1st order Markov */
else if (esl_opt_GetBoolean(go, "-r")) esl_rsq_CReverse ( targ, shuff->seq); /* reverse */
else if (esl_opt_IsOn (go, "-w")) { /* regionally shuffle */
int W= esl_opt_GetInteger(go, "-w"); esl_rsq_CShuffleWindows(r, targ, W, shuff->seq);
}
/* Set the name of the shuffled sequence */
if (N > 1) esl_sq_FormatName(shuff, "%s-shuffled-%d", sq->name, i);
else esl_sq_FormatName(shuff, "%s-shuffled", sq->name);
/* Output the resulting sequence */
esl_sqio_Write(ofp, shuff, outfmt, FALSE);
/* don't need to reuse the shuffled sequence: we will use exactly the same memory */
}
esl_sq_Reuse(sq);
}
if (status == eslEFORMAT) esl_fatal("Parse failed (sequence file %s):\n%s\n",
sqfp->filename, esl_sqfile_GetErrorBuf(sqfp));
else if (status != eslEOF) esl_fatal("Unexpected error %d reading sequence file %s",
status, sqfp->filename);
if (L>0) free(targ);
esl_sq_Destroy(shuff);
esl_sq_Destroy(sq);
esl_sqfile_Close(sqfp);
return eslOK;
ERROR:
if (targ != NULL) free(targ);
esl_sq_Destroy(shuff);
esl_sq_Destroy(sq);
esl_sqfile_Close(sqfp);
return status;
}
SV* isaSeq(const char *input){
ESL_ALPHABET *abc = NULL;
ESL_SQ *sq = NULL;
int format = eslSQFILE_FASTA;
int alphatype = eslAMINO;
int status;
int pos;
int cnt = 0;
int size;
char *copyInput = NULL;
char *copyInput2 = NULL;
char *hmmpgmd = NULL;
char buf[61];
HV* hash = newHV();
int newlines;
size = sizeof(char) * ( strlen(input) + 1);
copyInput = (char *)malloc(size);
copyInput2 = (char *)malloc(size);
strncpy( copyInput, input, size);
hv_store(hash, "type", strlen("type"), newSVpv("UNK", 3), 0);
abc = esl_alphabet_Create(alphatype);
sq = esl_sq_CreateDigital(abc);
status = eslOK;
while(status == eslOK){
status = esl_sqio_Parse(copyInput, strlen(copyInput) * sizeof(char), sq, format);
if(status != eslOK){
hv_store(hash, "error", strlen("error"), newSVpvn("Error parsing input", 19), 0);
break;
}
cnt++;
newlines = (int)( sq->n / 60 ) + 1;
if(hmmpgmd != NULL){
hmmpgmd = (char *) realloc(hmmpgmd, sizeof(char) * (strlen(hmmpgmd) + sq->n + strlen(sq->name) + 5) + newlines );
}else{
hmmpgmd = (char *) malloc( sizeof(char) * ( sq->n + strlen(sq->name) + 5) + newlines );
hmmpgmd[0] = '\0';
}
strcat(hmmpgmd, ">");
strcat(hmmpgmd, sq->name);
strcat(hmmpgmd, "\n");
buf[60] = '\0';
for (pos = 0; pos < sq->n; pos += 60){
if (sq->dsq != NULL) esl_abc_TextizeN(sq->abc, sq->dsq+pos+1, 60, buf);
else strncpy(buf, sq->seq+pos, 60);
strcat(hmmpgmd, buf);
strcat(hmmpgmd, "\n");
}
if(sq->eoff > 0 ){
strcpy(copyInput2, copyInput+(sq->eoff + 2));
copyInput = copyInput2;
}
if(strlen(copyInput)==0){
break;
}
esl_sq_Reuse(sq);
}
if(hmmpgmd != NULL){
hv_store(hash, "hmmpgmd", strlen("hmmpgmd"), newSVpv(hmmpgmd, strlen(hmmpgmd)), 0);
hv_store(hash, "count", strlen("count"), newSViv(cnt), 0);
if(cnt > 1){
hv_store(hash, "type", strlen("type"), newSVpv("MS", 2), 0);
}else{
hv_store(hash, "type", strlen("type"), newSVpv("SS", 2), 0);
}
hv_store(hash, "alpha", strlen("alpha"), newSViv(sq->abc->type), 0);
}
if(hmmpgmd != NULL) free(hmmpgmd);
if(copyInput != NULL) free(copyInput);
if(sq != NULL) esl_sq_Destroy(sq);
if(abc != NULL ) esl_alphabet_Destroy(abc);
return newRV_noinc((SV*) hash);
}
int
main(int argc, char **argv)
{
ESL_GETOPTS *go = esl_getopts_CreateDefaultApp(options, 2, argc, argv, banner, usage);
char *hmmfile = esl_opt_GetArg(go, 1);
char *seqfile = esl_opt_GetArg(go, 2);
float nu = esl_opt_GetReal(go, "--nu");
ESL_ALPHABET *abc = NULL;
P7_HMMFILE *hfp = NULL;
P7_HMM *hmm = NULL;
P7_BG *bg = NULL;
P7_PROFILE *gm = NULL;
P7_GMX *fwd = NULL;
ESL_SQ *sq = NULL;
ESL_SQFILE *sqfp = NULL;
P7_TRACE *tr = NULL;
int format = eslSQFILE_UNKNOWN;
float sc, nullsc, seqscore, P;
int status;
/* Read in one HMM */
if (p7_hmmfile_Open(hmmfile, NULL, &hfp) != eslOK) p7_Fail("Failed to open HMM file %s", hmmfile);
if (p7_hmmfile_Read(hfp, &abc, &hmm) != eslOK) p7_Fail("Failed to read HMM");
p7_hmmfile_Close(hfp);
/* Open sequence file */
sq = esl_sq_CreateDigital(abc);
status = esl_sqfile_Open(seqfile, format, NULL, &sqfp);
if (status == eslENOTFOUND) p7_Fail("No such file.");
else if (status == eslEFORMAT) p7_Fail("Format unrecognized.");
else if (status == eslEINVAL) p7_Fail("Can't autodetect stdin or .gz.");
else if (status != eslOK) p7_Fail("Open failed, code %d.", status);
/* Configure a profile from the HMM */
bg = p7_bg_Create(abc);
gm = p7_profile_Create(hmm->M, abc);
p7_ProfileConfig(hmm, bg, gm, sq->n, p7_LOCAL);
/* Allocate matrix */
fwd = p7_gmx_Create(gm->M, sq->n);
while ((status = esl_sqio_Read(sqfp, sq)) == eslOK)
{
p7_ReconfigLength(gm, sq->n);
p7_bg_SetLength(bg, sq->n);
p7_gmx_GrowTo(fwd, gm->M, sq->n);
/* Run MSV */
p7_GMSV(sq->dsq, sq->n, gm, fwd, nu, &sc);
/* Calculate bit score and P-value using standard null1 model*/
p7_bg_NullOne (bg, sq->dsq, sq->n, &nullsc);
seqscore = (sc - nullsc) / eslCONST_LOG2;
P = esl_gumbel_surv(seqscore, gm->evparam[p7_MMU], gm->evparam[p7_MLAMBDA]);
/* output suitable for direct use in profmark benchmark postprocessors:
* <Pvalue> <bitscore> <target name> <query name>
*/
printf("%g\t%.2f\t%s\t%s\n", P, seqscore, sq->name, hmm->name);
esl_sq_Reuse(sq);
}
if (status == eslEFORMAT) esl_fatal("Parse failed (sequence file %s):\n%s\n", sqfp->filename, esl_sqfile_GetErrorBuf(sqfp));
else if (status != eslEOF) esl_fatal("Unexpected error %d reading sequence file %s", status, sqfp->filename);
/* Cleanup */
esl_sqfile_Close(sqfp);
esl_sq_Destroy(sq);
p7_trace_Destroy(tr);
p7_gmx_Destroy(fwd);
p7_profile_Destroy(gm);
p7_bg_Destroy(bg);
p7_hmm_Destroy(hmm);
esl_alphabet_Destroy(abc);
esl_getopts_Destroy(go);
return 0;
}
/* Function: p7_CoreEmit()
* Incept: SRE, Tue Jan 9 10:20:51 2007 [Janelia]
*
* Purpose: Generate (sample) a sequence from a core HMM <hmm>.
*
* Optionally return the sequence and/or its trace in <sq>
* and <tr>, respectively, which the caller has
* allocated. Having the caller provide these reusable
* objects allows re-use of both <sq> and <tr> in repeated
* calls, saving malloc/free wastage. Either can be passed
* as <NULL> if it isn't needed.
*
* This does not set any fields in the <sq> except for the
* sequence itself. Caller must set the name, and any other
* annotation it wants to add.
*
* Trace is relative to the core model: it may include
* I_0 and I_M states, B->DD->M entry is explicit, and a
* 0 length generated sequence is possible.
*
* Args: r - source of randomness
* hmm - core HMM to generate from
* sq - opt: digital sequence sampled (or NULL)
* tr - opt: trace sampled (or NULL)
*
* Returns: <eslOK> on success;
* optionally return the digital sequence through <ret_sq>,
* and optionally return its trace in <ret_tr>.
*
* Throws: <eslECORRUPT> if emission gets us into an illegal state,
* probably indicating that a probability that should have
* been zero wasn't.
*
* Throws <eslEMEM> on a reallocation error.
*
* In these cases, the contents of <sq> and <tr> may be
* corrupted. Caller should not trust their data, but may
* safely reuse them.
*
* Xref: STL11/124.
*/
int
p7_CoreEmit(ESL_RANDOMNESS *r, const P7_HMM *hmm, ESL_SQ *sq, P7_TRACE *tr)
{
int k = 0; /* position in model nodes 1..M */
int i = 0; /* position in sequence 1..L */
char st = p7T_B; /* state type */
int x; /* sampled residue */
int status;
if (sq != NULL) esl_sq_Reuse(sq);
if (tr != NULL) {
if ((status = p7_trace_Reuse(tr)) != eslOK) goto ERROR;
if ((status = p7_trace_Append(tr, st, k, i)) != eslOK) goto ERROR;
}
while (st != p7T_E)
{
/* Sample next state type, given current state type (and current k) */
switch (st) {
case p7T_B:
case p7T_M:
switch (esl_rnd_FChoose(r, hmm->t[k], 3)) {
case 0: st = p7T_M; break;
case 1: st = p7T_I; break;
case 2: st = p7T_D; break;
default: ESL_XEXCEPTION(eslEINCONCEIVABLE, "impossible.");
}
break;
case p7T_I:
switch (esl_rnd_FChoose(r, hmm->t[k]+3, 2)) {
case 0: st = p7T_M; break;
case 1: st = p7T_I; break;
default: ESL_XEXCEPTION(eslEINCONCEIVABLE, "impossible.");
}
break;
case p7T_D:
switch (esl_rnd_FChoose(r, hmm->t[k]+5, 2)) {
case 0: st = p7T_M; break;
case 1: st = p7T_D; break;
default: ESL_XEXCEPTION(eslEINCONCEIVABLE, "impossible.");
}
break;
default: ESL_XEXCEPTION(eslECORRUPT, "impossible state reached during emission");
}
/* Bump k,i if needed, depending on new state type */
if (st == p7T_M || st == p7T_D) k++;
if (st == p7T_M || st == p7T_I) i++;
/* a transit to M_M+1 is a transit to the E state */
if (k == hmm->M+1) {
if (st == p7T_M) { st = p7T_E; k = 0; }
else ESL_XEXCEPTION(eslECORRUPT, "failed to reach E state properly");
}
/* Sample new residue x if in match or insert */
if (st == p7T_M) x = esl_rnd_FChoose(r, hmm->mat[k], hmm->abc->K);
else if (st == p7T_I) x = esl_rnd_FChoose(r, hmm->ins[k], hmm->abc->K);
else x = eslDSQ_SENTINEL;
/* Add state to trace */
if (tr != NULL) {
if ((status = p7_trace_Append(tr, st, k, i)) != eslOK) goto ERROR;
}
/* Add x to sequence */
if (sq != NULL && x != eslDSQ_SENTINEL)
if ((status = esl_sq_XAddResidue(sq, x)) != eslOK) goto ERROR;
}
/* Terminate the trace and sequence (both are optional, remember) */
if (tr != NULL) { tr->M = hmm->M; tr->L = i; }
if (sq != NULL && (status = esl_sq_XAddResidue(sq, eslDSQ_SENTINEL)) != eslOK) goto ERROR;
return eslOK;
ERROR:
return status;
}
int
main(int argc, char **argv)
{
ESL_GETOPTS *go = NULL; /* application configuration */
char *seqfile = NULL; /* sequence file name */
char *maskfile = NULL; /* mask coordinate file name */
int infmt = eslSQFILE_UNKNOWN; /* format code for seqfile */
int outfmt = eslSQFILE_FASTA; /* format code for output seqs */
ESL_SQFILE *sqfp = NULL; /* open sequence file */
ESL_FILEPARSER *maskefp = NULL; /* open mask coord file */
FILE *ofp = NULL; /* output stream for masked seqs */
char *source = NULL; /* name of current seq to mask */
char *p1, *p2; /* pointers used in parsing */
int64_t start, end; /* start, end coord for masking */
int64_t i, j, pos; /* coords in a sequence */
int64_t overmask; /* # of extra residues to mask */
ESL_SQ *sq = esl_sq_Create(); /* current sequence */
int do_fetching;
int do_lowercase;
int maskchar;
int status; /* easel return code */
/****************************************************************************
* Parse command line
****************************************************************************/
go = esl_getopts_Create(options);
if (esl_opt_ProcessCmdline(go, argc, argv) != eslOK) cmdline_failure(argv[0], "Failed to parse command line: %s\n", go->errbuf);
if (esl_opt_VerifyConfig(go) != eslOK) cmdline_failure(argv[0], "Error in configuration: %s\n", go->errbuf);
if (esl_opt_GetBoolean(go, "-h") ) cmdline_help (argv[0], go);
if (esl_opt_ArgNumber(go) != 2) cmdline_failure(argv[0], "Incorrect number of command line arguments.\n");
do_fetching = esl_opt_GetBoolean(go, "-R");
do_lowercase = esl_opt_GetBoolean(go, "-l");
overmask = (esl_opt_IsOn(go, "-x") ? esl_opt_GetInteger(go, "-x") : 0);
maskchar = (esl_opt_IsOn(go, "-m") ? esl_opt_GetChar(go, "-m") : 'X');
seqfile = esl_opt_GetArg(go, 1);
maskfile = esl_opt_GetArg(go, 2);
/* Open the <seqfile>: text mode, not digital */
if (esl_opt_GetString(go, "--informat") != NULL) {
infmt = esl_sqio_EncodeFormat(esl_opt_GetString(go, "--informat"));
if (infmt == eslSQFILE_UNKNOWN) cmdline_failure(argv[0], "%s is not a valid input sequence file format for --informat");
}
status = esl_sqfile_Open(seqfile, infmt, NULL, &sqfp);
if (status == eslENOTFOUND) cmdline_failure(argv[0], "Sequence file %s not found.\n", seqfile);
else if (status == eslEFORMAT) cmdline_failure(argv[0], "Format of file %s unrecognized.\n", seqfile);
else if (status == eslEINVAL) cmdline_failure(argv[0], "Can't autodetect stdin or .gz.\n");
else if (status != eslOK) cmdline_failure(argv[0], "Open failed, code %d.\n", status);
if(do_fetching)
{
status = esl_sqfile_OpenSSI(sqfp, NULL);
if (status == eslEFORMAT) cmdline_failure(argv[0], "SSI index is in incorrect format\n");
else if (status == eslERANGE) cmdline_failure(argv[0], "SSI index is in 64-bit format and we can't read it\n");
else if (status != eslOK) cmdline_failure(argv[0], "Failed to open SSI index\n");
}
/* Open the <maskfile> */
if (esl_fileparser_Open(maskfile, NULL, &maskefp) != eslOK)
cmdline_failure(argv[0], "Failed to open mask coordinate file %s\n", maskfile);
esl_fileparser_SetCommentChar(maskefp, '#');
/* Open the output file, if any */
if (esl_opt_GetString(go, "-o") != NULL)
{
if ((ofp = fopen(esl_opt_GetString(go, "-o"), "w")) == NULL)
cmdline_failure(argv[0], "Failed to open output file %s\n", esl_opt_GetString(go, "-o"));
}
else ofp = stdout;
/****************************************************************************
* Main loop over lines in <maskfile>
****************************************************************************/
/* Read one data line at a time from the <maskfile>;
* parse into data fields <seqname> <start> <end>
*/
while (esl_fileparser_NextLine(maskefp) == eslOK)
{
/* First field is sequence name */
if (esl_fileparser_GetTokenOnLine(maskefp, &source, NULL) != eslOK)
esl_fatal("Failed to read source seq name on line %d of file %s\n", maskefp->linenumber, maskfile);
/* Get the sequence */
if (do_fetching)
{ /* If the <seqfile> is SSI indexed, try to reposition it and read <source> seq by random access */
status = esl_sqio_Fetch(sqfp, source, sq);
if (status == eslENOTFOUND) esl_fatal("seq %s not found in SSI index for file %s\n", source, sqfp->filename);
else if (status == eslEINVAL) esl_fatal("No SSI index or can't reposition in file %s\n", sqfp->filename);
else if (status == eslEFORMAT) esl_fatal("Parse failed:\n%s\n", esl_sqfile_GetErrorBuf(sqfp));
else if (status != eslOK) esl_fatal("Unexpected failure in fetching %s from file %s\n", source, sqfp->filename);
}
else
{ /* else, assume we're reading sequentially; <sqfile> and <maskfile> have seqs in same order */
status = esl_sqio_Read(sqfp, sq);
if (status == eslEOF) esl_fatal("File %s ended prematurely; didn't find %s\n", sqfp->filename, source);
else if (status == eslEFORMAT) esl_fatal("Parse failed:\n%s\n", esl_sqfile_GetErrorBuf(sqfp));
else if (status != eslOK) esl_fatal("Unexpected error reading sequence file %s\n", sqfp->filename);
if ((strcmp(sq->name, source) != 0) && (strcmp(sq->acc, source) != 0))
esl_fatal("Sequences in <sqfile> and <maskfile> aren't in same order; try -R");
}
/* If we're masking by lowercase, first make sure everything's uppercase */
if (do_lowercase)
for (pos = 0; pos < sq->n; pos++)
if (isalpha(sq->seq[pos]))
sq->seq[pos] = toupper(sq->seq[pos]);
/* Next two fields are <start>, <end> for the masking */
/* possible future extension: wrap loop around this, enable multiple masked regions */
if (esl_fileparser_GetTokenOnLine(maskefp, &p1, NULL) != eslOK)
esl_fatal("Failed to read start coord on line %d of file %s\n", maskefp->linenumber, maskfile);
start = strtoll(p1, &p2, 0) - 1;
if (esl_fileparser_GetTokenOnLine(maskefp, &p2, NULL) != eslOK)
esl_fatal("Failed to read end coord on line %d of file %s\n", maskefp->linenumber, maskfile);
end = strtoll(p2, &p1, 0) - 1;
/* Do the masking */
if (esl_opt_GetBoolean(go, "-r")) /* Reverse masking */
{ /* leave start..end unmasked; mask prefix 0..start-1, end+1..L-1 */
i = 0;
j = ESL_MIN(sq->n-1, start - 1 + overmask);
for (pos = i; pos <= j; pos++)
if (isalpha(sq->seq[pos]))
sq->seq[pos] = (do_lowercase ? tolower(sq->seq[pos]) : maskchar);
i = ESL_MAX(0, end + 1 - overmask);
j = sq->n-1;
for (pos = i; pos <= j; pos++)
if (isalpha(sq->seq[pos]))
sq->seq[pos] = (do_lowercase ? tolower(sq->seq[pos]) : maskchar);
}
else
{ /* normal: mask start..end */
i = ESL_MAX(0, start - overmask);
j = ESL_MIN(sq->n-1, end + overmask);
for (pos = i; pos <= j; pos++)
if (isalpha(sq->seq[pos]))
sq->seq[pos] = (do_lowercase ? tolower(sq->seq[pos]) : maskchar);
}
esl_sqio_Write(ofp, sq, outfmt, FALSE);
esl_sq_Reuse(sq);
}
esl_sq_Destroy(sq);
esl_fileparser_Close(maskefp);
esl_sqfile_Close(sqfp);
esl_getopts_Destroy(go);
if (ofp != stdout) fclose(ofp);
return 0;
}
int
main(int argc, char **argv)
{
ESL_GETOPTS *go = p7_CreateDefaultApp(options, 2, argc, argv, banner, usage);
ESL_ALPHABET *abc = esl_alphabet_Create(eslAMINO);
char *qfile = esl_opt_GetArg(go, 1);
char *tfile = esl_opt_GetArg(go, 2);
ESL_SQFILE *qfp = NULL;
ESL_SQFILE *tfp = NULL;
ESL_SQ *qsq = esl_sq_CreateDigital(abc);
ESL_SQ *tsq = esl_sq_CreateDigital(abc);
ESL_SCOREMATRIX *S = esl_scorematrix_Create(abc);
ESL_DMATRIX *Q = NULL;
P7_BG *bg = p7_bg_Create(abc);
P7_HMM *hmm = NULL;
P7_PROFILE *gm = NULL;
P7_REFMX *vit = p7_refmx_Create(200, 400); /* will grow as needed */
double *fa = malloc(sizeof(double) * abc->K);
double popen = 0.02;
double pextend = 0.4;
double lambda;
float vsc;
float nullsc;
int status;
esl_composition_BL62(fa);
esl_vec_D2F(fa, abc->K, bg->f);
esl_scorematrix_Set("BLOSUM62", S);
esl_scorematrix_ProbifyGivenBG(S, fa, fa, &lambda, &Q);
esl_scorematrix_JointToConditionalOnQuery(abc, Q);
if (esl_sqfile_OpenDigital(abc, qfile, eslSQFILE_UNKNOWN, NULL, &qfp) != eslOK) esl_fatal("failed to open %s", qfile);
if (esl_sqio_Read(qfp, qsq) != eslOK) esl_fatal("failed to read query seq");
p7_Seqmodel(abc, qsq->dsq, qsq->n, qsq->name, Q, bg->f, popen, pextend, &hmm);
p7_hmm_SetComposition(hmm);
p7_hmm_SetConsensus(hmm, qsq);
gm = p7_profile_Create(hmm->M, abc);
p7_profile_ConfigUnilocal(gm, hmm, bg, 400);
if (esl_sqfile_OpenDigital(abc, tfile, eslSQFILE_UNKNOWN, NULL, &tfp) != eslOK) esl_fatal("failed to open %s", tfile);
while ((status = esl_sqio_Read(tfp, tsq)) == eslOK)
{
p7_bg_SetLength (bg, tsq->n);
p7_profile_SetLength(gm, tsq->n);
p7_ReferenceViterbi(tsq->dsq, tsq->n, gm, vit, NULL, &vsc);
p7_bg_NullOne(bg, tsq->dsq, tsq->n, &nullsc);
printf("%.4f %-25s %-25s\n", (vsc - nullsc) / eslCONST_LOG2, tsq->name, gm->name);
esl_sq_Reuse(tsq);
p7_refmx_Reuse(vit);
}
p7_refmx_Destroy(vit);
p7_profile_Destroy(gm);
p7_hmm_Destroy(hmm);
p7_bg_Destroy(bg);
esl_dmatrix_Destroy(Q);
esl_scorematrix_Destroy(S);
free(fa);
esl_sq_Destroy(qsq);
esl_sq_Destroy(tsq);
esl_sqfile_Close(qfp);
esl_sqfile_Close(tfp);
esl_alphabet_Destroy(abc);
esl_getopts_Destroy(go);
return 0;
}
/* Function: p7_ProfileEmit()
* Synopsis: Sample a sequence from the search form of the model.
* Incept: SRE, Mon Jan 22 10:23:28 2007 [Janelia]
*
* Purpose: Sample a sequence from the implicit
* probabilistic model of a Plan7 profile <gm>. This
* requires also having the core probabilities of
* the accompanying <hmm>, and the background
* frequencies of null1 model <bg>.
*
* Optionally return the sequence and/or its trace in <sq>
* and <tr>, respectively. Caller has allocated space for
* both of these, though they may get reallocated/grown
* here. Either can be passed as <NULL> if unneeded.
*
* Only the sequence field is set in the <sq>. Caller must
* set the name, plus any other fields it wants to set. If
* the <sq> was created in digital mode, this is the <sq->dsq>;
* if the <sq> was created in text mode, this is <sq->seq>.
*
* <p7_ProfileEmit()> deliberately uses an <ESL_SQ> object
* instead of a plain <ESL_DSQ *> or <char *> string, to
* take advantage of the object's support for dynamic
* reallocation of seq length, and to allow both digital and
* text mode generation.
*
* Args: r - source of randomness
* hmm - core probabilities of the profile
* gm - configured search profile
* sq - optRETURN: sampled sequence
* tr - optRETURN: sampled trace
*
* Throws: (no abnormal error conditions)
*/
int
p7_ProfileEmit(ESL_RANDOMNESS *r, const P7_HMM *hmm, const P7_PROFILE *gm, const P7_BG *bg, ESL_SQ *sq, P7_TRACE *tr)
{
char prv, st; /* prev, current state type */
int k = 0; /* position in model nodes 1..M */
int i = 0; /* position in sequence 1..L */
int x; /* sampled residue */
int kend = hmm->M; /* predestined end node */
int status;
float xt[p7P_NXSTATES][p7P_NXTRANS];
/* Backcalculate the probabilities in the special states (loop and length model) */
for (i = 0; i < p7P_NXSTATES; i++)
for (x = 0; x < p7P_NXTRANS; x++)
xt[i][x] = exp(gm->xsc[i][x]);
if (sq != NULL) esl_sq_Reuse(sq);
if (tr != NULL) {
if ((status = p7_trace_Reuse(tr)) != eslOK) goto ERROR;
if ((status = p7_trace_Append(tr, p7T_S, k, i)) != eslOK) goto ERROR;
if ((status = p7_trace_Append(tr, p7T_N, k, i)) != eslOK) goto ERROR;
}
st = p7T_N;
i = 0;
while (st != p7T_T)
{
/* Sample a state transition. After this section, prv and st (prev->current state) are set;
* k also gets set if we make a B->Mk entry transition.
*/
prv = st;
switch (st) {
case p7T_B:
if (p7_profile_IsLocal(gm))
{ /* local mode: enter the implicit profile: choose our entry and our predestined exit */
if ((status = sample_endpoints(r, gm, &k, &kend)) != eslOK) goto ERROR;
st = p7T_M; /* must be, because left wing is retracted */
}
else
{ /* glocal mode: treat B as M_0, use its transitions to MID. */
/* FIXME: this is wrong. It should sample from B->Mk distribution! */
switch (esl_rnd_FChoose(r, P7H_TMAT(hmm, 0), p7H_NTMAT)) {
case 0: st = p7T_M; k = 1; break;
case 1: st = p7T_I; k = 0; break;
case 2: st = p7T_D; k = 1; break;
default: ESL_XEXCEPTION(eslEINCONCEIVABLE, "impossible.");
}
}
break;
case p7T_M:
if (k == kend) st = p7T_E; /* check our preordained fate */
else {
switch (esl_rnd_FChoose(r, P7H_TMAT(hmm, k), p7H_NTMAT)) {
case 0: st = p7T_M; break;
case 1: st = p7T_I; break;
case 2: st = p7T_D; break;
default: ESL_XEXCEPTION(eslEINCONCEIVABLE, "impossible.");
}
}
break;
case p7T_D:
if (k == kend) st = p7T_E;
else st = (esl_rnd_FChoose(r, P7H_TDEL(hmm, k), p7H_NTDEL) == 0) ? p7T_M : p7T_D;
break;
case p7T_I: st = (esl_rnd_FChoose(r, P7H_TINS(hmm, k), p7H_NTINS) == 0) ? p7T_M : p7T_I; break;
case p7T_N: st = (esl_rnd_FChoose(r, xt[p7P_N], p7P_NXTRANS) == p7P_MOVE) ? p7T_B : p7T_N; break;
case p7T_E: st = (esl_rnd_FChoose(r, xt[p7P_E], p7P_NXTRANS) == p7P_MOVE) ? p7T_C : p7T_J; break;
case p7T_C: st = (esl_rnd_FChoose(r, xt[p7P_C], p7P_NXTRANS) == p7P_MOVE) ? p7T_T : p7T_C; break;
case p7T_J: st = (esl_rnd_FChoose(r, xt[p7P_J], p7P_NXTRANS) == p7P_MOVE) ? p7T_B : p7T_J; break;
default: ESL_XEXCEPTION(eslECORRUPT, "impossible state reached during emission");
}
/* Based on the transition we just sampled, update k. */
if (st == p7T_E) k = 0;
else if (st == p7T_M && prv != p7T_B) k++; /* be careful about B->Mk, where we already set k */
else if (st == p7T_D) k++;
/* Based on the transition we just sampled, generate a residue. */
if (st == p7T_M) x = esl_rnd_FChoose(r, hmm->mat[k], hmm->abc->K);
else if (st == p7T_I) x = esl_rnd_FChoose(r, hmm->ins[k], hmm->abc->K);
else if ((st == p7T_N || st == p7T_C || st == p7T_J) && prv==st) x = esl_rnd_FChoose(r, bg->f, hmm->abc->K);
else x = eslDSQ_SENTINEL;
if (x != eslDSQ_SENTINEL) i++;
/* Add residue (if any) to sequence */
if (sq != NULL && x != eslDSQ_SENTINEL && (status = esl_sq_XAddResidue(sq, x)) != eslOK) goto ERROR;
/* Add state to trace. */
if (tr != NULL) {
if ((status = p7_trace_Append(tr, st, k, i)) != eslOK) goto ERROR;
}
}
/* Terminate the trace and sequence (both are optional, remember) */
if (tr != NULL) { tr->M = hmm->M; tr->L = i; }
if (sq != NULL && (status = esl_sq_XAddResidue(sq, eslDSQ_SENTINEL)) != eslOK) goto ERROR;
return eslOK;
ERROR:
return status;
}
/* "generation" test
* Compare a randomly sampled profile to sequences sampled
* from that profile.
*
* This test is not very stringent, because we don't know the "true"
* envelopes. Rather, this is more of a test that nothing obviously
* bad happens, like a crash, or obviously incorrect data.
*
* We test:
* 1. Seq coordinates of each envelope are coherent:
* 1 <= oa <= ia <= i0 <= ib <= ob <= L
*
* 2. Envelopes do not overlap (assuming default threshold of
* 0.5 when defining them):
* ia(d) > ib(d-1) for d = 2..D
* (Outer envelopes, in contrast, can overlap.)
*
* 3. envsc(d) <= asc_sc <= fwdsc.
*
* 4. If D=1 (single domain) in both the generated trace
* and the inferred envelopes, and the domain coords in
* the trace are encompassed by the outer envelope,
* then envsc(d) >= generated trace score.
*/
static void
utest_generation(ESL_RANDOMNESS *rng, int M, const ESL_ALPHABET *abc, int N)
{
char msg[] = "reference_envelopes:: generation unit test failed";
ESL_SQ *sq = esl_sq_CreateDigital(abc);
P7_BG *bg = p7_bg_Create(abc);
P7_HMM *hmm = NULL;
P7_PROFILE *gm = p7_profile_Create(M, abc);
P7_TRACE *gtr = p7_trace_Create(); // generated trace
P7_TRACE *vtr = p7_trace_Create(); // Viterbi trace
P7_REFMX *rxf = p7_refmx_Create(M, 20); // Fwd, Vit ~~> ASC Decode UP
P7_REFMX *rxd = p7_refmx_Create(M, 20); // Bck, Decode ~~> ASC Decode DOWN
P7_REFMX *afu = p7_refmx_Create(M, 20); // ASC Fwd UP
P7_REFMX *afd = p7_refmx_Create(M, 20); // ASC Fwd DOWN
P7_REFMX *apu = rxf; // for 'clarity' we use two names for this mx
P7_REFMX *apd = rxd; // ... and this one too.
float *wrk = NULL;
P7_ANCHORS *anch = p7_anchors_Create();
P7_ANCHORHASH *ah = p7_anchorhash_Create();
P7_ENVELOPES *env = p7_envelopes_Create();
float tol = 0.001;
float gsc, fsc, asc;
int idx;
int d;
if ( p7_modelsample(rng, M, abc, &hmm) != eslOK) esl_fatal(msg);
if ( p7_profile_Config(gm, hmm, bg) != eslOK) esl_fatal(msg);
for (idx = 0; idx < N; idx++)
{
/* Emit sequence from model, using an arbitrary length model of <M>;
* restrict the emitted sequence length to 6M, arbitrarily, to
* keep it down to something reasonable.
*/
if ( p7_profile_SetLength(gm, M) != eslOK) esl_fatal(msg);
do {
esl_sq_Reuse(sq);
if (p7_ProfileEmit(rng, hmm, gm, bg, sq, gtr) != eslOK) esl_fatal(msg);
} while (sq->n > M * 6);
if (p7_trace_Index (gtr) != eslOK) esl_fatal(msg);
if (p7_trace_Score (gtr, sq->dsq, gm, &gsc) != eslOK) esl_fatal(msg);
/* Reset the length model to the actual length sq->n, then
* put it through the domain postprocessing analysis pipeline
*/
if ( p7_profile_SetLength(gm, sq->n) != eslOK) esl_fatal(msg);
/* First pass analysis */
if ( p7_ReferenceViterbi (sq->dsq, sq->n, gm, rxf, vtr, NULL) != eslOK) esl_fatal(msg);
if ( p7_ReferenceForward (sq->dsq, sq->n, gm, rxf, &fsc) != eslOK) esl_fatal(msg);
if ( p7_ReferenceBackward(sq->dsq, sq->n, gm, rxd, NULL) != eslOK) esl_fatal(msg);
if ( p7_ReferenceDecoding(sq->dsq, sq->n, gm, rxf, rxd, rxd) != eslOK) esl_fatal(msg);
/* Anchor determination (MPAS algorithm) */
if ( p7_reference_Anchors(rng, sq->dsq, sq->n, gm, rxf, rxd, vtr, &wrk, ah,
afu, afd, anch, &asc, NULL, NULL) != eslOK) esl_fatal(msg);
/* Reuse rxf,rxd as apu, apd; finish ASC analysis with Backward, Decoding */
p7_refmx_Reuse(apu); p7_refmx_Reuse(apd);
if ( p7_ReferenceASCBackward(sq->dsq, sq->n, gm, anch->a, anch->D, apu, apd, NULL) != eslOK) esl_fatal(msg);
if ( p7_ReferenceASCDecoding(sq->dsq, sq->n, gm, anch->a, anch->D, afu, afd, apu, apd, apu, apd) != eslOK) esl_fatal(msg);
/* Envelope calculation */
if ( p7_reference_Envelopes(sq->dsq, sq->n, gm, anch->a, anch->D, apu, apd, afu, afd, env) != eslOK) esl_fatal(msg);
/* Test 1. Coords of each domain are coherent */
if (anch->D != env->D) esl_fatal(msg);
for (d = 1; d <= anch->D; d++)
if (! (1 <= env->arr[d].oa &&
env->arr[d].oa <= env->arr[d].ia &&
env->arr[d].ia <= env->arr[d].i0 &&
env->arr[d].i0 <= env->arr[d].ib &&
env->arr[d].ib <= env->arr[d].ob &&
env->arr[d].ob <= sq->n)) esl_fatal(msg);
/* Test 2. Envelopes do not overlap. */
for (d = 1; d <= anch->D; d++)
if (! (env->arr[d].ia > env->arr[d-1].ib)) esl_fatal(msg);
/* Test 3. envsc(d) <= asc_sc <= fwdsc */
for (d = 1; d <= anch->D; d++)
if (! (env->arr[d].env_sc <= asc+tol && asc <= fsc+tol)) esl_fatal(msg);
/* Test 4, only on D=1 case with generated trace's domain
* encompassed by the outer envelope
*/
if (gtr->ndom == 1 && anch->D == 1 &&
gtr->sqfrom[0] >= env->arr[1].oa && // in <gtr>, domains are 0..D-1; in <env>, 1..D
gtr->sqto[0] <= env->arr[1].ob)
if (! ( env->arr[1].env_sc >= gsc)) esl_fatal(msg);
p7_envelopes_Reuse(env);
p7_anchors_Reuse(anch);
p7_anchorhash_Reuse(ah);
p7_refmx_Reuse(rxf); p7_refmx_Reuse(rxd);
p7_refmx_Reuse(afu); p7_refmx_Reuse(afd);
p7_trace_Reuse(gtr); p7_trace_Reuse(vtr);
esl_sq_Reuse(sq);
}
if (wrk) free(wrk);
p7_envelopes_Destroy(env);
p7_anchors_Destroy(anch);
p7_anchorhash_Destroy(ah);
p7_refmx_Destroy(afu); p7_refmx_Destroy(afd);
p7_refmx_Destroy(rxf); p7_refmx_Destroy(rxd);
p7_trace_Destroy(vtr); p7_trace_Destroy(gtr);
p7_profile_Destroy(gm);
p7_hmm_Destroy(hmm);
p7_bg_Destroy(bg);
esl_sq_Destroy(sq);
}
int
main(int argc, char **argv)
{
ESL_GETOPTS *go = p7_CreateDefaultApp(options, 2, argc, argv, banner, usage);
ESL_RANDOMNESS *rng = esl_randomness_CreateFast(esl_opt_GetInteger(go, "-s"));
ESL_ALPHABET *abc = NULL;
char *ghmmfile = esl_opt_GetArg(go, 1); /* HMMs parameterized for sequence generation */
char *ahmmfile = esl_opt_GetArg(go, 2); /* HMMs parameterized for alignment */
int N = esl_opt_GetInteger(go, "-N");
P7_HMMFILE *ghfp = NULL;
P7_HMMFILE *ahfp = NULL;
P7_HMM *ghmm = NULL;
P7_HMM *ahmm = NULL;
P7_PROFILE *ggm = NULL;
P7_PROFILE *agm = NULL;
P7_OPROFILE *aom = NULL;
P7_BG *bg = NULL;
ESL_SQ *sq = NULL;
P7_TRACE *reftr = p7_trace_Create();
P7_TRACE *testtr = p7_trace_Create();
P7_TRACE_METRICS *tmetrics = p7_trace_metrics_Create();
P7_REFMX *rmx = p7_refmx_Create(100,100);
// P7_FILTERMX *ox = NULL;
P7_HARDWARE *hw;
if ((hw = p7_hardware_Create ()) == NULL) p7_Fail("Couldn't get HW information data structure");
P7_SPARSEMASK *sm = p7_sparsemask_Create(100, 100, hw->simd);
P7_SPARSEMX *sxv = p7_sparsemx_Create(NULL);
int idx;
char errbuf[eslERRBUFSIZE];
int status;
p7_Init();
/* open HMM file containing models parameterized for generation (sampling) of seqs */
status = p7_hmmfile_OpenE(ghmmfile, NULL, &ghfp, errbuf);
if (status == eslENOTFOUND) p7_Fail("File existence/permissions problem in trying to open HMM file %s.\n%s\n", ghmmfile, errbuf);
else if (status == eslEFORMAT) p7_Fail("File format problem in trying to open HMM file %s.\n%s\n", ghmmfile, errbuf);
else if (status != eslOK) p7_Fail("Unexpected error %d in opening HMM file %s.\n%s\n", status, ghmmfile, errbuf);
/* open HMM file containing models parameterized for alignment (may be the same as ghmmfile) */
status = p7_hmmfile_OpenE(ahmmfile, NULL, &ahfp, errbuf);
if (status == eslENOTFOUND) p7_Fail("File existence/permissions problem in trying to open HMM file %s.\n%s\n", ahmmfile, errbuf);
else if (status == eslEFORMAT) p7_Fail("File format problem in trying to open HMM file %s.\n%s\n", ahmmfile, errbuf);
else if (status != eslOK) p7_Fail("Unexpected error %d in opening HMM file %s.\n%s\n", status, ahmmfile, errbuf);
while ( (status = p7_hmmfile_Read(ghfp, &abc, &ghmm)) == eslOK) /* <abc> gets set on first read */
{
/* read the counterpart HMM from <ahfp> */
status = p7_hmmfile_Read(ahfp, &abc, &ahmm);
if (status == eslEFORMAT) p7_Fail("Bad file format in HMM file %s:\n%s\n", ahfp->fname, ahfp->errbuf);
else if (status == eslEINCOMPAT) p7_Fail("HMM in %s is not in the expected %s alphabet\n", ahfp->fname, esl_abc_DecodeType(abc->type));
else if (status == eslEOF) p7_Fail("Empty HMM file %s? No HMM data found.\n", ahfp->fname);
else if (status != eslOK) p7_Fail("Unexpected error in reading HMMs from %s\n", ahfp->fname);
/* try to validate that they're the "same" */
if (ahmm->M != ghmm->M || strcmp(ahmm->name, ghmm->name) != 0) p7_Fail("<gen-hmmfile>, <ali-hmmfile> contain different set or order of models");
/* deferred one-time creation of structures that need to know the alphabet */
if (!bg) bg = p7_bg_Create(abc);
if (!sq) sq = esl_sq_CreateDigital(abc);
ggm = p7_profile_Create(ghmm->M, abc);
agm = p7_profile_Create(ahmm->M, abc);
aom = p7_oprofile_Create(ahmm->M, abc, hw->simd);
p7_profile_ConfigCustom(ggm, ghmm, bg, esl_opt_GetInteger(go, "--gL"), esl_opt_GetReal(go, "--gnj"), esl_opt_GetReal(go, "--gpglocal"));
p7_profile_ConfigCustom(agm, ahmm, bg, 100, esl_opt_GetReal(go, "--anj"), esl_opt_GetReal(go, "--apglocal"));
p7_oprofile_Convert(agm, aom);
for (idx = 1; idx <= N; idx++)
{
p7_ProfileEmit(rng, ghmm, ggm, bg, sq, reftr);
if (esl_opt_GetBoolean(go, "--dumpseqs")) {
esl_sq_FormatName(sq, "seq%d", idx);
esl_sqio_Write(stdout, sq, eslSQFILE_FASTA, FALSE);
}
p7_bg_SetLength(bg, sq->n);
p7_profile_SetLength(agm, sq->n);
p7_sparsemask_Reinit(sm, agm->M, sq->n);
p7_sparsemask_AddAll(sm);
if (esl_opt_GetBoolean(go, "--vit")) p7_ReferenceViterbi(sq->dsq, sq->n, agm, rmx, testtr, /*opt_vsc=*/NULL);
else p7_SparseViterbi (sq->dsq, sq->n, agm, sm, sxv, testtr, /*opt_vsc=*/NULL);
p7_trace_metrics(reftr, testtr, tmetrics);
p7_sparsemask_Reuse(sm);
p7_sparsemx_Reuse(sxv);
//p7_filtermx_Reuse(ox);
p7_refmx_Reuse(rmx);
esl_sq_Reuse(sq);
p7_trace_Reuse(reftr);
p7_trace_Reuse(testtr);
}
p7_oprofile_Destroy(aom);
p7_profile_Destroy(ggm);
p7_profile_Destroy(agm);
p7_hmm_Destroy(ghmm);
p7_hmm_Destroy(ahmm);
}
/* we leave the loop with <status> set by a p7_hmmfile_Read() on ghfp; if all is well, status=eslEOF */
if (status == eslEFORMAT) p7_Fail("Bad file format in HMM file %s:\n%s\n", ghfp->fname, ghfp->errbuf);
else if (status == eslEINCOMPAT) p7_Fail("HMM in %s is not in the expected %s alphabet\n", ghfp->fname, esl_abc_DecodeType(abc->type));
else if (status != eslEOF) p7_Fail("Unexpected error in reading HMMs from %s\n", ghfp->fname);
p7_trace_metrics_Dump(stdout, tmetrics);
p7_hmmfile_Close(ghfp);
p7_hmmfile_Close(ahfp);
// p7_filtermx_Destroy(ox);
p7_sparsemask_Destroy(sm);
p7_sparsemx_Destroy(sxv);
p7_refmx_Destroy(rmx);
p7_trace_metrics_Destroy(tmetrics);
p7_trace_Destroy(testtr);
p7_trace_Destroy(reftr);
p7_bg_Destroy(bg);
esl_alphabet_Destroy(abc);
esl_randomness_Destroy(rng);
esl_getopts_Destroy(go);
}
int
p7_seqcache_Open(char *seqfile, P7_SEQCACHE **ret_cache, char *errbuf)
{
int i;
int inx;
int val;
int status;
int32_t seq_cnt;
int32_t db_cnt;
int32_t db_inx[32];
uint32_t db_key;
uint64_t res_cnt;
uint64_t res_size;
uint64_t hdr_size;
char *hdr_ptr;
char *res_ptr;
char *desc_ptr;
char *ptr;
char buffer[512];
off_t offset;
uint64_t total_mem;
SEQ_DB *db = NULL;
P7_SEQCACHE *cache = NULL;
ESL_RANDOMNESS *rnd = NULL;
ESL_SQFILE *sqfp = NULL;
ESL_SQ *sq = NULL;
ESL_ALPHABET *abc = NULL;
ESL_SQASCII_DATA *ascii = NULL;
if (errbuf) errbuf[0] = '\0'; /* CURRENTLY UNUSED. FIXME */
/* Open the target sequence database */
if ((status = esl_sqfile_Open(seqfile, eslSQFILE_FASTA, NULL, &sqfp)) != eslOK) return status;
/* This is a bit of a hack. The first line contains database information.
*
* #<res_count> <seq_count> <db_count> <db_sequences_1> <db_sequences_before_removing_duplicates_1> <db_sequences_2> <db_sequences_before_removing_duplicates_2> ... <date_stamp>
*
* The rest of the file is a fasta format. The fasta header is just
* sequence number followed by a binary number indicating which
* database this sequence occurs in.
*
* The header line will be read in, parsed and saved. Then the
* parser will be repositioned after the line and used normally.
*/
ascii = &sqfp->data.ascii;
fseek(ascii->fp, 0L, SEEK_SET);
if (fgets(buffer, sizeof(buffer), ascii->fp) == NULL) return eslEFORMAT;
if (buffer[0] != '#') return eslEFORMAT;
ptr = buffer + 1;
res_cnt = strtoll(ptr, &ptr, 10);
seq_cnt = strtol(ptr, &ptr, 10);
db_cnt = strtol(ptr, &ptr, 10);
if (db_cnt > (sizeof(db_inx)/sizeof(db_inx[0]))) return eslEFORMAT;
total_mem = sizeof(P7_SEQCACHE);
ESL_ALLOC(cache, sizeof(P7_SEQCACHE));
memset(cache, 0, sizeof(P7_SEQCACHE));
if (esl_strdup(seqfile, -1, &cache->name) != eslOK) goto ERROR;
total_mem += (sizeof(HMMER_SEQ) * seq_cnt);
ESL_ALLOC(cache->list, sizeof(HMMER_SEQ) * seq_cnt);
memset(cache->list, 0, sizeof(HMMER_SEQ) * seq_cnt);
total_mem += (sizeof(SEQ_DB) * db_cnt);
ESL_ALLOC(db, sizeof(SEQ_DB) * db_cnt);
for (i = 0; i < db_cnt; ++i) {
db[i].count = strtol(ptr, &ptr, 10);
db[i].K = strtol(ptr, &ptr, 10);
total_mem += (sizeof(HMMER_SEQ *) * db[i].count);
ESL_ALLOC(db[i].list, sizeof(HMMER_SEQ *) * db[i].count);
memset(db[i].list, 0, sizeof(HMMER_SEQ *) * db[i].count);
}
/* grab the unique identifier */
while (*ptr && isspace(*ptr)) ++ptr;
i = strlen(ptr);
ESL_ALLOC(cache->id, i+1);
strcpy(cache->id, ptr);
while (--i > 0 && isspace(cache->id[i])) cache->id[i] = 0;
res_size = res_cnt + seq_cnt + 1;
hdr_size = seq_cnt * 10;
total_mem += res_size + hdr_size;
ESL_ALLOC(cache->residue_mem, res_size);
ESL_ALLOC(cache->header_mem, hdr_size);
/* position the sequence file to the start of the first sequence.
* this will force any buffers associated with the file to be reset.
*/
offset = ftell(ascii->fp);
if ((status = esl_sqfile_Position(sqfp, offset)) != eslOK) goto ERROR;
abc = esl_alphabet_Create(eslAMINO);
sq = esl_sq_CreateDigital(abc);
cache->db_cnt = db_cnt;
cache->db = db;
cache->abc = abc;
cache->res_size = res_size;
cache->hdr_size = hdr_size;
cache->count = seq_cnt;
hdr_ptr = cache->header_mem;
res_ptr = cache->residue_mem;
for (i = 0; i < db_cnt; ++i) db_inx[i] = 0;
strcpy(buffer, "000000001");
inx = 0;
while ((status = esl_sqio_Read(sqfp, sq)) == eslOK) {
/* sanity checks */
if (inx >= seq_cnt) { printf("inx: %d\n", inx); return eslEFORMAT; }
if (sq->n + 1 > res_size) { printf("inx: %d size %d %d\n", inx, (int)sq->n + 1, (int)res_size); return eslEFORMAT; }
if (hdr_size <= 0) { printf("inx: %d hdr %d\n", inx, (int)hdr_size); return eslEFORMAT; }
/* generate the database key - modified to take the first word in the desc line.
* The remaining part of the desc is then cached as the description. */
ptr = sq->desc;;
desc_ptr = strchr(sq->desc, ' ');
if(desc_ptr != NULL) {
*desc_ptr= '\0';
++desc_ptr;
}
val = 1;
db_key = 0;
while (*ptr) {
if (*ptr == '1') db_key += val;
val <<= 1;
++ptr;
}
if (db_key >= (1 << (db_cnt + 1))) { printf("inx: %d db %d %s\n", inx, db_key, sq->desc); return eslEFORMAT; }
cache->list[inx].name = hdr_ptr;
cache->list[inx].dsq = (ESL_DSQ *)res_ptr;
cache->list[inx].n = sq->n;
cache->list[inx].idx = inx;
cache->list[inx].db_key = db_key;
if(desc_ptr != NULL) esl_strdup(desc_ptr, -1, &(cache->list[inx].desc));
/* copy the digitized sequence */
memcpy(res_ptr, sq->dsq, sq->n + 1);
res_ptr += (sq->n + 1);
res_size -= (sq->n + 1);
/* copy the index to the header */
strcpy(hdr_ptr, buffer);
hdr_ptr += 10;
hdr_size -= 10;
/* increment the buffer string */
++buffer[8];
for (i = 8; i > 0; --i) {
if (buffer[i] > '9') {
buffer[i] = '0';
buffer[i-1]++;
}
}
esl_sq_Reuse(sq);
++inx;
}
if (status != eslEOF) { printf("Unexpected error %d at %d\n", status, inx); return status; }
if (inx != seq_cnt) { printf("inx:: %d %d\n", inx, seq_cnt); return eslEFORMAT; }
if (hdr_size != 0) { printf("inx:: %d hdr %d\n", inx, (int)hdr_size); return eslEFORMAT; }
if (res_size != 1) { printf("inx:: %d size %d %d\n", inx, (int)sq->n + 1, (int)res_size); return eslEFORMAT; }
/* copy the final sentinel character */
*res_ptr++ = eslDSQ_SENTINEL;
--res_size;
/* sort the order of the database sequences */
rnd = esl_randomness_CreateFast(seq_cnt);
for (i = 0 ; i < seq_cnt; ++i) {
rnd->x = rnd->x * 69069 + 1;
cache->list[i].idx = rnd->x;
}
esl_randomness_Destroy(rnd);
qsort(cache->list, seq_cnt, sizeof(HMMER_SEQ), sort_seq);
/* fill in the different databases and fix the index */
for (i = 0 ; i < seq_cnt; ++i) {
inx = 0;
db_key = cache->list[i].db_key;
while (db_key) {
if (db_key & 1) {
SEQ_DB *db = cache->db + inx;
if (db_inx[inx] >= db->count) { printf("sort:: %d %d\n", db_inx[inx], db->count); return eslEFORMAT; }
db->list[db_inx[inx]] = &cache->list[i];
++db_inx[inx];
}
db_key >>= 1;
++inx;
}
cache->list[i].idx = (cache->list[i].name - cache->header_mem) / 10 + 1;
}
for (i = 0; i < cache->db_cnt; ++i) {
printf("sequence database (%d):: %d %d\n", i, cache->db[i].count, db_inx[i]);
}
printf("\nLoaded sequence db file %s; total memory %" PRId64 "\n", seqfile, total_mem);
esl_sqfile_Close(sqfp);
esl_sq_Destroy(sq);
*ret_cache = cache;
return eslOK;
ERROR:
if (sq != NULL) esl_sq_Destroy(sq);
if (abc != NULL) esl_alphabet_Destroy(abc);
if (cache != NULL) {
if (cache->header_mem != NULL) free(cache->header_mem);
if (cache->residue_mem != NULL) free(cache->residue_mem);
if (cache->name != NULL) free(cache->name);
if (cache->id != NULL) free(cache->id);
free(cache);
}
for (i = 0; i < db_cnt; ++i) {
if (db[i].list != NULL) free(db[i].list);
}
return eslEMEM;
}
/*
* 1. Compare accscore to GOptimalAccuracy().
* 2. Compare trace to GOATrace().
*
* Note: This test is subject to some expected noise and can fail
* for entirely innocent reasons. Generic Forward/Backward calculations with
* p7_GForward(), p7_GBackward() use coarse-grain table lookups to sum
* log probabilities, and sufficient roundoff error can accumulate to
* change the optimal accuracy traceback, causing this test to fail.
* So, if optacc_utest fails, before you go looking for bugs, first
* go to ../logsum.c, change the #ifdef to activate the slow/accurate
* version, recompile and rerun optacc_utest. If the failure goes away,
* you can ignore it. - SRE, Wed Dec 17 09:45:31 2008
*/
static void
utest_optacc(ESL_GETOPTS *go, ESL_RANDOMNESS *r, ESL_ALPHABET *abc, P7_BG *bg, int M, int L, int N)
{
char *msg = "optimal accuracy unit test failed";
P7_HMM *hmm = NULL;
P7_PROFILE *gm = NULL;
P7_OPROFILE *om = NULL;
ESL_SQ *sq = esl_sq_CreateDigital(abc);
P7_OMX *ox1 = p7_omx_Create(M, L, L);
P7_OMX *ox2 = p7_omx_Create(M, L, L);
P7_GMX *gx1 = p7_gmx_Create(M, L);
P7_GMX *gx2 = p7_gmx_Create(M, L);
P7_TRACE *tr = p7_trace_CreateWithPP();
P7_TRACE *trg = p7_trace_CreateWithPP();
P7_TRACE *tro = p7_trace_CreateWithPP();
float accscore_o;
float fsc, bsc, accscore;
float fsc_g, bsc_g, accscore_g, accscore_g2;
float pptol = 0.01;
float sctol = 0.001;
float gtol;
p7_FLogsumInit();
gtol = ( (p7_FLogsumError(-0.4, -0.5) > 0.0001) ? 0.1 : 0.001);
if (p7_oprofile_Sample(r, abc, bg, M, L, &hmm, &gm, &om)!= eslOK) esl_fatal(msg);
while (N--)
{
if (p7_ProfileEmit(r, hmm, gm, bg, sq, tro) != eslOK) esl_fatal(msg);
if (p7_omx_GrowTo(ox1, M, sq->n, sq->n) != eslOK) esl_fatal(msg);
if (p7_omx_GrowTo(ox2, M, sq->n, sq->n) != eslOK) esl_fatal(msg);
if (p7_gmx_GrowTo(gx1, M, sq->n) != eslOK) esl_fatal(msg);
if (p7_gmx_GrowTo(gx2, M, sq->n) != eslOK) esl_fatal(msg);
if (p7_Forward (sq->dsq, sq->n, om, ox1, &fsc) != eslOK) esl_fatal(msg);
if (p7_Backward(sq->dsq, sq->n, om, ox1, ox2, &bsc) != eslOK) esl_fatal(msg);
if (p7_Decoding(om, ox1, ox2, ox2) != eslOK) esl_fatal(msg);
if (p7_OptimalAccuracy(om, ox2, ox1, &accscore) != eslOK) esl_fatal(msg);
#if 0
p7_omx_FDeconvert(ox1, gx1);
p7_gmx_Dump(stdout, gx1, p7_DEFAULT);
p7_omx_FDeconvert(ox2, gx1);
p7_gmx_Dump(stdout, gx1, p7_DEFAULT);
#endif
if (p7_OATrace(om, ox2, ox1, tr) != eslOK) esl_fatal(msg);
if (p7_GForward (sq->dsq, sq->n, gm, gx1, &fsc_g) != eslOK) esl_fatal(msg);
if (p7_GBackward(sq->dsq, sq->n, gm, gx2, &bsc_g) != eslOK) esl_fatal(msg);
#if 0
p7_gmx_Dump(stdout, gx1, p7_DEFAULT); /* fwd */
p7_gmx_Dump(stdout, gx2, p7_DEFAULT); /* bck */
#endif
if (p7_GDecoding(gm, gx1, gx2, gx2) != eslOK) esl_fatal(msg);
if (p7_GOptimalAccuracy(gm, gx2, gx1, &accscore_g) != eslOK) esl_fatal(msg);
#if 0
p7_gmx_Dump(stdout, gx1, p7_DEFAULT); /* oa */
p7_gmx_Dump(stdout, gx2, p7_DEFAULT); /* pp */
#endif
if (p7_GOATrace(gm, gx2, gx1, trg) != eslOK) esl_fatal(msg);
if (p7_trace_SetPP(tro, gx2) != eslOK) esl_fatal(msg);
if (esl_opt_GetBoolean(go, "--traces"))
{
p7_trace_Dump(stdout, tro, gm, sq->dsq);
p7_trace_Dump(stdout, tr, gm, sq->dsq);
p7_trace_Dump(stdout, trg, gm, sq->dsq);
}
if (p7_trace_Validate(tr, abc, sq->dsq, NULL) != eslOK) esl_fatal(msg);
if (p7_trace_Validate(trg, abc, sq->dsq, NULL) != eslOK) esl_fatal(msg);
if (p7_trace_Compare(tr, trg, pptol) != eslOK) esl_fatal(msg);
accscore_o = p7_trace_GetExpectedAccuracy(tro); /* according to gx2; see p7_trace_SetPP() call above */
accscore_g2 = p7_trace_GetExpectedAccuracy(trg);
#if 0
printf("%f %f %f %f\n", accscore, accscore_g, accscore_g2, accscore_o);
#endif
if (esl_FCompare(fsc, bsc, sctol) != eslOK) esl_fatal(msg);
if (esl_FCompare(fsc_g, bsc_g, gtol) != eslOK) esl_fatal(msg);
if (esl_FCompare(fsc, fsc_g, gtol) != eslOK) esl_fatal(msg);
if (esl_FCompare(accscore, accscore_g, gtol) != eslOK) esl_fatal(msg);
if (esl_FCompare(accscore_g, accscore_g2, gtol) != eslOK) esl_fatal(msg);
if (accscore_g2 < accscore_o) esl_fatal(msg);
/* the above deserves explanation:
* - accscore_o is the accuracy of the originally emitted trace, according
* to the generic posterior decoding matrix <gx2>. This is a lower bound
* on the expected # of accurately aligned residues found by a DP
* optimization.
* - accscore is the accuracy found by the fast (vector) code DP implementation.
* - accscore_g is the accuracy found by the generic DP implementation.
* accscore and accscore_g should be nearly identical,
* within tolerance of roundoff error accumulation and
* the imprecision of Logsum() tables.
* - accscore_g2 is the accuracy of the traceback identified by the generic
* DP implementation. It should be identical (within order-of-evaluation
* roundoff error) to accscore_g.
*
* the "accscore_g2 < accscore_o" test is carefully contrived.
* accscore_o is a theoretical lower bound but because of fp error,
* accscore and (much more rarely) even accscore_g can exceed accscore_o.
* accscore_g2, however, is calculated with identical order of evaluation
* as accscore_o if the optimal trace does turn out to be identical to
* the originally emitted trace. It should be extremely unlikely (though
* not impossible) for accscore_o to exceed accscore_g2. (The DP algorithm
* would have to identify a trace that was different than the original trace,
* which the DP algorithm, by order-of-evaluation, assigned higher accuracy,
* but order-of-evaluation in traceback dependent code assigned lower accuracy.
* [xref J5/29]
*/
esl_sq_Reuse(sq);
p7_trace_Reuse(tr);
p7_trace_Reuse(trg);
p7_trace_Reuse(tro);
}
p7_trace_Destroy(tro);
p7_trace_Destroy(trg);
p7_trace_Destroy(tr);
p7_gmx_Destroy(gx2);
p7_gmx_Destroy(gx1);
p7_omx_Destroy(ox2);
p7_omx_Destroy(ox1);
esl_sq_Destroy(sq);
p7_oprofile_Destroy(om);
p7_profile_Destroy(gm);
p7_hmm_Destroy(hmm);
}
static int
serial_loop(WORKER_INFO *info, ESL_SQFILE *dbfp, int n_targetseqs, FILE *ofp)
{
int sstatus;
ESL_SQ *dbsq = NULL; /* one target sequence (digital) */
int seq_cnt = 0;
dbsq = esl_sq_CreateDigital(info->om->abc);
SEQ_INFO *sqinfo = (SEQ_INFO *)malloc(INFO_SIZE * sizeof(SEQ_INFO));
size_t capacity = INFO_SIZE;
FILE *ftmp = fopen("tmp.txt", "w+");
/* Main loop: */
while ( (n_targetseqs==-1 || seq_cnt<n_targetseqs) && (sstatus = esl_sqio_Read(dbfp, dbsq)) == eslOK)
{
// p7_pli_NewSeq(info->pli, dbsq);
// p7_bg_SetLength(info->bg, dbsq->n);
// p7_oprofile_ReconfigLength(info->om, dbsq->n);
// p7_Pipeline(info->pli, info->om, info->bg, dbsq, info->th);
if (seq_cnt > capacity) {
capacity = capacity << 1;
SEQ_INFO * newsq = (SEQ_INFO * )realloc(sqinfo, capacity * sizeof(SEQ_INFO));
if (newsq) sqinfo = newsq;
else {
printf("Mem Error\n");
exit(1);
}
}
sqinfo[seq_cnt].index = seq_cnt;
sqinfo[seq_cnt].length = dbsq->n;
esl_sq_Textize(dbsq);
// if (dbsq->seq) printf("Text -- %s\n", dbsq->seq);
// if (dbsq->name) printf("Name -- %s\n", dbsq->name);
// if (dbsq->ss) printf("%s\n", dbsq->ss);
// if (dbsq->desc) printf("Desc -- %s\n", dbsq->desc);
// printf("%s\n", dbsq->desc);
// printf("Pos1 -- %d ", pos);
sqinfo[seq_cnt].pos = ftell(ftmp);
fprintf(ftmp, ">%s %s\n%s\n", dbsq->name, dbsq->desc, dbsq->seq);
// printf(" %d\n", pos);
sqinfo[seq_cnt].size = ftell(ftmp) - sqinfo[seq_cnt].pos; //strlen(dbsq->name) + strlen(dbsq->desc) + strlen(dbsq->seq) + 4;
seq_cnt++;
esl_sq_Reuse(dbsq);
// p7_pipeline_Reuse(info->pli);
}
qsort(sqinfo, seq_cnt, sizeof(SEQ_INFO), seqSortByLength);
// esl_sqfile_Position(dbfp, 0);
char *buf = (char *)malloc(sqinfo[0].size + 1);
buf[sqinfo[0].size] = '\0';
for (int i = 0; i < seq_cnt; ++i) {
// printf("index: %d length: %d \n", sqinfo[i].index, sqinfo[i].length);
// int ret = esl_sqfile_Position(dbfp, i);// sqinfo[i].index);
// printf("return: %d \n", ret);
fseek(ftmp, sqinfo[i].pos, SEEK_SET);
fread(buf, 1, sqinfo[i].size, ftmp);
if ( i == 0 ) printf("II %s", buf);
fwrite(buf, 1, sqinfo[i].size, ofp);
int ret = esl_sqio_Read(dbfp, dbsq);
// printf("2return: %d \n", ret);
// printf("i: %d length: %d \n", i, dbsq->n);
esl_sq_Reuse(dbsq);
}
if (n_targetseqs!=-1 && seq_cnt==n_targetseqs)
sstatus = eslEOF;
free(sqinfo);
fclose(ftmp);
esl_sq_Destroy(dbsq);
return sstatus;
}
int
main(int argc, char **argv)
{
ESL_GETOPTS *go = NULL;
char *seqfile = NULL;
ESL_SQFILE *sqfp = NULL;
int infmt = eslSQFILE_UNKNOWN;
int alphatype = eslUNKNOWN;
ESL_ALPHABET *abc = NULL;
ESL_SQ *sq = NULL;
int64_t nseq = 0;
int64_t nres = 0;
int64_t small = 0;
int64_t large = 0;
double *monoc = NULL; /* monoresidue composition per sequence */
double *monoc_all = NULL; /* monoresidue composition over all seqs */
int do_comp = FALSE;
int status = eslOK;
int wstatus;
int i;
int x;
/* Parse command line */
go = esl_getopts_Create(options);
if (esl_opt_ProcessCmdline(go, argc, argv) != eslOK) cmdline_failure(argv[0], "Failed to parse command line: %s\n", go->errbuf);
if (esl_opt_VerifyConfig(go) != eslOK) cmdline_failure(argv[0], "Error in app configuration: %s\n", go->errbuf);
if (esl_opt_GetBoolean(go, "-h") ) cmdline_help(argv[0], go);
if (esl_opt_ArgNumber(go) != 1) cmdline_failure(argv[0], "Incorrect number of command line arguments.\n");
seqfile = esl_opt_GetArg(go, 1);
do_comp = esl_opt_GetBoolean(go, "-c");
if (esl_opt_GetString(go, "--informat") != NULL) {
infmt = esl_sqio_FormatCode(esl_opt_GetString(go, "--informat"));
if (infmt == eslSQFILE_UNKNOWN) esl_fatal("%s is not a valid input sequence file format for --informat");
}
status = esl_sqfile_Open(seqfile, infmt, NULL, &sqfp);
if (status == eslENOTFOUND) esl_fatal("No such file %s", seqfile);
else if (status == eslEFORMAT) esl_fatal("Format of seqfile %s unrecognized.", seqfile);
else if (status != eslOK) esl_fatal("Open failed, code %d.", status);
if (esl_opt_GetBoolean(go, "--rna")) alphatype = eslRNA;
else if (esl_opt_GetBoolean(go, "--dna")) alphatype = eslDNA;
else if (esl_opt_GetBoolean(go, "--amino")) alphatype = eslAMINO;
else {
status = esl_sqfile_GuessAlphabet(sqfp, &alphatype);
if (status == eslEAMBIGUOUS) esl_fatal("Couldn't guess alphabet from first sequence in %s", seqfile);
else if (status == eslEFORMAT) esl_fatal("Sequence file parse error, line %d of file %s:\n%s\n",
sqfp->linenumber, seqfile, sqfp->errbuf);
else if (status == eslENODATA) esl_fatal("Sequence file %s contains no data?", seqfile);
else if (status != eslOK) esl_fatal("Failed to guess alphabet (error code %d)\n", status);
}
abc = esl_alphabet_Create(alphatype);
sq = esl_sq_CreateDigital(abc);
esl_sqfile_SetDigital(sqfp, abc);
if (do_comp) {
ESL_ALLOC(monoc, (abc->Kp) * sizeof(double));
ESL_ALLOC(monoc_all, (abc->Kp) * sizeof(double));
esl_vec_DSet(monoc_all, abc->Kp, 0.0);
esl_vec_DSet(monoc, abc->Kp, 0.0);
}
while ((wstatus = esl_sqio_ReadWindow(sqfp, 0, 4096, sq)) != eslEOF)
{
if (wstatus == eslOK)
{
if (do_comp)
for (i = 1; i <= sq->n; i++)
monoc[sq->dsq[i]]++;
}
else if (wstatus == eslEOD)
{
if (nseq == 0) { small = large = sq->L; }
else {
small = ESL_MIN(small, sq->L);
large = ESL_MAX(large, sq->L);
}
if (esl_opt_GetBoolean(go, "-a")) {
printf("= %-20s %8" PRId64 " %s\n", sq->name, sq->L, (sq->desc != NULL) ? sq->desc : "");
}
nres += sq->L;
nseq++;
esl_sq_Reuse(sq);
if (do_comp) {
esl_vec_DAdd(monoc_all, monoc, abc->Kp);
esl_vec_DSet(monoc, abc->Kp, 0.0);
}
}
else if (wstatus == eslEFORMAT)
{
esl_fatal("Failed to parse sequence at line %ld, file %s:\n%s",
(long) sqfp->linenumber, sqfp->filename, sqfp->errbuf);
}
else
esl_fatal("Failed in reading sequence:\n%s\n", sqfp->errbuf);
}
printf("Format: %s\n", esl_sqio_DescribeFormat(sqfp->format));
printf("Alphabet type: %s\n", esl_abc_DescribeType(abc->type));
printf("Number of sequences: %" PRId64 "\n", nseq);
printf("Total # residues: %" PRId64 "\n", nres);
printf("Smallest: %" PRId64 "\n", small);
printf("Largest: %" PRId64 "\n", large);
printf("Average length: %.1f\n", (float) nres / (float) nseq);
if (do_comp) {
printf("\nResidue composition:\n");
for (x = 0; x < abc->Kp; x++)
if (x < abc->K || monoc_all[x] > 0)
printf("residue: %c %10.0f %.4f\n", abc->sym[x], monoc_all[x], monoc_all[x] / (double) nres);
free(monoc);
free(monoc_all);
}
esl_alphabet_Destroy(abc);
esl_sq_Destroy(sq);
esl_sqfile_Close(sqfp);
esl_getopts_Destroy(go);
return 0;
ERROR:
return status;
}
int
main(int argc, char **argv)
{
ESL_GETOPTS *go = p7_CreateDefaultApp(options, 2, argc, argv, banner, usage);
char *hmmfile = esl_opt_GetArg(go, 1);
char *seqfile = esl_opt_GetArg(go, 2);
ESL_ALPHABET *abc = NULL;
P7_HMMFILE *hfp = NULL;
P7_HMM *hmm = NULL;
P7_BG *bg = NULL;
P7_PROFILE *gm = NULL;
P7_GMX *fwd = NULL;
P7_GMX *bck = NULL;
ESL_SQ *sq = NULL;
ESL_SQFILE *sqfp = NULL;
int format = eslSQFILE_UNKNOWN;
float fsc, bsc;
float nullsc;
int status;
/* Initialize log-sum calculator */
p7_FLogsumInit();
/* Read in one HMM */
if (p7_hmmfile_OpenE(hmmfile, NULL, &hfp, NULL) != eslOK) p7_Fail("Failed to open HMM file %s", hmmfile);
if (p7_hmmfile_Read(hfp, &abc, &hmm) != eslOK) p7_Fail("Failed to read HMM");
p7_hmmfile_Close(hfp);
/* Read in one sequence */
sq = esl_sq_CreateDigital(abc);
status = esl_sqfile_Open(seqfile, format, NULL, &sqfp);
if (status == eslENOTFOUND) p7_Fail("No such file.");
else if (status == eslEFORMAT) p7_Fail("Format unrecognized.");
else if (status == eslEINVAL) p7_Fail("Can't autodetect stdin or .gz.");
else if (status != eslOK) p7_Fail("Open failed, code %d.", status);
/* Configure a profile from the HMM */
bg = p7_bg_Create(abc);
gm = p7_profile_Create(hmm->M, abc);
/* Now reconfig the models however we were asked to */
if (esl_opt_GetBoolean(go, "--fs")) p7_ProfileConfig(hmm, bg, gm, sq->n, p7_LOCAL);
else if (esl_opt_GetBoolean(go, "--sw")) p7_ProfileConfig(hmm, bg, gm, sq->n, p7_UNILOCAL);
else if (esl_opt_GetBoolean(go, "--ls")) p7_ProfileConfig(hmm, bg, gm, sq->n, p7_GLOCAL);
else if (esl_opt_GetBoolean(go, "--s")) p7_ProfileConfig(hmm, bg, gm, sq->n, p7_UNIGLOCAL);
/* Allocate matrices */
fwd = p7_gmx_Create(gm->M, sq->n);
bck = p7_gmx_Create(gm->M, sq->n);
printf("%-30s %-10s %-10s %-10s %-10s\n", "# seq name", "fwd (raw)", "bck (raw) ", "fwd (bits)", "bck (bits)");
printf("%-30s %10s %10s %10s %10s\n", "#--------------", "----------", "----------", "----------", "----------");
while ( (status = esl_sqio_Read(sqfp, sq)) != eslEOF)
{
if (status == eslEFORMAT) p7_Fail("Parse failed (sequence file %s)\n%s\n", sqfp->filename, sqfp->get_error(sqfp));
else if (status != eslOK) p7_Fail("Unexpected error %d reading sequence file %s", status, sqfp->filename);
/* Resize the DP matrices if necessary */
p7_gmx_GrowTo(fwd, gm->M, sq->n);
p7_gmx_GrowTo(bck, gm->M, sq->n);
/* Set the profile and null model's target length models */
p7_bg_SetLength(bg, sq->n);
p7_ReconfigLength(gm, sq->n);
/* Run Forward, Backward */
p7_GForward (sq->dsq, sq->n, gm, fwd, &fsc);
p7_GBackward(sq->dsq, sq->n, gm, bck, &bsc);
p7_gmx_Dump(stdout, fwd, p7_DEFAULT);
/* Those scores are partial log-odds likelihoods in nats.
* Subtract off the rest of the null model, convert to bits.
*/
p7_bg_NullOne(bg, sq->dsq, sq->n, &nullsc);
printf("%-30s %10.4f %10.4f %10.4f %10.4f\n",
sq->name,
fsc, bsc,
(fsc - nullsc) / eslCONST_LOG2, (bsc - nullsc) / eslCONST_LOG2);
p7_gmx_Reuse(fwd);
p7_gmx_Reuse(bck);
esl_sq_Reuse(sq);
}
/* Cleanup */
esl_sqfile_Close(sqfp);
esl_sq_Destroy(sq);
p7_gmx_Destroy(fwd);
p7_gmx_Destroy(bck);
p7_profile_Destroy(gm);
p7_bg_Destroy(bg);
p7_hmm_Destroy(hmm);
esl_alphabet_Destroy(abc);
esl_getopts_Destroy(go);
return 0;
}
int
main(int argc, char **argv)
{
ESL_GETOPTS *go = esl_getopts_CreateDefaultApp(options, 2, argc, argv, banner, usage);
char *hmmfile = esl_opt_GetArg(go, 1);
char *seqfile = esl_opt_GetArg(go, 2);
ESL_ALPHABET *abc = NULL;
P7_HMMFILE *hfp = NULL;
P7_HMM *hmm = NULL;
P7_BG *bg = NULL;
P7_PROFILE *gm = NULL;
P7_OPROFILE *om = NULL;
P7_OMX *ox = NULL;
P7_GMX *gx = NULL;
ESL_SQ *sq = NULL;
ESL_SQFILE *sqfp = NULL;
int format = eslSQFILE_UNKNOWN;
float vfraw, nullsc, vfscore;
float graw, gscore;
double P, gP;
int status;
/* Read in one HMM */
if (p7_hmmfile_Open(hmmfile, NULL, &hfp) != eslOK) p7_Fail("Failed to open HMM file %s", hmmfile);
if (p7_hmmfile_Read(hfp, &abc, &hmm) != eslOK) p7_Fail("Failed to read HMM");
/* Read in one sequence */
sq = esl_sq_CreateDigital(abc);
status = esl_sqfile_Open(seqfile, format, NULL, &sqfp);
if (status == eslENOTFOUND) p7_Fail("No such file.");
else if (status == eslEFORMAT) p7_Fail("Format unrecognized.");
else if (status == eslEINVAL) p7_Fail("Can't autodetect stdin or .gz.");
else if (status != eslOK) p7_Fail("Open failed, code %d.", status);
/* create default null model, then create and optimize profile */
bg = p7_bg_Create(abc);
p7_bg_SetLength(bg, sq->n);
gm = p7_profile_Create(hmm->M, abc);
p7_ProfileConfig(hmm, bg, gm, sq->n, p7_LOCAL);
om = p7_oprofile_Create(gm->M, abc);
p7_oprofile_Convert(gm, om);
/* allocate DP matrices, both a generic and an optimized one */
ox = p7_omx_Create(gm->M, 0, sq->n);
gx = p7_gmx_Create(gm->M, sq->n);
/* Useful to place and compile in for debugging:
p7_oprofile_Dump(stdout, om); dumps the optimized profile
p7_omx_SetDumpMode(ox, TRUE); makes the fast DP algorithms dump their matrices
p7_gmx_Dump(stdout, gx); dumps a generic DP matrix
*/
while ((status = esl_sqio_Read(sqfp, sq)) == eslOK)
{
p7_oprofile_ReconfigLength(om, sq->n);
p7_ReconfigLength(gm, sq->n);
p7_bg_SetLength(bg, sq->n);
p7_omx_GrowTo(ox, om->M, 0, sq->n);
p7_gmx_GrowTo(gx, gm->M, sq->n);
p7_ViterbiFilter (sq->dsq, sq->n, om, ox, &vfraw);
p7_bg_NullOne (bg, sq->dsq, sq->n, &nullsc);
vfscore = (vfraw - nullsc) / eslCONST_LOG2;
P = esl_gumbel_surv(vfscore, om->evparam[p7_VMU], om->evparam[p7_VLAMBDA]);
p7_GViterbi (sq->dsq, sq->n, gm, gx, &graw);
gscore = (graw - nullsc) / eslCONST_LOG2;
gP = esl_gumbel_surv(gscore, gm->evparam[p7_VMU], gm->evparam[p7_VLAMBDA]);
if (esl_opt_GetBoolean(go, "-1"))
{
printf("%-30s\t%-20s\t%9.2g\t%7.2f\t%9.2g\t%7.2f\n", sq->name, hmm->name, P, vfscore, gP, gscore);
}
else if (esl_opt_GetBoolean(go, "-P"))
{ /* output suitable for direct use in profmark benchmark postprocessors: */
printf("%g\t%.2f\t%s\t%s\n", P, vfscore, sq->name, hmm->name);
}
else
{
printf("target sequence: %s\n", sq->name);
printf("vit filter raw score: %.2f nats\n", vfraw);
printf("null score: %.2f nats\n", nullsc);
printf("per-seq score: %.2f bits\n", vfscore);
printf("P-value: %g\n", P);
printf("GViterbi raw score: %.2f nats\n", graw);
printf("GViterbi seq score: %.2f bits\n", gscore);
printf("GViterbi P-value: %g\n", gP);
}
esl_sq_Reuse(sq);
}
/* cleanup */
esl_sq_Destroy(sq);
esl_sqfile_Close(sqfp);
p7_omx_Destroy(ox);
p7_gmx_Destroy(gx);
p7_oprofile_Destroy(om);
p7_profile_Destroy(gm);
p7_bg_Destroy(bg);
p7_hmm_Destroy(hmm);
p7_hmmfile_Close(hfp);
esl_alphabet_Destroy(abc);
esl_getopts_Destroy(go);
return 0;
}
int
main(int argc, char **argv)
{
ESL_GETOPTS *go = esl_getopts_CreateDefaultApp(options, 2, argc, argv, banner, usage);
char *hmmfile = esl_opt_GetArg(go, 1);
char *seqfile = esl_opt_GetArg(go, 2);
ESL_ALPHABET *abc = NULL;
P7_HMMFILE *hfp = NULL;
P7_HMM *hmm = NULL;
P7_BG *bg = NULL;
ESL_SQFILE *sqfp = NULL;
int format = eslSQFILE_UNKNOWN;
ESL_SQ *sq = NULL;
float nullsc, filtersc, H;
int status;
/* Read one HMM from <hmmfile> */
if (p7_hmmfile_Open(hmmfile, NULL, &hfp) != eslOK) p7_Fail("Failed to open HMM file %s", hmmfile);
if (p7_hmmfile_Read(hfp, &abc, &hmm) != eslOK) p7_Fail("Failed to read HMM");
p7_hmmfile_Close(hfp);
/* Open <seqfile> for reading */
status = esl_sqfile_OpenDigital(abc, seqfile, format, NULL, &sqfp);
if (status == eslENOTFOUND) esl_fatal("No such file.");
else if (status == eslEFORMAT) esl_fatal("Format unrecognized.");
else if (status != eslOK) esl_fatal("Open failed, code %d.", status);
sq = esl_sq_CreateDigital(abc);
bg = p7_bg_Create(abc);
p7_bg_SetFilterByHMM(bg, hmm);
H = esl_vec_FEntropy(bg->f, bg->abc->K);
printf("bg iid H = %.4f\n", H);
H = esl_vec_FEntropy(bg->mcomp, bg->abc->K);
printf("modelcomp H = %.4f\n", H);
while ((status = esl_sqio_Read(sqfp, sq)) == eslOK)
{
p7_bg_SetLength(bg, sq->n);
p7_bg_NullOne (bg, sq->dsq, sq->n, &nullsc);
p7_bg_FilterScore(bg, sq->dsq, sq->n, &filtersc);
printf("%-20s %5d %8.5f %8.5f %8.5f\n", sq->name, (int) sq->n, nullsc, filtersc, filtersc-nullsc);
esl_sq_Reuse(sq);
}
if (status == eslEFORMAT) esl_fatal("Parse failed (sequence file %s line %" PRId64 "):\n%s\n",
sqfp->filename, sqfp->linenumber, sqfp->errbuf);
else if (status != eslEOF) esl_fatal("Unexpected error %d reading sequence file %s",
status, sqfp->filename);
esl_sqfile_Close(sqfp);
esl_sq_Destroy(sq);
p7_bg_Destroy(bg);
p7_hmm_Destroy(hmm);
esl_alphabet_Destroy(abc);
esl_getopts_Destroy(go);
return 0;
}
int main(int argc, char **argv)
{
ESL_GETOPTS *go = esl_getopts_CreateDefaultApp(options, 2, argc, argv, banner, usage);
char *hmmfile = esl_opt_GetArg(go, 1);
char *seqfile = esl_opt_GetArg(go, 2);
ESL_STOPWATCH *w = esl_stopwatch_Create();
ESL_RANDOMNESS*r = esl_randomness_Create(esl_opt_GetInteger(go, "-s"));
ESL_ALPHABET*abc = NULL;
P7_HMMFILE *hfp = NULL;
P7_HMM *hmm = NULL;
P7_BG *bg = NULL;
P7_PROFILE *gm1, *gm2;
int L = 2000; // esl_opt_GetInteger(go, "-L");
int N = esl_opt_GetInteger(go, "-N");
int MaxPart = esl_opt_GetInteger(go, "-M");
__m128 resdata[10];
float *sc1 = (float*) (resdata+0); // uses 1 __m128s
ESL_DSQ *dsq = NULL;
int i, j;
ESL_SQFILE *sqfp = NULL;
DATA_STREAM *dstream;
struct timeb tbstart, tbend;
int sumlengths = 0;
if (p7_hmmfile_Open(hmmfile, NULL, &hfp)!= eslOK) p7_Fail("Failed to open HMM file %s", hmmfile);
if (p7_hmmfile_Read(hfp, &abc, &hmm) != eslOK) p7_Fail("Failed to read HMM");
bg = p7_bg_Create(abc);
p7_bg_SetLength(bg, L);
gm1 = p7_profile_Create(hmm->M, abc);
gm2 = p7_profile_Create(hmm->M, abc);
p7_ProfileConfig(hmm, bg, gm1, L, p7_UNILOCAL);
p7_ProfileConfig(hmm, bg, gm2, L, p7_UNILOCAL);
dstream = p7_ViterbiStream_Create(gm1);
p7_ViterbiStream_Setup(dstream, L+100, MaxPart); // use max L
dstream->L = L;
// No. of partitions computed without full parallelism ( == no. of threads active while some are idle)
int Niters_part = dstream->Npartitions % NTHREADS;
// No. of Model lines that could be computed but are wasted by idle threads waiting on the end
int Nwasted_threads = dstream->partition * ((NTHREADS-Niters_part) % NTHREADS);
// No. of lines in the last partition that go beyond M. It's wasted comp time by a single thread
int Nwasted_leftover= (dstream->partition - gm1->M % dstream->partition) % dstream->partition;
// Total number of wasted lines
int wastedcomp = Nwasted_threads + Nwasted_leftover;
// Total number of lines computed and waited for
int totalcomp = wastedcomp + gm1->M; // same as: roundtop(gm1->M, dstream->partition * NTHREADS);
printf("Viterbi Stream Word with %d Threads, model %s: Modelsize %d, #Segms: %d, SeqL: %d, Nseqs %d, Part %d, #Parts %d\n",
NTHREADS, hmmfile, gm1->M, (int) ceil(gm1->M/8.0), L, 8*N, dstream->partition, dstream->Npartitions);
printf("Total Comp Lines: %d | Wasted Comp Lines: %d\n", totalcomp, wastedcomp);
// for ViterbiFilter
P7_OPROFILE *om = p7_oprofile_Create(hmm->M, gm1->abc);
p7_oprofile_Convert(gm1, om);
P7_OMX *ox = p7_omx_Create(hmm->M, 0, 0);
dsq_cmp_t **seqsdb= calloc(8*N+64, sizeof(dsq_cmp_t*));
if(0)
{ ESL_SQ* sq = esl_sq_CreateDigital(abc);
if (esl_sqfile_OpenDigital(abc, seqfile, eslSQFILE_FASTA, NULL, &sqfp) != eslOK)
{ p7_Fail("Failed to open sequence file\n"); return -1; }
for (j = 0; j < 8*N; j++)
{
int res = esl_sqio_Read(sqfp, sq);
if (res != eslOK) { printf("ATENCAO: faltam sequencias\n"); break; }
int len = sq->n;
dsq = sq->dsq;
seqsdb[j] = malloc(sizeof(dsq_cmp_t));
seqsdb[j]->length = len;
seqsdb[j]->seq = malloc((len+4)*sizeof(ESL_DSQ));
memcpy(seqsdb[j]->seq, dsq, len+2);
sumlengths += len;
esl_sq_Reuse(sq);
}
ftime(&tbstart);
N = j/8;
printf("N = %d\n", N);
// Sort sequences by length
qsort(seqsdb, N*8, sizeof(dsq_cmp_t*), compare_seqs);
}
else if(0)
for (i = 0; i < N; i++)
{
for (j = 0; j < 8; j++)
{
int len = L - rand()%1000;
seqsdb[i*8+j] = malloc(sizeof(dsq_cmp_t));
seqsdb[i*8+j]->seq = malloc(len+4);
seqsdb[i*8+j]->length = len;
esl_rsq_xfIID(r, bg->f, abc->K, len, seqsdb[i*8+j]->seq);
sumlengths += len;
}
}
// double sumerrors = 0;
float* results = (float*) alloc_m128_aligned64(N*2+2);
ftime(&tbstart);
for (j = 0; j < N; j++)
for (i = 0; i < N; i++)
{
// if (i % 10000 == 0) printf("START %d\n", i);
p7_ViterbiStream(dstream, seqsdb+i*8, sc1);
// memcpy(results+i*8, sc1, 32);
}
ftime(&tbend);
double secs = TIMEDIFF(tbstart,tbend);
// printf("Qsort time: %6.3f | Viterbi time: %6.3f\n", TIMEDIFF(tbqsort,tbstart), secs);
w->elapsed = w->user = secs;
esl_stopwatch_Display(stdout, w, "# Opt CPU time: ");
printf("# %.0fM cells in %.1f Mc/s\n", (sumlengths * (double) gm1->M) / 1e6, (sumlengths * (double) gm1->M * 1e-6) / secs);
if(0) // compare results against base version
for (i = 0; i < 1000 && i < N; i++)
{
int maxll = 0; float sc2;
for (j = 0; j < 8; j++)
if (maxll < seqsdb[i*8+j]->length)
maxll = seqsdb[i*8+j]->length;
// for (j = 0; j < 8; j++) printf("%d ", seqsdb[i*8+j]->length); printf("\n");
// if (i % 10 == 0) printf("i %d\n", i);
p7_oprofile_ReconfigRestLength(om, maxll);
p7_ReconfigLength(gm2, maxll); // fazer Reconfig aqui para emular compl o VitStream
for (j = 0; j < 8; j++)
{
// p7_ReconfigLength(gm2, seqsdb[i*8+j]->length);
// p7_Viterbi_unilocal(seqsdb[i*8+j]->seq, seqsdb[i*8+j]->length, gm2, &sc3);
// p7_Viterbi_unilocal_word(seqsdb[i*8+j]->seq, seqsdb[i*8+j]->length, gm2, &sc2);
// p7_oprofile_ReconfigLength(om, seqsdb[i*8+j]->length);
p7_ViterbiFilter(seqsdb[i*8+j]->seq, seqsdb[i*8+j]->length, om, ox, &sc2);
//sumerrors += fabs(sc1[j]- sc2);
if (fabs(results[i*8+j] - sc2) > 0.00001)
{ printf("%3d-%d L %4d: VS %f d %f\t| Base SerI %f\n", i, j, seqsdb[i*8+j]->length,
results[i*8+j], fabs(results[i*8+j] - sc2), sc2);
getc(stdin);
}
}
}
return 0;
}
/* multifetch:
* given a file containing lines with one name or key per line;
* parse the file line-by-line;
* if we have an SSI index available, retrieve the seqs by key
* as we see each line;
* else, without an SSI index, store the keys in a hash, then
* read the entire seq file in a single pass, outputting seqs
* that are in our keylist.
*
* Note that with an SSI index, you get the seqs in the order they
* appear in the <keyfile>, but without an SSI index, you get seqs in
* the order they occur in the seq file.
*/
static void
multifetch(ESL_GETOPTS *go, FILE *ofp, char *keyfile, ESL_SQFILE *sqfp)
{
ESL_KEYHASH *keys = esl_keyhash_Create();
ESL_FILEPARSER *efp = NULL;
int nseq = 0;
int nkeys = 0;
char *key;
int keylen;
int keyidx;
int status;
if (esl_fileparser_Open(keyfile, NULL, &efp) != eslOK) esl_fatal("Failed to open key file %s\n", keyfile);
esl_fileparser_SetCommentChar(efp, '#');
while (esl_fileparser_NextLine(efp) == eslOK)
{
if (esl_fileparser_GetTokenOnLine(efp, &key, &keylen) != eslOK)
esl_fatal("Failed to read seq name on line %d of file %s\n", efp->linenumber, keyfile);
status = esl_keyhash_Store(keys, key, keylen, &keyidx);
if (status == eslEDUP) esl_fatal("seq key %s occurs more than once in file %s\n", key, keyfile);
/* if we have an SSI index, just fetch them as we go. */
if (sqfp->data.ascii.ssi != NULL) { onefetch(go, ofp, key, sqfp); nseq++; }
nkeys++;
}
/* If we don't have an SSI index, we haven't fetched anything yet; do it now. */
if (sqfp->data.ascii.ssi == NULL)
{
ESL_SQ *sq = esl_sq_Create();
while ((status = esl_sqio_Read(sqfp, sq)) == eslOK)
{
if ( (sq->name[0] != '\0' && esl_keyhash_Lookup(keys, sq->name, -1, NULL) == eslOK) ||
(sq->acc[0] != '\0' && esl_keyhash_Lookup(keys, sq->acc, -1, NULL) == eslOK))
{
if (esl_opt_GetBoolean(go, "-r") )
if (esl_sq_ReverseComplement(sq) != eslOK)
esl_fatal("Failed to reverse complement %s\n", sq->name);
esl_sqio_Write(ofp, sq, eslSQFILE_FASTA, FALSE);
nseq++;
}
esl_sq_Reuse(sq);
}
if (status == eslEFORMAT) esl_fatal("Parse failed (sequence file %s):\n%s\n",
sqfp->filename, esl_sqfile_GetErrorBuf(sqfp));
else if (status != eslEOF) esl_fatal("Unexpected error %d reading sequence file %s",
status, sqfp->filename);
esl_sq_Destroy(sq);
}
if (nkeys != nseq) esl_fatal("Tried to retrieve %d keys, but only retrieved %d sequences\n", nkeys, nseq);
if (ofp != stdout) printf("\nRetrieved %d sequences.\n", nseq);
esl_keyhash_Destroy(keys);
esl_fileparser_Close(efp);
return;
}
