/* Function: p7_Handmodelmaker()
*
* Purpose: Manual model construction.
* Construct an HMM from a digital alignment, where the
* <#=RF> line of the alignment file is used to indicate the
* columns assigned to matches vs. inserts.
*
* The <msa> must be in digital mode, and it must have
* a reference annotation line.
*
* NOTE: <p7_Handmodelmaker()> will slightly revise the
* alignment if necessary, if the assignment of columns
* implies DI and ID transitions.
*
* Returns both the HMM in counts form (ready for applying
* Dirichlet priors as the next step), and fake tracebacks
* for each aligned sequence.
*
* Models must have at least one node, so if the <msa> defined
* no consensus columns, a <eslENORESULT> error is returned.
*
* Args: msa - multiple sequence alignment
* bld - holds information on regions requiring masking, optionally NULL -> no masking
* ret_hmm - RETURN: counts-form HMM
* opt_tr - optRETURN: array of tracebacks for aseq's
*
* Return: <eslOK> on success. <ret_hmm> and <opt_tr> are allocated
* here, and must be free'd by caller.
*
* Returns <eslENORESULT> if no consensus columns were annotated;
* in this case, <ret_hmm> and <opt_tr> are returned NULL.
*
* Returns <eslEFORMAT> if the <msa> doesn't have a reference
* annotation line.
*
* Throws: <eslEMEM> on allocation failure. Throws <eslEINVAL> if the <msa>
* isn't in digital mode.
*/
int
p7_Handmodelmaker(ESL_MSA *msa, P7_BUILDER *bld, P7_HMM **ret_hmm, P7_TRACE ***opt_tr)
{
int status;
int *matassign = NULL; /* MAT state assignments if 1; 1..alen */
int apos; /* counter for aligned columns */
if (! (msa->flags & eslMSA_DIGITAL)) ESL_XEXCEPTION(eslEINVAL, "need a digital msa");
if (msa->rf == NULL) return eslEFORMAT;
ESL_ALLOC(matassign, sizeof(int) * (msa->alen+1));
/* Watch for off-by-one. rf is [0..alen-1]; matassign is [1..alen] */
for (apos = 1; apos <= msa->alen; apos++)
matassign[apos] = (esl_abc_CIsGap(msa->abc, msa->rf[apos-1])? FALSE : TRUE);
/* matassign2hmm leaves ret_hmm, opt_tr in their proper state: */
if ((status = matassign2hmm(msa, matassign, ret_hmm, opt_tr)) != eslOK) goto ERROR;
free(matassign);
return eslOK;
ERROR:
if (matassign != NULL) free(matassign);
return status;
}
/* map_rfpos_to_apos
*
* Given an MSA, determine the alignment position of each
* non-gap RF (reference) position. The abc is only necessary
* for defining gap characters.
*
* rf2a_map[0..rfpos..rflen-1] = apos, apos is the alignment position (0..msa->alen-1) that
* is non-gap RF position rfpos+1 (for rfpos in 0..rflen-1)
*/
static int map_rfpos_to_apos(ESL_MSA *msa, ESL_ALPHABET *abc, char *errbuf, int64_t alen, int **ret_i_am_rf, int **ret_rf2a_map, int *ret_rflen)
{
int status;
int rflen = 0;
int *rf2a_map = NULL;
int *i_am_rf = NULL;
int rfpos = 0;
int apos = 0;
/* contract check */
if(msa->rf == NULL) ESL_FAIL(eslEINVAL, errbuf, "Error, trying to map RF positions to alignment positions, but msa->rf is NULL.");
/* count non-gap RF columns */
for(apos = 0; apos < alen; apos++) {
if((! esl_abc_CIsGap(abc, msa->rf[apos])) &&
(! esl_abc_CIsMissing(abc, msa->rf[apos])) &&
(! esl_abc_CIsNonresidue(abc, msa->rf[apos])))
{
rflen++;
/* I don't use esl_abc_CIsResidue() b/c that would return FALSE for 'x' with RNA and DNA */
}
}
/* build map */
ESL_ALLOC(i_am_rf, sizeof(int) * alen);
ESL_ALLOC(rf2a_map, sizeof(int) * rflen);
for(apos = 0; apos < alen; apos++) {
if((! esl_abc_CIsGap(abc, msa->rf[apos])) &&
(! esl_abc_CIsMissing(abc, msa->rf[apos])) &&
(! esl_abc_CIsNonresidue(abc, msa->rf[apos]))) {
i_am_rf[apos] = TRUE;
rf2a_map[rfpos++] = apos;
}
else {
i_am_rf[apos] = FALSE;
}
}
*ret_i_am_rf = i_am_rf;
*ret_rf2a_map = rf2a_map;
*ret_rflen = rflen;
return eslOK;
ERROR:
if(i_am_rf != NULL) free(i_am_rf);
if(rf2a_map != NULL) free(rf2a_map);
ESL_FAIL(status, errbuf, "Error, out of memory while mapping RF positions to alignment positions.");
}
/* map_rfpos_to_apos
*
* Given an MSA, determine the alignment position each
* reference position refers to.
*/
static int map_rfpos_to_apos(ESL_MSA *msa, int **ret_rf2a_map, int **ret_a2rf_map, int *ret_rflen)
{
int status;
int rflen = 0;
int *rf2a_map = NULL;
int *a2rf_map = NULL;
int rfpos = 0;
int apos = 0;
/* contract check */
if(msa->rf == NULL) { status = eslEINVAL; goto ERROR; }
/* count reference columns */
for(apos = 1; apos <= msa->alen; apos++)
if((! esl_abc_CIsGap(msa->abc, msa->rf[(apos-1)])) &&
(! esl_abc_CIsMissing(msa->abc, msa->rf[(apos-1)])) &&
(! esl_abc_CIsNonresidue(msa->abc, msa->rf[(apos-1)]))) rflen++;
/* build maps */
ESL_ALLOC(rf2a_map, sizeof(int) * (rflen+1));
ESL_ALLOC(a2rf_map, sizeof(int) * (msa->alen+1));
esl_vec_ISet(a2rf_map, msa->alen+1, -1);
rf2a_map[0] = -1;
for(apos = 1; apos <= msa->alen; apos++) {
if((! esl_abc_CIsGap(msa->abc, msa->rf[(apos-1)])) &&
(! esl_abc_CIsMissing(msa->abc, msa->rf[(apos-1)])) &&
(! esl_abc_CIsNonresidue(msa->abc, msa->rf[(apos-1)]))) {
rf2a_map[++rfpos] = apos;
a2rf_map[apos] = rfpos;
}
/* else a2rf_map[apos] remains -1 as it was initialized */
}
if(ret_rf2a_map != NULL) *ret_rf2a_map = rf2a_map;
else free(rf2a_map);
if(ret_a2rf_map != NULL) *ret_a2rf_map = a2rf_map;
else free(a2rf_map);
if(ret_rflen != NULL) *ret_rflen = rflen;
return eslOK;
ERROR:
if(rf2a_map != NULL) free(rf2a_map);
if(a2rf_map != NULL) free(a2rf_map);
return status;
}
/* get_pp_idx
*
* Given a #=GR PP or #=GC PP_cons character, return the appropriate index
* in a pp_ct[] vector.
* '0' return 0;
* '1' return 1;
* '2' return 2;
* '3' return 3;
* '4' return 4;
* '5' return 5;
* '6' return 6;
* '7' return 7;
* '8' return 8;
* '9' return 9;
* '*' return 10;
* gap return 11;
*
* Anything else (including missing or nonresidue) return -1;
*/
static int get_pp_idx(ESL_ALPHABET *abc, char ppchar)
{
if(esl_abc_CIsGap(abc, ppchar)) return 11;
if(ppchar == '*') return 10;
if(ppchar == '9') return 9;
if(ppchar == '8') return 8;
if(ppchar == '7') return 7;
if(ppchar == '6') return 6;
if(ppchar == '5') return 5;
if(ppchar == '4') return 4;
if(ppchar == '3') return 3;
if(ppchar == '2') return 2;
if(ppchar == '1') return 1;
if(ppchar == '0') return 0;
return -1;
}
/* Function: p7_Alimask_MakeModel2AliMap()
* Synopsis: Compute map of coordinate in the alignment corresponding to each model position.
*
* Args: msa - The alignment for which the mapped model is to be computed. We assume
* the MSA has already been manipulated to account for model building
* flags (e.g. weighting).
* do_hand - TRUE when the model is to follow a hand-build RF line (which must be
* part of the file.
* symfraq - if weighted occupancy exceeds this value, include the column in the model.
* map - int array into which the map values will be stored. Calling function
* must allocate (msa->alen+1) ints.
*
* Returns: The number of mapped model positions.
*/
int
p7_Alimask_MakeModel2AliMap(ESL_MSA *msa, int do_hand, float symfrac, int *map )
{
int i = 0;
int apos, idx;
float r; /* weighted residue count */
float totwgt; /* weighted residue+gap count */
i = 0;
if ( do_hand ) {
if (msa->rf == NULL) p7_Fail("Model file does not contain an RF line, required for --hand.\n");
/* Watch for off-by-one. rf is [0..alen-1]*/
for (apos = 1; apos <= msa->alen; apos++) {
if (!esl_abc_CIsGap(msa->abc, msa->rf[apos-1]) ) {
map[i] = apos;
i++;
}
}
} else {
for (apos = 1; apos <= msa->alen; apos++)
{
r = totwgt = 0.;
for (idx = 0; idx < msa->nseq; idx++)
{
if (esl_abc_XIsResidue(msa->abc, msa->ax[idx][apos])) { r += msa->wgt[idx]; totwgt += msa->wgt[idx]; }
else if (esl_abc_XIsGap(msa->abc, msa->ax[idx][apos])) { totwgt += msa->wgt[idx]; }
else if (esl_abc_XIsMissing(msa->abc, msa->ax[idx][apos])) continue;
}
if (r > 0. && r / totwgt >= symfrac) {
map[i] = apos;
i++;
}
}
}
return i;
}
static int
rejustify_insertions_text(const ESL_ALPHABET *abc, ESL_MSA *msa, const int *inserts, const int *matmap, const int *matuse, int M)
{
int idx;
int k;
int apos;
int nins;
int npos, opos;
for (idx = 0; idx < msa->nseq; idx++)
{
for (k = 0; k < M; k++)
if (inserts[k] > 1)
{
for (nins = 0, apos = matmap[k]; apos < matmap[k+1]-matuse[k+1]; apos++)
if (esl_abc_CIsResidue(abc, msa->aseq[idx][apos])) nins++;
if (k == 0) nins = 0; /* N-terminus is right justified */
else nins /= 2; /* split in half; nins now = # of residues left left-justified */
opos = npos = -1+matmap[k+1]-matuse[k+1];
while (opos >= matmap[k]+nins) {
if (esl_abc_CIsGap(abc, msa->aseq[idx][opos])) opos--;
else {
msa->aseq[idx][npos] = msa->aseq[idx][opos];
if (msa->pp != NULL && msa->pp[idx] != NULL) msa->pp[idx][npos] = msa->pp[idx][opos];
npos--;
opos--;
}
}
while (npos >= matmap[k]+nins) {
msa->aseq[idx][npos] = '.';
if (msa->pp != NULL && msa->pp[idx] != NULL) msa->pp[idx][npos] = '.';
npos--;
}
}
}
return eslOK;
}
/* Function: esl_msashuffle_CQRNA()
* Synopsis: Gap-preserving column shuffle of a pairwise alignment.
* Incept: SRE, Tue Jan 22 08:45:34 2008 [Market Street Cafe, Leesburg]
*
* Purpose: Shuffle a pairwise alignment <x>,<y> while preserving the
* position of gaps, using the random number generator <r>.
* Return the shuffled alignment in <xs>,
* <ys>. Caller provides allocated space for <xs> and <ys>.
*
* An alphabet <abc> must also be provided, solely for the
* definition of gap characters. Because Easel's default
* alphabets (DNA, RNA, and protein) all use the same
* definition of gap characters <-_.>, you can actually
* provide any alphabet here, and get the same results.
* (This may save having to determine the alphabet of input
* sequences.)
*
* Works by doing three separate
* shuffles, of (1) columns with residues in both
* <x> and <y>, (2) columns with residue in <x> and gap in <y>,
* and (3) columns with gap in <x> and residue in <y>.
*
* <xs>,<x> and <ys>,<y> may be identical: that is, to shuffle
* an alignment "in place", destroying the original
* alignment, just call <esl_msashuffle_CQRNA(r, abc, x,y,x,y)>.
*
* Returns: <eslOK> on success, and the shuffled alignment is
* returned in <xs>, <ys>.
*
* Throws: <eslEMEM> on allocation failure.
*/
int
esl_msashuffle_CQRNA(ESL_RANDOMNESS *r, ESL_ALPHABET *abc, char *x, char *y, char *xs, char *ys)
{
int L;
int *xycol = NULL;
int *xcol = NULL;
int *ycol = NULL;
int nxy, nx, ny;
int i;
int pos, c;
char xsym, ysym;
int status;
if (xs != x) strcpy(xs, x);
if (ys != y) strcpy(ys, y);
/* First, construct three arrays containing lists of the column positions
* of the three types of columns. (If a column contains gaps in both x and y,
* we've already simply copied it to the shuffled sequence.)
*/
L = strlen(x);
if (strlen(y) != L) ESL_XEXCEPTION(eslEINVAL, "sequences of different lengths in qrna shuffle");
ESL_ALLOC(xycol, sizeof(int) * L);
ESL_ALLOC(xcol, sizeof(int) * L);
ESL_ALLOC(ycol, sizeof(int) * L);
nxy = nx = ny = 0;
for (i = 0; i < L; i++)
{
if ( esl_abc_CIsGap(abc, x[i]) && esl_abc_CIsGap(abc, y[i])) { continue; }
else if (! esl_abc_CIsGap(abc, x[i]) && ! esl_abc_CIsGap(abc, y[i])) { xycol[nxy] = i; nxy++; }
else if ( esl_abc_CIsGap(abc, x[i])) { ycol[ny] = i; ny++; }
else if ( esl_abc_CIsGap(abc, y[i])) { xcol[nx] = i; nx++; }
}
/* Second, shuffle the sequences indirectly, via shuffling these arrays.
* Yow, careful with those indices, and with order of the statements...
*/
for (; nxy > 1; nxy--) {
pos = esl_rnd_Roll(r, nxy);
xsym = xs[xycol[pos]]; ysym = ys[xycol[pos]]; c = xycol[pos];
xs[xycol[pos]] = xs[xycol[nxy-1]]; ys[xycol[pos]] = ys[xycol[nxy-1]]; xycol[pos] = xycol[nxy-1];
xs[xycol[nxy-1]] = xsym; ys[xycol[nxy-1]] = ysym; xycol[pos] = xycol[nxy-1];
}
for (; nx > 1; nx--) {
pos = esl_rnd_Roll(r, nx);
xsym = xs[xcol[pos]]; ysym = ys[xcol[pos]]; c = xcol[pos];
xs[xcol[pos]] = xs[xcol[nx-1]]; ys[xcol[pos]] = ys[xcol[nx-1]]; xcol[pos] = xcol[nx-1];
xs[xcol[nx-1]] = xsym; ys[xcol[nx-1]] = ysym; xcol[nx-1] = c;
}
for (; ny > 1; ny--) {
pos = esl_rnd_Roll(r, ny);
xsym = xs[ycol[pos]]; ysym = ys[ycol[pos]]; c = ycol[pos];
xs[ycol[pos]] = xs[ycol[ny-1]]; ys[ycol[pos]] = ys[ycol[ny-1]]; ycol[pos] = ycol[ny-1];
xs[ycol[ny-1]] = xsym; ys[ycol[ny-1]] = ysym; ycol[ny-1] = c;
}
free(xycol); free(xcol); free(ycol);
return eslOK;
ERROR:
if (xycol != NULL) free(xycol);
if (xcol != NULL) free(xcol);
if (ycol != NULL) free(ycol);
return status;
}
int
main(int argc, char **argv)
{
ESL_GETOPTS *go; /* application configuration */
int kstatus, tstatus;/* return code from Easel routine */
int fmt; /* expected format of kfile, tfile */
char *kfile, *tfile; /* known, test structure file */
ESL_MSAFILE *kfp, *tfp; /* open kfile, tfile */
ESL_MSA *ka, *ta; /* known, trusted alignment */
int64_t klen, tlen; /* lengths of dealigned seqs */
int i; /* counter over sequences */
int apos; /* counter over alignment columns */
int rfpos; /* counter over consensus (non-gap RF) columns */
int is_rfpos; /* TRUE if current apos is a consensus pos, FALSE if not */
int uapos; /* counter over unaligned residue positions */
int nali; /* number of alignment we're on in each file */
int **kp; /* [0..i..nseq-1][1..r..sq->n] = x known non-gap RF position of residue r in sequence i */
int **tp; /* [0..i..nseq-1][1..r..sq->n] = x predicted non-gap RF position of residue r in sequence i */
/* for both kp and pp, if x <= 0, residue r for seq i is not aligned to a non-gap RF position, but rather as an 'insert'
* after non-gap RF position (x * -1)
*/
int *km_pos; /* [0..rflen] = x, in known aln, number of residues aligned to non-gap RF column x; special case: mct[0] = 0 */
int *ki_pos; /* [0..rflen] = x, in known aln, number of residues inserted after non-gap RF column x */
int *tm_pos; /* [0..rflen] = x, in predicted aln, number of residues aligned to non-gap RF column x; special case: mct[0] = 0 */
int *ti_pos; /* [0..rflen] = x, in predicted aln, number of residues inserted after non-gap RF column x */
int *cor_tm_pos; /* [0..rflen] = x, in predicted aln, number of correctly predicted residues aligned to non-gap RF column x; special case: mct[0] = 0 */
int *cor_ti_pos; /* [0..rflen] = x, in predicted aln, number of correctly predicted residues inserted after non-gap RF column x */
int *km_seq; /* [0..i..nseq-1] = x, in known aln, number of residues aligned to non-gap RF columns in seq i; */
int *ki_seq; /* [0..i..nseq-1] = x, in known aln, number of residues inserted in seq i */
int *tm_seq; /* [0..i..nseq-1] = x, in predicted aln, number of residues aligned to non-gap RF columns in seq i; */
int *ti_seq; /* [0..i..nseq-1] = x, in predicted aln, number of residues inserted in seq i */
int *cor_tm_seq; /* [0..i..nseq-1] = x, in predicted aln, number of correctly predicted residues aligned to non-gap RF columns in seq i */
int *cor_ti_seq; /* [0..i..nseq-1] = x, in predicted aln, number of correctly predicted residues inserted in seq i */
int *seqlen; /* [0..i..nseq-1] = x, unaligned seq i has length x */
ESL_ALPHABET *abc = NULL; /* alphabet for all alignments */
int rflen, t_rflen; /* non-gap RF length (consensus lengths) */
int status;
char *namedashes;
int ni;
int namewidth = 8; /* length of 'seq name' */
int cor_tm, cor_ti, km, ki; /* correct predicted match, correct predicted insert, total match, total insert */
char *mask = NULL;
int masklen;
ESL_DSQ *ks;
ESL_DSQ *ts;
FILE *dfp = NULL; /* for --c2dfile */
/* variables needed for -p and related options */
int do_post = FALSE; /* TRUE if -p enabled */
int do_post_for_this_rfpos = FALSE; /* set for each consensus position, always TRUE unless --mask-p2xm */
int p; /* counter over integerized posteriors */
int *ptm = NULL; /* [0..p..10] number of total matches with posterior value p (10="*")*/
int *pti = NULL; /* [0..p..10] number of total inserts with posterior value p */
int *cor_ptm = NULL; /* [0..p..10] number of correct matches with posterior value p */
int *cor_pti = NULL; /* [0..p..10] number of correct inserts with posterior value p */
int npostvals = 11; /* number of posterior values 0-9, * */
int ppidx; /* index of PP */
char ppchars[11] = "0123456789*";
int cm_cor_ptm, cm_cor_pti, cm_ptm, cm_pti, cm_incor_ptm, cm_incor_pti; /* cumulative counts of posteriors */
// int tot_cor_ptm, tot_cor_pti, tot_ptm, tot_pti; /* total counts of posteriors */
// int tot_incor_ptm,tot_incor_pti; // SRE: commented out; don't seem to be used; need to silence compiler warning
char errbuf[eslERRBUFSIZE];
/***********************************************
* Parse command line
***********************************************/
go = esl_getopts_Create(options);
if (esl_opt_ProcessCmdline(go, argc, argv) != eslOK ||
esl_opt_VerifyConfig(go) != eslOK)
{
printf("Failed to parse command line: %s\n", go->errbuf);
esl_usage(stdout, argv[0], usage);
printf("\nTo see more help on available options, do %s -h\n\n", argv[0]);
exit(1);
}
if (esl_opt_GetBoolean(go, "-h") )
{
esl_banner(stdout, argv[0], banner);
esl_usage (stdout, argv[0], usage);
puts("\n where options are:");
esl_opt_DisplayHelp(stdout, go, 1, 2, 80);
esl_opt_DisplayHelp(stdout, go, 2, 2, 80);
exit(EXIT_SUCCESS);
}
if (esl_opt_ArgNumber(go) != 2)
{
printf("Incorrect number of command line arguments.\n");
esl_usage(stdout, argv[0], usage);
printf("\nTo see more help on available options, do %s -h\n\n", argv[0]);
exit(1);
}
kfile = esl_opt_GetArg(go, 1);
tfile = esl_opt_GetArg(go, 2);
fmt = eslMSAFILE_STOCKHOLM;
/***********************************************
* Open the two Stockholm files.
***********************************************/
if (esl_opt_GetBoolean(go, "--amino")) abc = esl_alphabet_Create(eslAMINO);
else if (esl_opt_GetBoolean(go, "--dna")) abc = esl_alphabet_Create(eslDNA);
else if (esl_opt_GetBoolean(go, "--rna")) abc = esl_alphabet_Create(eslRNA);
if ( (kstatus = esl_msafile_Open(&abc, kfile, NULL, fmt, NULL, &kfp)) != eslOK) esl_msafile_OpenFailure(kfp, kstatus);
if ( (tstatus = esl_msafile_Open(&abc, tfile, NULL, fmt, NULL, &tfp)) != eslOK) esl_msafile_OpenFailure(tfp, tstatus);
do_post = esl_opt_GetBoolean(go, "-p");
/* read the mask file if --p-mask is enabled */
if(! esl_opt_IsDefault(go, "--p-mask")) {
if((status = read_mask_file(esl_opt_GetString(go, "--p-mask"), errbuf, &mask, &masklen)) != eslOK) esl_fatal(errbuf);
}
/* open the c2dfile for output, if nec */
if (esl_opt_IsOn(go, "--c2dfile")) {
if ((dfp = fopen(esl_opt_GetString(go, "--c2dfile"), "w")) == NULL) esl_fatal("Failed to open --c2dfile output file %s\n", esl_opt_GetString(go, "--c2dfile"));
}
/***********************************************
* Do alignment comparisons, one seq at a time;
* this means looping over all seqs in all alignments.
***********************************************/
nali = 0;
while ( (kstatus = esl_msafile_Read(kfp, &ka)) != eslEOF)
{
if ( kstatus != eslOK) esl_msafile_ReadFailure(kfp, kstatus);
if ( (tstatus = esl_msafile_Read(tfp, &ta)) != eslOK) esl_msafile_ReadFailure(tfp, tstatus);
nali++;
if((nali > 1) && (esl_opt_IsOn(go, "--c2dfile"))) esl_fatal("--c2dfile is only meant for msafiles with single alignments");
/* Sanity check on alignment
*/
if (ka->nseq != ta->nseq)
esl_fatal("trusted, test alignments don't have same seq #\n");
if (ka->rf == NULL)
esl_fatal("trusted alignment has no reference annotation\n");
if (ta->rf == NULL)
esl_fatal("test alignment has no reference annotation\n");
/* make sure the sequences are all identical */
ESL_ALLOC(seqlen, sizeof(int) * ka->nseq);
for(i = 0; i < ka->nseq; i++) {
if(strcmp(ka->sqname[i], ta->sqname[i]) != 0) esl_fatal("sequence %d of trusted alignment %s has different name than seq %d of predicted alignment %s\n", (i+1), ka->sqname[i], (i+1), ta->sqname[i]);
ESL_ALLOC(ks, sizeof(ESL_DSQ) * (ka->alen+2));
memcpy(ks, ka->ax[i], (ka->alen+2) * sizeof(ESL_DSQ));
esl_abc_XDealign(ka->abc, ks, ka->ax[i], &klen);
ESL_ALLOC(ts, sizeof(ESL_DSQ) * (ta->alen+2));
memcpy(ts, ta->ax[i], (ta->alen+2) * sizeof(ESL_DSQ));
esl_abc_XDealign(ta->abc, ts, ta->ax[i], &tlen);
if (tlen != klen)
esl_fatal("dealigned sequence mismatch, seq %d, when dealigned, is %d residues in the known alignment, but %d residues in the trusted alignment.", (i+1), klen, tlen);
if (memcmp(ks, ts, sizeof(ESL_DSQ) * klen) != 0)
esl_fatal("dealigned sequence mismatch, seq %d %s, when dealigned, are not identical.", (i+1), ka->sqname[i]);
seqlen[i] = tlen;
free(ks);
free(ts);
}
/* determine non-gap RF length */
rflen = 0;
for(apos = 1; apos <= ka->alen; apos++) {
if((! esl_abc_CIsGap (ka->abc, ka->rf[apos-1])) &&
(! esl_abc_CIsMissing(ka->abc, ka->rf[apos-1]))) rflen++;
}
t_rflen = 0;
for(apos = 1; apos <= ta->alen; apos++) {
if((! esl_abc_CIsGap (ta->abc, ta->rf[apos-1])) &&
(! esl_abc_CIsMissing (ta->abc, ta->rf[apos-1]))) t_rflen++;
}
if(t_rflen != rflen) esl_fatal("Trusted alignment non-gap RF length (%d) != predicted alignment non-gap RF length (%d).\n", rflen, t_rflen);
/* if -p, make sure the test alignment has posterior probabilities, and allocate our counters for correct/incorrect per post value */
if(do_post) {
if(! esl_opt_IsDefault(go, "--p-mask")) {
if(masklen != rflen) {
esl_fatal("Length of mask in %s (%d) not equal to non-gap RF len of alignments (%d)\n", esl_opt_GetString(go, "--p-mask"), masklen, rflen);
}
}
if(ta->pp == NULL) esl_fatal("-p requires \"#=GR PP\" annotation in the test alignment, but none exists");
ESL_ALLOC(ptm, sizeof(int) * npostvals);
ESL_ALLOC(pti, sizeof(int) * npostvals);
ESL_ALLOC(cor_ptm, sizeof(int) * npostvals);
ESL_ALLOC(cor_pti, sizeof(int) * npostvals);
esl_vec_ISet(ptm, npostvals, 0);
esl_vec_ISet(pti, npostvals, 0);
esl_vec_ISet(cor_ptm, npostvals, 0);
esl_vec_ISet(cor_pti, npostvals, 0);
}
/* allocate and initialize our counters */
ESL_ALLOC(kp, sizeof(int *) * ka->nseq);
ESL_ALLOC(tp, sizeof(int *) * ta->nseq);
for(i = 0; i < ka->nseq; i++) {
ESL_ALLOC(kp[i], sizeof(int) * (seqlen[i]+1));
ESL_ALLOC(tp[i], sizeof(int) * (seqlen[i]+1));
esl_vec_ISet(kp[i], seqlen[i]+1, -987654321);
esl_vec_ISet(tp[i], seqlen[i]+1, -987654321);
}
ESL_ALLOC(km_pos, sizeof(int) * (rflen+1));
ESL_ALLOC(ki_pos, sizeof(int) * (rflen+1));
ESL_ALLOC(tm_pos, sizeof(int) * (rflen+1));
ESL_ALLOC(ti_pos, sizeof(int) * (rflen+1));
ESL_ALLOC(cor_tm_pos, sizeof(int) * (rflen+1));
ESL_ALLOC(cor_ti_pos, sizeof(int) * (rflen+1));
esl_vec_ISet(km_pos, rflen+1, 0);
esl_vec_ISet(ki_pos, rflen+1, 0);
esl_vec_ISet(tm_pos, rflen+1, 0);
esl_vec_ISet(ti_pos, rflen+1, 0);
esl_vec_ISet(cor_tm_pos, rflen+1, 0);
esl_vec_ISet(cor_ti_pos, rflen+1, 0);
ESL_ALLOC(km_seq, sizeof(int) * ka->nseq);
ESL_ALLOC(ki_seq, sizeof(int) * ka->nseq);
ESL_ALLOC(tm_seq, sizeof(int) * ka->nseq);
ESL_ALLOC(ti_seq, sizeof(int) * ka->nseq);
ESL_ALLOC(cor_tm_seq, sizeof(int) * ka->nseq);
ESL_ALLOC(cor_ti_seq, sizeof(int) * ka->nseq);
esl_vec_ISet(km_seq, ka->nseq, 0);
esl_vec_ISet(ki_seq, ka->nseq, 0);
esl_vec_ISet(tm_seq, ka->nseq, 0);
esl_vec_ISet(ti_seq, ka->nseq, 0);
esl_vec_ISet(cor_tm_seq, ka->nseq, 0);
esl_vec_ISet(cor_ti_seq, ka->nseq, 0);
/* determine non-gap RF location of each residue in known alignment */
for(i = 0; i < ka->nseq; i++) {
uapos = rfpos = 0;
for(apos = 1; apos <= ka->alen; apos++) {
is_rfpos = FALSE;
if((! esl_abc_CIsGap (ka->abc, ka->rf[apos-1])) &&
(! esl_abc_CIsMissing (ka->abc, ka->rf[apos-1]))) {
rfpos++; is_rfpos = TRUE;
}
if(esl_abc_XIsResidue(ka->abc, ka->ax[i][apos])) {
uapos++;
kp[i][uapos] = (is_rfpos) ? rfpos : (-1 * rfpos);
if(is_rfpos) { km_pos[rfpos]++; km_seq[i]++; }
else { ki_pos[rfpos]++; ki_seq[i]++; }
}
}
}
/* determine non-gap RF location of each residue in predicted alignment */
for(i = 0; i < ta->nseq; i++) {
uapos = rfpos = 0;
for(apos = 1; apos <= ta->alen; apos++) {
is_rfpos = FALSE;
if((! esl_abc_CIsGap (abc, ta->rf[apos-1])) &&
(! esl_abc_CIsMissing (abc, ta->rf[apos-1]))) {
rfpos++; is_rfpos = TRUE;
if(do_post) {
do_post_for_this_rfpos = (mask != NULL && mask[rfpos-1] == '0') ? FALSE : TRUE;
}
}
if(esl_abc_XIsResidue(ta->abc, ta->ax[i][apos])) {
uapos++;
tp[i][uapos] = (is_rfpos) ? rfpos : (-1 * rfpos);
if(do_post) {
if(esl_abc_CIsGap(abc, ta->pp[i][(apos-1)])) esl_fatal("gap PP value for nongap residue: ali: %d seq: %d apos: %d\n", nali, i, apos);
ppidx = get_pp_idx(abc, ta->pp[i][(apos-1)]);
if(ppidx == -1) esl_fatal("unrecognized PP value (%c) for nongap residue: ali: %d seq: %d apos: %d\n", ta->pp[i][(apos-1)], nali, i, apos);
}
if(is_rfpos) {
tm_pos[rfpos]++; tm_seq[i]++;
if(do_post_for_this_rfpos) ptm[ppidx]++;
}
else {
ti_pos[rfpos]++; ti_seq[i]++;
if(do_post) pti[ppidx]++;
}
if(kp[i][uapos] == tp[i][uapos]) { /* correctly predicted this residue */
if(is_rfpos) {
cor_tm_seq[i]++; cor_tm_pos[rfpos]++;
if(do_post_for_this_rfpos) cor_ptm[ppidx]++;
}
else {
cor_ti_seq[i]++; cor_ti_pos[rfpos]++;
if(do_post) cor_pti[ppidx]++;
}
}
}
}
}
if((! (esl_opt_GetBoolean(go, "-c"))) && (! esl_opt_GetBoolean(go, "-p"))) {
/* print per sequence statistics */
/* determine the longest name in msa */
for(ni = 0; ni < ka->nseq; ni++) namewidth = ESL_MAX(namewidth, strlen(ka->sqname[ni]));
ESL_ALLOC(namedashes, sizeof(char) * namewidth+1);
namedashes[namewidth] = '\0';
for(ni = 0; ni < namewidth; ni++) namedashes[ni] = '-';
printf("# %-*s %6s %28s %28s %28s\n", namewidth, "seq name", "len", "match columns", "insert columns", "all columns");
printf("# %-*s %6s %28s %28s %28s\n", namewidth, namedashes, "------", "----------------------------", "----------------------------", "----------------------------");
for(i = 0; i < ta->nseq; i++) {
printf(" %-*s %6d %8d / %8d (%.3f) %8d / %8d (%.3f) %8d / %8d (%.3f)\n", namewidth, ka->sqname[i], seqlen[i],
cor_tm_seq[i], km_seq[i], (km_seq[i] == 0) ? 0. : ((float) cor_tm_seq[i] / (float) km_seq[i]),
cor_ti_seq[i], ki_seq[i], (ki_seq[i] == 0) ? 0. : ((float) cor_ti_seq[i] / (float) ki_seq[i]),
(cor_tm_seq[i] + cor_ti_seq[i]), (km_seq[i] + ki_seq[i]), ((float) (cor_tm_seq[i] + cor_ti_seq[i]) / ((float) km_seq[i] + ki_seq[i])));
}
cor_tm = esl_vec_ISum(cor_tm_seq, ka->nseq);
cor_ti = esl_vec_ISum(cor_ti_seq, ka->nseq);
km = esl_vec_ISum(km_seq, ka->nseq);
ki = esl_vec_ISum(ki_seq, ka->nseq);
printf("# %-*s %6s %28s %28s %28s\n", namewidth, namedashes, "-----", "----------------------------", "----------------------------", "----------------------------");
printf("# %-*s %6s %8d / %8d (%.3f) %8d / %8d (%.3f) %8d / %8d (%.3f)\n",
namewidth, "*all*", "-",
cor_tm, km, ((float) cor_tm / (float) km),
cor_ti, ki, ((float) cor_ti / (float) ki),
(cor_tm+cor_ti), (km+ki), (((float) (cor_tm + cor_ti))/ ((float) (km + ki))));
free(namedashes);
for(i = 0; i < ka->nseq; i++) {
free(kp[i]);
free(tp[i]);
}
}
else if(esl_opt_GetBoolean(go, "-c")) { /* print per column statistics */
printf("# %5s %20s %20s %20s\n", "rfpos", "match", "insert", "both");
printf("# %5s %20s %20s %20s\n", "-----", "--------------------", "--------------------", "--------------------");
for(rfpos = 0; rfpos <= rflen; rfpos++) {
printf(" %5d %4d / %4d (%.3f) %4d / %4d (%.3f) %4d / %4d (%.3f)\n", rfpos,
cor_tm_pos[rfpos], km_pos[rfpos], (km_pos[rfpos] == 0) ? 0. : ((float) cor_tm_pos[rfpos] / (float) km_pos[rfpos]),
cor_ti_pos[rfpos], ki_pos[rfpos], (ki_pos[rfpos] == 0) ? 0. : ((float) cor_ti_pos[rfpos] / (float) ki_pos[rfpos]),
(cor_tm_pos[rfpos] + cor_ti_pos[rfpos]), (km_pos[rfpos] + ki_pos[rfpos]), ((float) (cor_tm_pos[rfpos] + cor_ti_pos[rfpos]) / ((float) km_pos[rfpos] + ki_pos[rfpos])));
}
}
else if(do_post) { /* do posterior output */
if(mask == NULL) {
printf("# %2s %29s %29s\n", "", " match columns ", " insert columns ");
printf("# %2s %29s %29s\n", "", "-----------------------------", "-----------------------------") ;
printf("# %2s %8s %8s %9s %8s %8s %9s\n", "PP", "ncorrect", "ntotal", "fractcor", "ncorrect", "ntotal", "fractcor");
printf("# %2s %8s %8s %9s %8s %8s %9s\n", "--", "--------", "--------", "---------", "--------", "--------", "---------");
}
else {
printf("# %2s %29s %29s\n", "", " match columns within mask ", " insert columns ");
printf("# %2s %29s %29s\n", "", "-----------------------------", "-----------------------------") ;
printf("# %2s %8s %8s %9s %8s %8s %9s\n", "PP", "ncorrect", "ntotal", "fractcor", "ncorrect", "ntotal", "fractcor");
printf("# %2s %8s %8s %9s %8s %8s %9s\n", "--", "--------", "--------", "---------", "--------", "--------", "---------");
}
cm_ptm = cm_pti = cm_cor_ptm = cm_cor_pti = cm_incor_ptm = cm_incor_pti = 0;
//tot_ptm = esl_vec_ISum(ptm, npostvals);
//tot_pti = esl_vec_ISum(pti, npostvals);
//tot_cor_ptm = esl_vec_ISum(cor_ptm, npostvals);
//tot_cor_pti = esl_vec_ISum(cor_pti, npostvals);
//tot_incor_ptm = tot_ptm - tot_cor_ptm;
//tot_incor_pti = tot_pti - tot_cor_pti;
for(p = (npostvals-1); p >= 0; p--) {
cm_cor_ptm += cor_ptm[p];
cm_cor_pti += cor_pti[p];
cm_ptm += ptm[p];
cm_pti += pti[p];
cm_incor_ptm += ptm[p] - cor_ptm[p];
cm_incor_pti += pti[p] - cor_pti[p];
printf(" %2c %8d / %8d (%.5f) %8d / %8d (%.5f)\n",
ppchars[p], cor_ptm[p], ptm[p],
(ptm[p] == 0) ? 0. : (float) cor_ptm[p] / (float) ptm[p],
cor_pti[p], pti[p],
(pti[p] == 0) ? 0. : (float) cor_pti[p] / (float) pti[p]);
}
}
/* handle --c2dfile */
if (dfp != NULL) {
/* match stats, 4 fields, CMYK color values */
for(rfpos = 1; rfpos <= rflen; rfpos++) {
if(km_pos[rfpos] == 0) { /* special case, no known alignment residues, a blank position */
fprintf(dfp, "%.3f %.3f %.3f %.3f\n", 0., 0., 0., 0.);
}
else {
fprintf(dfp, "%.3f %.3f %.3f %.3f\n",
0., /* cyan */
1. - ((float) cor_tm_pos[rfpos] / (float) km_pos[rfpos]), /* magenta, fraction incorrect */
1. - ((float) km_pos[rfpos] / ta->nseq), /* yellow, 1 - fraction of seqs with residue in column */
0.);
}
}
fprintf(dfp, "//\n");
/* insert stats, 4 fields, CMYK color values */
rfpos = 0; /* special case, combine insert posn 0 and 1 together */
if(ki_pos[rfpos] == 0) { /* special case, no known alignment residues, a blank position */
fprintf(dfp, "%.3f %.3f %.3f %.3f\n", 0., 0., 0., 0.);
}
else {
fprintf(dfp, "%.3f %.3f %.3f %.3f\n",
0., /* cyan */
1. - ((float) (cor_ti_pos[0] + cor_ti_pos[1]) / ((float) (ki_pos[0] + ki_pos[1]))), /* magenta, fraction correct */
0.,
0.);
}
/* insert stats posn 2..rflen */
for(rfpos = 2; rfpos <= rflen; rfpos++) {
if(ki_pos[rfpos] == 0) { /* special case, no known alignment residues, a blank position */
fprintf(dfp, "%.3f %.3f %.3f %.3f\n", 0., 0., 0., 0.);
}
else {
fprintf(dfp, "%.3f %.3f %.3f %.3f\n",
0., /* cyan */
1. - ((float) cor_ti_pos[rfpos] / (float) ki_pos[rfpos]), /* magenta, fraction correct */
0.,
0.);
}
}
fprintf(dfp, "//\n");
}
if(ptm != NULL) free(ptm);
if(pti != NULL) free(pti);
if(cor_ptm != NULL) free(cor_ptm);
if(cor_ptm != NULL) free(cor_pti);
free(kp);
free(tp);
free(km_seq);
free(ki_seq);
free(tm_seq);
free(ti_seq);
free(cor_tm_seq);
free(cor_ti_seq);
free(km_pos);
free(ki_pos);
free(tm_pos);
free(ti_pos);
free(cor_tm_pos);
free(cor_ti_pos);
free(seqlen);
esl_msa_Destroy(ka);
esl_msa_Destroy(ta);
}
if(mask != NULL) free(mask);
if(dfp != NULL) {
fclose(dfp);
printf("# Draw file of per-column stats saved to file: %s\n", esl_opt_GetString(go, "--c2dfile"));
}
if(abc) esl_alphabet_Destroy(abc);
esl_getopts_Destroy(go);
esl_msafile_Close(tfp);
esl_msafile_Close(kfp);
return 0;
ERROR:
return status;
}
/* Function: p7_alidisplay_Backconvert()
* Synopsis: Convert an alidisplay to a faux trace and subsequence.
* Incept: SRE, Wed Dec 10 09:49:28 2008 [Janelia]
*
* Purpose: Convert alignment display object <ad> to a faux subsequence
* and faux subsequence trace, returning them in <ret_sq> and
* <ret_tr>.
*
* The subsequence <*ret_sq> is digital; ascii residues in
* <ad> are digitized using digital alphabet <abc>.
*
* The subsequence and trace are suitable for passing as
* array elements to <p7_MultipleAlignment>. This is the
* main purpose of backconversion. Results of a profile
* search are stored in a hit list as a processed
* <P7_ALIDISPLAY>, not as a <P7_TRACE> and <ESL_SQ>, to
* reduce space and to reduce communication overhead in
* parallelized search implementations. After reduction
* to a final hit list, a master may want to construct a
* multiple alignment of all the significant hits.
*
* Returns: <eslOK> on success.
*
* Throws: <eslEMEM> on allocation failures. <eslECORRUPT> on unexpected internal
* data corruption. On any exception, <*ret_sq> and <*ret_tr> are
* <NULL>.
*
* Xref: J4/29.
*/
int
p7_alidisplay_Backconvert(const P7_ALIDISPLAY *ad, const ESL_ALPHABET *abc, ESL_SQ **ret_sq, P7_TRACE **ret_tr)
{
ESL_SQ *sq = NULL; /* RETURN: faux subsequence */
P7_TRACE *tr = NULL; /* RETURN: faux trace */
int subL = 0; /* subsequence length in the <ad> */
int a, i, k; /* coords for <ad>, <sq->dsq>, model */
char st; /* state type: MDI */
int status;
/* Make a first pass over <ad> just to calculate subseq length */
for (a = 0; a < ad->N; a++)
if (! esl_abc_CIsGap(abc, ad->aseq[a])) subL++;
/* Allocations */
if ((sq = esl_sq_CreateDigital(abc)) == NULL) { status = eslEMEM; goto ERROR; }
if ((status = esl_sq_GrowTo(sq, subL)) != eslOK) goto ERROR;
if ((tr = (ad->ppline == NULL) ? p7_trace_Create() : p7_trace_CreateWithPP()) == NULL) { status = eslEMEM; goto ERROR; }
if ((status = p7_trace_GrowTo(tr, subL+6)) != eslOK) goto ERROR; /* +6 is for SNB/ECT */
/* Construction of dsq, trace */
sq->dsq[0] = eslDSQ_SENTINEL;
if ((status = ((ad->ppline == NULL) ? p7_trace_Append(tr, p7T_S, 0, 0) : p7_trace_AppendWithPP(tr, p7T_S, 0, 0, 0.0))) != eslOK) goto ERROR;
if ((status = ((ad->ppline == NULL) ? p7_trace_Append(tr, p7T_N, 0, 0) : p7_trace_AppendWithPP(tr, p7T_N, 0, 0, 0.0))) != eslOK) goto ERROR;
if ((status = ((ad->ppline == NULL) ? p7_trace_Append(tr, p7T_B, 0, 0) : p7_trace_AppendWithPP(tr, p7T_B, 0, 0, 0.0))) != eslOK) goto ERROR;
k = ad->hmmfrom;
i = 1;
for (a = 0; a < ad->N; a++)
{
if (esl_abc_CIsResidue(abc, ad->model[a])) { st = (esl_abc_CIsResidue(abc, ad->aseq[a]) ? p7T_M : p7T_D); } else st = p7T_I;
if ((status = ((ad->ppline == NULL) ? p7_trace_Append(tr, st, k, i) : p7_trace_AppendWithPP(tr, st, k, i, p7_alidisplay_DecodePostProb(ad->ppline[a])))) != eslOK) goto ERROR;
switch (st) {
case p7T_M: sq->dsq[i] = esl_abc_DigitizeSymbol(abc, ad->aseq[a]); k++; i++; break;
case p7T_I: sq->dsq[i] = esl_abc_DigitizeSymbol(abc, ad->aseq[a]); i++; break;
case p7T_D: k++; break;
}
}
if ((status = ((ad->ppline == NULL) ? p7_trace_Append(tr, p7T_E, 0, 0) : p7_trace_AppendWithPP(tr, p7T_E, 0, 0, 0.0))) != eslOK) goto ERROR;
if ((status = ((ad->ppline == NULL) ? p7_trace_Append(tr, p7T_C, 0, 0) : p7_trace_AppendWithPP(tr, p7T_C, 0, 0, 0.0))) != eslOK) goto ERROR;
if ((status = ((ad->ppline == NULL) ? p7_trace_Append(tr, p7T_T, 0, 0) : p7_trace_AppendWithPP(tr, p7T_T, 0, 0, 0.0))) != eslOK) goto ERROR;
sq->dsq[i] = eslDSQ_SENTINEL;
/* some sanity checks */
if (tr->N != ad->N + 6) ESL_XEXCEPTION(eslECORRUPT, "backconverted trace ended up with unexpected size (%s/%s)", ad->sqname, ad->hmmname);
if (k != ad->hmmto + 1) ESL_XEXCEPTION(eslECORRUPT, "backconverted trace didn't end at expected place on model (%s/%s)", ad->sqname, ad->hmmname);
if (i != subL + 1) ESL_XEXCEPTION(eslECORRUPT, "backconverted subseq didn't end at expected length (%s/%s)", ad->sqname, ad->hmmname);
/* Set up <sq> annotation as a subseq of a source sequence */
if ((status = esl_sq_FormatName(sq, "%s/%ld-%ld", ad->sqname, ad->sqfrom, ad->sqto)) != eslOK) goto ERROR;
if ((status = esl_sq_FormatDesc(sq, "[subseq from] %s", ad->sqdesc[0] != '\0' ? ad->sqdesc : ad->sqname)) != eslOK) goto ERROR;
if ((status = esl_sq_SetSource (sq, ad->sqname)) != eslOK) goto ERROR;
if (ad->sqacc[0] != '\0') { if ((status = esl_sq_SetAccession (sq, ad->sqacc)) != eslOK) goto ERROR; }
sq->n = subL;
sq->start = ad->sqfrom;
sq->end = ad->sqto;
sq->C = 0;
sq->W = subL;
sq->L = ad->L;
tr->M = ad->M;
tr->L = ad->L;
*ret_sq = sq;
*ret_tr = tr;
return eslOK;
ERROR:
if (sq != NULL) esl_sq_Destroy(sq);
if (tr != NULL) p7_trace_Destroy(tr);
*ret_sq = NULL;
*ret_tr = NULL;
return status;
}
