/* Function: p7_Tau()
* Synopsis: Determine Forward tau by brief simulation.
* Incept: SRE, Thu Aug 9 15:08:39 2007 [Janelia]
*
* Purpose: Determine the <tau> parameter for an exponential tail fit
* to the Forward score distribution for model <om>, on
* random sequences with the composition of the background
* model <bg>. This <tau> parameter is for an exponential
* distribution anchored from $P=1.0$, so it's not really a
* tail per se; but it's only an accurate fit in the tail
* of the Forward score distribution, from about $P=0.001$
* or so.
*
* The determination of <tau> is done by a brief simulation
* in which we fit a Gumbel distribution to a small number
* of Forward scores of random sequences, and use that to
* predict the location of the tail at probability <tailp>.
*
* The Gumbel is of course inaccurate, but we can use it
* here solely as an empirical distribution to determine
* the location of a reasonable <tau> more accurately on a
* smaller number of samples than we could do with raw
* order statistics.
*
* Typical choices are L=100, N=200, tailp=0.04, which
* typically yield estimates $\hat{\mu}$ with a precision
* (standard deviation) of $\pm$ 0.2 bits, corresponding to
* a $\pm$ 15\% error in E-values. See [J1/135].
*
* The use of Gumbel fitting to a small number of $N$
* samples and the extrapolation of $\hat{\mu}$ from the
* estimated location of the 0.04 tail mass are both
* empirical and carefully optimized against several
* tradeoffs. Most importantly, around this choice of tail
* probability, a systematic error introduced by the use of
* the Gumbel fit is being cancelled by systematic error
* introduced by the use of a higher tail probability than
* the regime in which the exponential tail is a valid
* approximation. See [J1/135] for discussion.
*
* This function changes the length configuration of both
* <om> and <bg>. The caller must remember to reconfigure
* both of their length models appropriately for any
* subsequent alignments.
*
* Args: r : source of randomness
* om : configured profile to sample sequences from
* bg : null model (for background residue frequencies)
* L : mean length model for seq emission from profile
* N : number of sequences to generate
* lambda : expected slope of the exponential tail (from p7_Lambda())
* tailp : tail mass from which we will extrapolate mu
* ret_mu : RETURN: estimate for the Forward mu (base of exponential tail)
*
* Returns: <eslOK> on success, and <*ret_fv> is the score difference
* in bits.
*
* Throws: <eslEMEM> on allocation error, and <*ret_fv> is 0.
*/
int
p7_Tau(ESL_RANDOMNESS *r, P7_OPROFILE *om, P7_BG *bg, int L, int N, double lambda, double tailp, double *ret_tau)
{
P7_OMX *ox = p7_omx_Create(om->M, 0, L); /* DP matrix: for ForwardParser, L rows */
ESL_DSQ *dsq = NULL;
double *xv = NULL;
float fsc, nullsc;
double gmu, glam;
int status;
int i;
ESL_ALLOC(xv, sizeof(double) * N);
ESL_ALLOC(dsq, sizeof(ESL_DSQ) * (L+2));
if (ox == NULL) { status = eslEMEM; goto ERROR; }
p7_oprofile_ReconfigLength(om, L);
p7_bg_SetLength(bg, L);
for (i = 0; i < N; i++)
{
if ((status = esl_rsq_xfIID(r, bg->f, om->abc->K, L, dsq)) != eslOK) goto ERROR;
if ((status = p7_ForwardParser(dsq, L, om, ox, &fsc)) != eslOK) goto ERROR;
if ((status = p7_bg_NullOne(bg, dsq, L, &nullsc)) != eslOK) goto ERROR;
xv[i] = (fsc - nullsc) / eslCONST_LOG2;
}
if ((status = esl_gumbel_FitComplete(xv, N, &gmu, &glam)) != eslOK) goto ERROR;
/* Explanation of the eqn below: first find the x at which the Gumbel tail
* mass is predicted to be equal to tailp. Then back up from that x
* by log(tailp)/lambda to set the origin of the exponential tail to 1.0
* instead of tailp.
*/
*ret_tau = esl_gumbel_invcdf(1.0-tailp, gmu, glam) + (log(tailp) / lambda);
free(xv);
free(dsq);
p7_omx_Destroy(ox);
return eslOK;
ERROR:
*ret_tau = 0.;
if (xv != NULL) free(xv);
if (dsq != NULL) free(dsq);
if (ox != NULL) p7_omx_Destroy(ox);
return status;
}
static int
output_result(ESL_GETOPTS *go, struct cfg_s *cfg, char *errbuf, P7_HMM *hmm, double *scores, int *alilens)
{
ESL_HISTOGRAM *h = NULL;
int i;
double tailp;
double x10;
double mu, lambda, E10;
double mufix, E10fix;
double mufix2, E10fix2;
double E10p;
double almean, alvar; /* alignment length mean and variance (optional output) */
double pmu, plambda;
int status;
/* fetch statistical params from HMM for expected distribution */
if (esl_opt_GetBoolean(go, "--vit")) { pmu = hmm->evparam[p7_VMU]; plambda = hmm->evparam[p7_VLAMBDA]; }
else if (esl_opt_GetBoolean(go, "--msv")) { pmu = hmm->evparam[p7_MMU]; plambda = hmm->evparam[p7_MLAMBDA]; }
else if (esl_opt_GetBoolean(go, "--fwd")) { pmu = hmm->evparam[p7_FTAU]; plambda = hmm->evparam[p7_FLAMBDA]; }
/* Optional output of scores/alignment lengths: */
if (cfg->xfp) fwrite(scores, sizeof(double), cfg->N, cfg->xfp);
if (cfg->alfp) for (i = 0; i < cfg->N; i++) fprintf(cfg->alfp, "%d %.3f\n", alilens[i], scores[i]);
if (esl_opt_GetBoolean(go, "-v")) for (i = 0; i < cfg->N; i++) printf("%.3f\n", scores[i]);
/* optional "filter power" data file: <hmm name> <# seqs <= P threshold> <fraction of seqs <= P threshold> */
if (cfg->ffp) output_filter_power(go, cfg, errbuf, hmm, scores);
/* Count the scores into a histogram object. */
if ((h = esl_histogram_CreateFull(-50., 50., 0.2)) == NULL) ESL_XFAIL(eslEMEM, errbuf, "allocation failed");
for (i = 0; i < cfg->N; i++) esl_histogram_Add(h, scores[i]);
/* For viterbi, MSV, and hybrid, fit data to a Gumbel, either with known lambda or estimated lambda. */
if (esl_opt_GetBoolean(go, "--vit") || esl_opt_GetBoolean(go, "--msv"))
{
esl_histogram_GetRank(h, 10, &x10);
tailp = 1.0;
/* mu, lambda, E10 fields are for ML Gumbel fit to the observed data */
if (esl_gumbel_FitComplete(scores, cfg->N, &mu, &lambda) != eslOK) esl_fatal("gumbel complete data fit failed");
E10 = cfg->N * esl_gumbel_surv(x10, mu, lambda);
/* mufix, E10fix fields: assume lambda = log2; fit an ML mu to the data */
if (esl_gumbel_FitCompleteLoc(scores, cfg->N, 0.693147, &mufix) != eslOK) esl_fatal("gumbel mu- (location-)only data fit failed for lambda = log2");
E10fix = cfg->N * esl_gumbel_surv(x10, mufix, 0.693147);
/* mufix2, E10fix2 fields: assume H3's own lambda estimate; fit ML mu */
if (esl_gumbel_FitCompleteLoc(scores, cfg->N, plambda, &mufix2) != eslOK) esl_fatal("gumbel mu- (location-)only data fit failed for fitted lambda");
E10fix2 = cfg->N * esl_gumbel_surv(x10, mufix2, plambda);
/* pmu, plambda, E10p: use H3 expectation estimates (pmu, plambda) */
E10p = cfg->N * esl_gumbel_surv(x10, pmu, plambda);
fprintf(cfg->ofp, "%-20s %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f",
hmm->name, tailp, mu, lambda, E10, mufix, E10fix, mufix2, E10fix2, pmu, plambda, E10p);
if (esl_opt_GetBoolean(go, "-a")) {
esl_stats_IMean(alilens, cfg->N, &almean, &alvar);
fprintf(cfg->ofp, " %8.4f %8.4f\n", almean, sqrt(alvar));
} else
fprintf(cfg->ofp, "\n");
if (cfg->survfp != NULL) {
double xmax = esl_opt_IsOn(go, "--xmax") ? esl_opt_GetReal(go, "--xmax") : h->xmax + 5.;
esl_histogram_PlotSurvival(cfg->survfp, h);
esl_gumbel_Plot(cfg->survfp, pmu, plambda, esl_gumbel_surv, h->xmin - 5., xmax, 0.1);
esl_gumbel_Plot(cfg->survfp, mu, lambda, esl_gumbel_surv, h->xmin - 5., xmax, 0.1);
esl_gumbel_Plot(cfg->survfp, mufix, 0.693147, esl_gumbel_surv, h->xmin - 5., xmax, 0.1);
}
if (cfg->efp != NULL) {
double x;
fprintf(cfg->efp, "# %s\n", hmm->name);
for (i = 1; i <= 1000 && i <= cfg->N; i++) {
esl_histogram_GetRank(h, i, &x);
fprintf(cfg->efp, "%d %g\n", i, cfg->N * esl_gumbel_surv(x, pmu, plambda));
}
fprintf(cfg->efp, "&\n");
}
}
/* For Forward, fit tail to exponential tails, for a range of tail mass choices. */
else if (esl_opt_GetBoolean(go, "--fwd"))
{
double tmin = esl_opt_GetReal(go, "--tmin");
double tmax = esl_opt_GetReal(go, "--tmax");
double tpoints = (double) esl_opt_GetInteger(go, "--tpoints");
int do_linear = esl_opt_GetBoolean(go, "--tlinear");
double *xv;
double tau;
int n;
esl_histogram_GetRank(h, 10, &x10);
tailp = tmin;
do {
if (tailp > 1.0) tailp = 1.0;
esl_histogram_GetTailByMass(h, tailp, &xv, &n, NULL);
if (esl_exp_FitComplete(xv, n, &mu, &lambda) != eslOK) esl_fatal("exponential fit failed");
E10 = cfg->N * tailp * esl_exp_surv(x10, mu, lambda);
mufix = mu;
E10fix = cfg->N * tailp * esl_exp_surv(x10, mu, 0.693147);
E10p = cfg->N * esl_exp_surv(x10, pmu, plambda); /* the pmu is relative to a P=1.0 tail origin. */
tau = mu + log(tailp) / lambda;
fprintf(cfg->ofp, "%-20s %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f\n",
hmm->name, tailp, mu, lambda, E10, mufix, E10fix, pmu, plambda, E10p);
if (tpoints == 1) break;
else if (do_linear) tailp += (tmax-tmin) / (tpoints-1);
else tailp *= exp(log(tmax/tmin) / (tpoints-1));
} while (tailp <= tmax+1e-7);
if (cfg->survfp)
{
double xmax = esl_opt_IsOn(go, "--xmax") ? esl_opt_GetReal(go, "--xmax") : h->xmax + 5.;
esl_histogram_PlotSurvival(cfg->survfp, h);
esl_exp_Plot(cfg->survfp, pmu, plambda, esl_exp_surv, pmu, xmax, 0.1);
esl_exp_Plot(cfg->survfp, tau, lambda, esl_exp_surv, tau, xmax, 0.1);
esl_exp_Plot(cfg->survfp, tau, 0.693147, esl_exp_surv, tau, xmax, 0.1);
}
if (cfg->efp != NULL) {
double x;
fprintf(cfg->efp, "# %s\n", hmm->name);
for (i = 1; i <= 1000 && i <= cfg->N; i++) {
esl_histogram_GetRank(h, i, &x);
fprintf(cfg->efp, "%d %g\n", i, cfg->N * esl_exp_surv(x, pmu, plambda));
}
fprintf(cfg->efp, "&\n");
}
}
/* fallthrough: both normal, error cases execute same cleanup code */
status = eslOK;
ERROR:
esl_histogram_Destroy(h);
return status;
}
