/* Function: ExtremeValueFitHistogram()
* Date: SRE, Sat Nov 15 17:16:15 1997 [St. Louis]
*
* Purpose: Fit a score histogram to the extreme value
* distribution. Set the parameters lambda
* and mu in the histogram structure. Calculate
* a chi-square test as a measure of goodness of fit.
*
* Methods: Uses a maximum likelihood method [Lawless82].
* Lower outliers are removed by censoring the data below the peak.
* Upper outliers are removed iteratively using method
* described by [Mott92].
*
* Args: h - histogram to fit
* censor - TRUE to censor data left of the peak
* high_hint - score cutoff; above this are `real' hits that aren't fit
*
* Return: 1 if fit is judged to be valid.
* else 0 if fit is invalid (too few seqs.)
*/
int
ExtremeValueFitHistogram(struct histogram_s *h, int censor, float high_hint)
{
float *x; /* array of EVD samples to fit */
int *y; /* histogram counts */
int n; /* number of observed samples */
int z; /* number of censored samples */
int hsize; /* size of histogram */
float lambda, mu; /* new estimates of lambda, mu */
int sc; /* loop index for score */
int lowbound; /* lower bound of fitted region*/
int highbound; /* upper bound of fitted region*/
int new_highbound;
int iteration;
/* Determine lower bound on fitted region;
* if we're censoring the data, choose the peak of the histogram.
* if we're not, then we take the whole histogram.
*/
lowbound = h->lowscore;
if (censor)
{
int max = -1;
for (sc = h->lowscore; sc <= h->highscore; sc++)
if (h->histogram[sc - h->min] > max)
{
max = h->histogram[sc - h->min];
lowbound = sc;
}
}
/* Determine initial upper bound on fitted region.
*/
highbound = MIN(high_hint, h->highscore);
/* Now, iteratively converge on our lambda, mu:
*/
for (iteration = 0; iteration < 100; iteration++)
{
/* Construct x, y vectors.
*/
x = NULL;
y = NULL;
hsize = highbound - lowbound + 1;
if (hsize < 5) goto FITFAILED; /* require at least 5 bins or we don't fit */
x = MallocOrDie(sizeof(float) * hsize);
y = MallocOrDie(sizeof(int) * hsize);
n = 0;
for (sc = lowbound; sc <= highbound; sc++)
{
x[sc-lowbound] = (float) sc + 0.5; /* crude, but tests OK */
y[sc-lowbound] = h->histogram[sc - h->min];
n += h->histogram[sc - h->min];
}
if (n < 100) goto FITFAILED; /* require fitting to at least 100 points */
/* If we're censoring, estimate z, the number of censored guys
* left of the bound. Our initial estimate is crudely that we're
* missing e^-1 of the total distribution (which would be exact
* if we censored exactly at mu; but we censored at the observed peak).
* Subsequent estimates are more exact based on our current estimate of mu.
*/
if (censor)
{
if (iteration == 0)
z = MIN(h->total-n, (int) (0.58198 * (float) n));
else
{
double psx;
psx = EVDDistribution((float) lowbound, mu, lambda);
z = MIN(h->total-n, (int) ((double) n * psx / (1. - psx)));
}
}
/* Do an ML fit
*/
if (censor) {
if (! EVDCensoredFit(x, y, hsize, z, (float) lowbound, &mu, &lambda))
goto FITFAILED;
} else
if (! EVDMaxLikelyFit(x, y, hsize, &mu, &lambda))
goto FITFAILED;
/* Find the Eval = 1 point as a new highbound;
* the total number of samples estimated to "belong" to the EVD is n+z
*/
new_highbound = (int)
(mu - (log (-1. * log((double) (n+z-1) / (double)(n+z))) / lambda));
free(x);
free(y);
if (new_highbound >= highbound) break;
highbound = new_highbound;
}
/* Set the histogram parameters;
* - we fit from lowbound to highbound; thus we lose 2 degrees of freedom
* for fitting mu, lambda, but we get 1 back because we're unnormalized
* in this interval, hence we pass 2-1 = 1 as ndegrees.
*/
ExtremeValueSetHistogram(h, mu, lambda, lowbound, highbound, 1);
return 1;
FITFAILED:
UnfitHistogram(h);
if (x != NULL) free(x);
if (y != NULL) free(y);
return 0;
}
int ExtremeValueFitHistogram(Histogram * h, int censor, float high_hint)
{
float *x; /* array of EVD samples to fit */
int *y; /* histogram counts */
int n; /* number of observed samples */
int z; /* number of censored samples */
int hsize; /* size of histogram */
float lambda, mu; /* new estimates of lambda, mu */
int sc; /* loop index for score */
int lowbound; /* lower bound of fitted region*/
int highbound; /* upper bound of fitted region*/
int new_highbound;
int iteration;
/* Determine lower bound on fitted region;
* if we're censoring the data, choose the peak of the histogram.
* if we're not, then we take the whole histogram.
*/
lowbound = h->lowscore;
if (censor)
{
int max = -1;
for (sc = h->lowscore; sc <= h->highscore; sc++)
if (h->histogram[sc - h->min] > max)
{
max = h->histogram[sc - h->min];
lowbound = sc;
}
}
/* Determine initial upper bound on fitted region.
*/
highbound = MIN(high_hint, h->highscore);
/* Now, iteratively converge on our lambda, mu:
*/
for (iteration = 0; iteration < 100; iteration++)
{
/* Construct x, y vectors.
*/
x = NULL;
y = NULL;
hsize = highbound - lowbound + 1;
if (hsize < 5) {
warn("On iteration %d, got %d bins, which is not fitable",iteration,hsize);
goto FITFAILED; /* require at least 5 bins or we don't fit */
}
x = ckalloc(sizeof(float) * hsize);
y = ckalloc(sizeof(int) * hsize);
if( x == NULL || y == NULL ) {
warn("Out of temporary memory for evd fitting... exiting with error, though I'd worry about this");
return 0;
}
n = 0;
for (sc = lowbound; sc <= highbound; sc++)
{
x[sc-lowbound] = (float) sc + 0.5; /* crude, but tests OK */
y[sc-lowbound] = h->histogram[sc - h->min];
n += h->histogram[sc - h->min];
}
if (n < 100) {
warn("On iteration %d, got only %d points, which is not fitable",iteration,n);
goto FITFAILED; /* require fitting to at least 100 points */
}
/* If we're censoring, estimate z, the number of censored guys
* left of the bound. Our initial estimate is crudely that we're
* missing e^-1 of the total distribution (which would be exact
* if we censored exactly at mu; but we censored at the observed peak).
* Subsequent estimates are more exact based on our current estimate of mu.
*/
if (censor)
{
if (iteration == 0)
z = MIN(h->total-n, (int) (0.58198 * (float) n));
else
{
double psx;
psx = EVDDistribution((float) lowbound, mu, lambda);
z = MIN(h->total-n, (int) ((double) n * psx / (1. - psx)));
}
}
/* Do an ML fit
*/
if (censor) {
if (! EVDCensoredFit(x, y, hsize, z, (float) lowbound, &mu, &lambda)) {
warn("On iteration %d, unable to make maxlikehood evd fit with censor",iteration);
goto FITFAILED;
}
} else {
if (! EVDMaxLikelyFit(x, y, hsize, &mu, &lambda)) {
warn("On iteration %d, unable to make maxlikehood evd fit without censor",iteration);
goto FITFAILED;
}
}
/* Find the Eval = 1 point as a new highbound;
* the total number of samples estimated to "belong" to the EVD is n+z
*/
new_highbound = (int)
(mu - (log (-1. * log((double) (n+z-1) / (double)(n+z))) / lambda));
free(x);
free(y);
if (new_highbound >= highbound) break;
highbound = new_highbound;
}
/* Set the histogram parameters;
* - the wonka factor is n+z / h->total : e.g. that's the fraction of the
* hits that we expect to match the EVD, others are generally lower
* - we fit from lowbound to highbound; thus we lose 2 degrees of freedom
* for fitting mu, lambda, but we get 1 back because we're unnormalized
* in this interval, hence we pass 2-1 = 1 as ndegrees.
*
* Mon Jan 19 06:18:14 1998: wonka = 1.0, temporarily disabled.
*/
ExtremeValueSetHistogram(h, mu, lambda, lowbound, highbound, 1.0, 1);
return 1;
FITFAILED:
UnfitHistogram(h);
if (x != NULL) free(x);
if (y != NULL) free(y);
return 0;
}
