/* Function: main_loop_threaded()
* Date: SRE, Wed Dec 1 12:43:09 1999 [St. Louis]
*
* Purpose: Given an HMM and parameters for synthesizing random
* sequences; return a histogram of scores.
* (Threaded version.)
*
* Args: hmm - an HMM to calibrate.
* seed - random number seed
* nsample - number of seqs to synthesize
* lenmean - mean length of random sequence
* lensd - std dev of random seq length
* fixedlen - if nonzero, override lenmean, always this len
* nthreads - number of threads to start
* ret_hist - RETURN: the score histogram
* ret_max - RETURN: highest score seen in simulation
* twatch - RETURN: accumulation of thread times
*
* Returns: (void)
* hist is alloc'ed here, and must be free'd by caller.
*/
static void
main_loop_threaded(struct plan7_s *hmm, int seed, int nsample,
float lenmean, float lensd, int fixedlen,
int nthreads,
struct histogram_s **ret_hist, float *ret_max,
Stopwatch_t *twatch)
{
struct histogram_s *hist;
float randomseq[MAXABET];
float p1;
struct workpool_s *wpool; /* pool of worker threads */
/* Initialize.
* We assume we've already set the alphabet (safe, because
* HMM input sets the alphabet).
*/
sre_srandom(seed);
P7Logoddsify(hmm, TRUE);
P7DefaultNullModel(randomseq, &p1);
hist = AllocHistogram(-200, 200, 100);
wpool = workpool_start(hmm, lenmean, lensd, fixedlen, randomseq, nsample,
hist, nthreads);
workpool_stop(wpool);
*ret_hist = hist;
*ret_max = wpool->max_score;
StopwatchInclude(twatch, &(wpool->watch));
workpool_free(wpool);
return;
}
/* Function: main_loop_serial()
* Date: SRE, Tue Aug 18 16:18:28 1998 [St. Louis]
*
* Purpose: Given an HMM and parameters for synthesizing random
* sequences; return a histogram of scores.
* (Serial version)
*
* Args: hmm - an HMM to calibrate.
* seed - random number seed
* nsample - number of seqs to synthesize
* lenmean - mean length of random sequence
* lensd - std dev of random seq length
* fixedlen - if nonzero, override lenmean, always this len
* ret_hist - RETURN: the score histogram
* ret_max - RETURN: highest score seen in simulation
*
* Returns: (void)
* hist is alloc'ed here, and must be free'd by caller.
*/
static void
main_loop_serial(struct plan7_s *hmm, int seed, int nsample,
float lenmean, float lensd, int fixedlen,
struct histogram_s **ret_hist, float *ret_max)
{
struct histogram_s *hist;
float randomseq[MAXABET];
float p1;
float max;
char *seq;
char *dsq;
float score;
int sqlen;
int idx;
/* Initialize.
* We assume we've already set the alphabet (safe, because
* HMM input sets the alphabet).
*/
sre_srandom(seed);
P7Logoddsify(hmm, TRUE);
P7DefaultNullModel(randomseq, &p1);
hist = AllocHistogram(-200, 200, 100);
max = -FLT_MAX;
for (idx = 0; idx < nsample; idx++)
{
/* choose length of random sequence */
if (fixedlen) sqlen = fixedlen;
else do sqlen = (int) Gaussrandom(lenmean, lensd); while (sqlen < 1);
/* generate it */
seq = RandomSequence(Alphabet, randomseq, Alphabet_size, sqlen);
dsq = DigitizeSequence(seq, sqlen);
if (P7ViterbiSize(sqlen, hmm->M) <= RAMLIMIT)
score = P7Viterbi(dsq, sqlen, hmm, NULL);
else
score = P7SmallViterbi(dsq, sqlen, hmm, NULL);
AddToHistogram(hist, score);
if (score > max) max = score;
free(dsq);
free(seq);
}
*ret_hist = hist;
*ret_max = max;
return;
}
void HMMCreateWPoolTask::runUnsafe() {
const UHMMCalibrateSettings& settings = pt->getSettings();
WorkPool_s* wpool = pt->getWorkPool();
SetAlphabet(wpool->hmm->atype);
sre_srandom(settings.seed);
wpool->fixedlen = settings.fixedlen;
wpool->hist = AllocHistogram(-200, 200, 100);
wpool->lenmean = settings.lenmean;
wpool->lensd = settings.lensd;
wpool->nsample = settings.nsample;
wpool->nseq = 0;
wpool->randomseq.resize(MAXABET);
wpool->max_score = -FLT_MAX;
float p1;
P7Logoddsify(wpool->hmm, TRUE);
P7DefaultNullModel(wpool->randomseq.data(), &p1);
}
QList<UHMMSearchResult> UHMMSearch::search(plan7_s* _hmm, const char* seq, int seqLen, const UHMMSearchSettings& s, TaskStateInfo& si)
{
plan7_s * hmm = HMMIO::cloneHMM( _hmm );
//Set up optional Pfam score thresholds.
threshold_s thresh; // contains all threshold (cutoff) info
thresh.globE = s.globE; // use a reasonable Eval threshold
thresh.globT = -FLT_MAX; // but no bit threshold
thresh.domT = s.domT; // no domain bit threshold
thresh.domE = s.domE; // and no domain Eval threshold
thresh.autocut = CUT_NONE; // and no Pfam cutoffs used
thresh.Z = s.eValueNSeqs; // Z not preset; use actual # of seqs
int do_null2 = TRUE; // TRUE to adjust scores with null model #2
int do_forward = FALSE; // TRUE to use Forward() not Viterbi()
int do_xnu = FALSE; // TRUE to filter sequences thru XNU
QList<UHMMSearchResult> res; // the results of the method
//get HMMERTaskLocalData
HMMERTaskLocalData *tld = getHMMERTaskLocalData();
alphabet_s *al = &tld->al;
SetAlphabet(hmm->atype);
P7Logoddsify(hmm, !do_forward); //TODO: clone model to avoid changes in it or make it thread safe??
if (do_xnu && al->Alphabet_type == hmmNUCLEIC) {
si.setError( "The HMM is a DNA model, and you can't use the --xnu filter on DNA data" );
return res;
}
/*****************************************************************
* Set up optional Pfam score thresholds.
* Can do this before starting any searches, since we'll only use 1 HMM.
*****************************************************************/
if (!SetAutocuts(&thresh, hmm)) {
si.setError( "HMM did not contain the GA, TC, or NC cutoffs you needed" );
return res;
}
// set up structures for storing output
histogram_s *histogram = AllocHistogram(-200, 200, 100); //keeps full histogram of all scores
tophit_s *ghit = AllocTophits(200); // per-seq hits: 200=lumpsize
tophit_s *dhit = AllocTophits(200); // domain hits: 200=lumpsize
int nseq = 0; // number of sequences searched
#ifdef UGENE_CELL
if( HMMSearchAlgo_CellOptimized == s.alg ) {
if( hmm->M < MAX_HMM_LENGTH ) {
main_loop_spe(hmm, seq, seqLen, &thresh, do_forward, do_null2, do_xnu, histogram, ghit, dhit, &nseq, si);
} else {
main_loop_serial(hmm, seq, seqLen, &thresh, do_forward, do_null2, do_xnu, histogram, ghit, dhit, &nseq, si);
}
} else
#elif defined(HMMER_BUILD_WITH_SSE2)
if( HMMSearchAlgo_SSEOptimized == s.alg ) {
main_loop_opt(hmm, seq, seqLen, &thresh, do_forward, do_null2, do_xnu, histogram, ghit, dhit, &nseq, si, sseScoring);
} else
#endif
if( HMMSearchAlgo_Conservative == s.alg ) {
main_loop_serial(hmm, seq, seqLen, &thresh, do_forward, do_null2, do_xnu, histogram, ghit, dhit, &nseq, si);
}
else {
assert( false && "bad hmmsearch algorithm selected" );
}
// Process hit lists, produce text output
// Set the theoretical EVD curve in our histogram using calibration in the HMM, if available.
if (hmm->flags & PLAN7_STATS) {
ExtremeValueSetHistogram(histogram, hmm->mu, hmm->lambda, histogram->lowscore, histogram->highscore, 0);
}
if (!thresh.Z) {
thresh.Z = nseq; // set Z for good now that we're done
}
//report our output
FullSortTophits(dhit);
//int namewidth = MAX(8, TophitsMaxName(ghit)); // max width of sequence name
// Report domain hits (sorted on E-value)
for (int i = 0; i < dhit->num && !si.cancelFlag; i++) {
float sc; // score of an HMM search
double pvalue; // pvalue of an HMM score
double evalue; // evalue of an HMM score
char *name, *desc; // hit sequence name and description
double motherp; // pvalue of a whole seq HMM score
float mothersc; // score of a whole seq parent of domain
int sqfrom, sqto; // coordinates in sequence
int sqlen; // length of seq that was hit
int hmmfrom, hmmto; // coordinate in HMM
int ndom; // total # of domains in this seq
int domidx; // number of this domain
GetRankedHit(dhit, i, &pvalue, &sc, &motherp, &mothersc,
&name, NULL, &desc,
&sqfrom, &sqto, &sqlen, // seq position info
&hmmfrom, &hmmto, NULL, // HMM position info
&domidx, &ndom, // domain info
NULL); // alignment info
evalue = pvalue * (double) thresh.Z;
if (motherp * (double) thresh.Z > thresh.globE || mothersc < thresh.globT) {
continue;
} else if (evalue <= thresh.domE && sc >= thresh.domT) {
// hmm reports results in range [1...N] -> translate it to [0..N)
res.append(UHMMSearchResult(U2Region(sqfrom-1, sqto-sqfrom+1), sc, evalue));
}
}
//Clean-up and exit.
FreeHistogram(histogram);
FreeTophits(ghit);
FreeTophits(dhit);
FreePlan7( hmm );
return res;
}
static void main_loop_serial(struct plan7_s *hmm, int seed, int nsample,
float lenmean, float lensd, int fixedlen,
struct histogram_s **ret_hist, float *ret_max, int& cancelFlag, int& progress)
{
struct histogram_s *hist;
struct dpmatrix_s *mx;
float randomseq[MAXABET];
float p1;
float max;
char *seq;
unsigned char *dsq;
float score;
int sqlen;
int idx;
// Initialize.
// We assume we've already set the alphabet (safe, because
// HMM input sets the alphabet).
sre_srandom(seed);
//get HMMERTaskLocalData
HMMERTaskLocalData *tls = getHMMERTaskLocalData();
alphabet_s &al = tls->al;
SetAlphabet(hmm->atype);
P7Logoddsify(hmm, TRUE);
P7DefaultNullModel(randomseq, &p1);
hist = AllocHistogram(-200, 200, 100);
mx = CreatePlan7Matrix(1, hmm->M, 25, 0);
max = -FLT_MAX;
progress = 0;
int pStub;
for (idx = 0; idx < nsample && !cancelFlag; idx++) {
// choose length of random sequence
if (fixedlen) {
sqlen = fixedlen;
} else {
do sqlen = (int) Gaussrandom(lenmean, lensd); while (sqlen < 1);
}
// generate it
seq = RandomSequence(al.Alphabet, randomseq, al.Alphabet_size, sqlen);
dsq = DigitizeSequence(seq, sqlen);
if (P7ViterbiSpaceOK(sqlen, hmm->M, mx)) {
score = P7Viterbi(dsq, sqlen, hmm, mx, NULL);
} else {
score = P7SmallViterbi(dsq, sqlen, hmm, mx, NULL, pStub);
}
AddToHistogram(hist, score);
max = qMax(score, max);
progress = int(100*idx/float(nsample));
free(dsq);
free(seq);
}
FreePlan7Matrix(mx);
*ret_hist = hist;
*ret_max = max;
}
int
main(void)
{
int master_tid; /* PVM TID of our master */
int slaveidx; /* my slave index (0..nslaves-1) */
struct plan7_s *hmm; /* HMM to calibrate, sent from master */
struct histogram_s *hist; /* score histogram */
int hmmidx; /* index of this HMM */
char *seq; /* synthetic random sequence */
char *dsq; /* digitized seq */
int len; /* length of seq */
float sc; /* score of seq aligned to HMM */
float max; /* maximum score seen in sample */
int seed; /* random number seed */
int nsample; /* number of seqs to sample */
int fixedlen; /* if nonzero, fixed length of seq */
float lenmean; /* Gaussian mean length of seq */
float lensd; /* Gaussian length std. dev. for seq */
int fitok; /* TRUE if EVD fit was OK */
float randomseq[MAXABET]; /* iid frequencies of residues */
float p1;
int alphatype; /* alphabet type, hmmAMINO or hmmNUCLEIC */
int idx;
int code;
/* Register leave_pvm() cleanup function so any exit() call
* first calls pvm_exit().
*/
if (atexit(leave_pvm) != 0) { pvm_exit(); Die("slave couldn't register leave_pvm()"); }
/*****************************************************************
* initialization.
* Master broadcasts the problem to us: parameters of the
* HMM calibration.
******************************************************************/
master_tid = pvm_parent(); /* who's our master? */
pvm_recv(master_tid, HMMPVM_INIT);
pvm_upkint(&nsample, 1, 1);
pvm_upkint(&fixedlen, 1, 1);
pvm_upkfloat(&lenmean, 1, 1);
pvm_upkfloat(&lensd, 1, 1);
/* tell the master we're OK and ready to go (or not)
*/
code = HMMPVM_OK;
pvm_initsend(PvmDataDefault);
pvm_pkint(&code, 1, 1);
pvm_send(master_tid, HMMPVM_RESULTS);
/*****************************************************************
* Main loop.
* Receive a random number seed, then an HMM to search against.
* If we receive a -1 seed, we shut down.
*****************************************************************/
slaveidx = -1;
for (;;)
{
pvm_recv(master_tid, HMMPVM_WORK);
pvm_upkint(&seed, 1, 1);
if (seed == -1) break; /* shutdown signal */
pvm_upkint(&hmmidx, 1, 1);
pvm_upkint(&alphatype,1, 1);
SetAlphabet(alphatype);
hmm = PVMUnpackHMM();
if (hmm == NULL) Die("oh no, the HMM never arrived");
if (slaveidx == -1) slaveidx = hmmidx;
P7DefaultNullModel(randomseq, &p1);
sre_srandom(seed);
P7Logoddsify(hmm, TRUE);
hist = AllocHistogram(-200, 200, 100);
max = -FLT_MAX;
for (idx = 0; idx < nsample; idx++)
{
/* choose length of random sequence */
if (fixedlen) len = fixedlen;
else do len = (int) Gaussrandom(lenmean, lensd); while (len < 1);
/* generate it */
seq = RandomSequence(Alphabet, randomseq, Alphabet_size, len);
dsq = DigitizeSequence(seq, len);
if (P7ViterbiSize(len, hmm->M) <= RAMLIMIT)
sc = P7Viterbi(dsq, len, hmm, NULL);
else
sc = P7SmallViterbi(dsq, len, hmm, NULL);
AddToHistogram(hist, sc);
if (sc > max) max = sc;
free(seq);
free(dsq);
}
/* Fit an EVD to the observed histogram.
* The TRUE left-censors and fits only the right slope of the histogram.
* The 9999. is an arbitrary high number that means we won't trim outliers
* on the right.
*/
fitok = ExtremeValueFitHistogram(hist, TRUE, 9999.);
/* Return output to master.
* Currently we don't send the histogram back, but we could.
*/
pvm_initsend(PvmDataDefault);
pvm_pkint(&slaveidx, 1, 1);
pvm_pkint(&hmmidx, 1, 1);
PVMPackString(hmm->name);
pvm_pkint(&fitok, 1, 1);
pvm_pkfloat(&(hist->param[EVD_MU]), 1, 1);
pvm_pkfloat(&(hist->param[EVD_LAMBDA]), 1, 1);
pvm_pkfloat(&max, 1, 1);
pvm_send(master_tid, HMMPVM_RESULTS);
/* cleanup
*/
FreeHistogram(hist);
FreePlan7(hmm);
}
/***********************************************
* Cleanup, return.
***********************************************/
return 0; /* pvm_exit() is called by atexit() registration. */
}
/* Function: main_loop_pvm()
* Date: SRE, Wed Aug 19 13:59:54 1998 [St. Louis]
*
* Purpose: Given an HMM and parameters for synthesizing random
* sequences; return a histogram of scores.
* (PVM version)
*
* Args: hmm - an HMM to calibrate.
* seed - random number seed
* nsample - number of seqs to synthesize
* lumpsize- # of seqs per slave exchange; controls granularity
* lenmean - mean length of random sequence
* lensd - std dev of random seq length
* fixedlen- if nonzero, override lenmean, always this len
* hist - RETURN: the score histogram
* ret_max - RETURN: highest score seen in simulation
* extrawatch - RETURN: total CPU time spend in slaves.
* ret_nslaves- RETURN: number of PVM slaves run.
*
* Returns: (void)
* hist is alloc'ed here, and must be free'd by caller.
*/
static void
main_loop_pvm(struct plan7_s *hmm, int seed, int nsample, int lumpsize,
float lenmean, float lensd, int fixedlen,
struct histogram_s **ret_hist, float *ret_max,
Stopwatch_t *extrawatch, int *ret_nslaves)
{
struct histogram_s *hist;
int master_tid;
int *slave_tid;
int nslaves;
int nsent; /* # of seqs we've asked for so far */
int ndone; /* # of seqs we've got results for so far */
int packet; /* # of seqs to have a slave do */
float max;
int slaveidx; /* id of a slave */
float *sc; /* scores returned by a slave */
Stopwatch_t slavewatch;
int i;
StopwatchZero(extrawatch);
hist = AllocHistogram(-200, 200, 100);
max = -FLT_MAX;
/* Initialize PVM
*/
if ((master_tid = pvm_mytid()) < 0)
Die("pvmd not responding -- do you have PVM running?");
#if DEBUGLEVEL >= 1
pvm_catchout(stderr); /* catch output for debugging */
#endif
PVMSpawnSlaves("hmmcalibrate-pvm", &slave_tid, &nslaves);
/* Initialize the slaves
*/
pvm_initsend(PvmDataDefault);
pvm_pkfloat(&lenmean, 1, 1);
pvm_pkfloat(&lensd, 1, 1);
pvm_pkint( &fixedlen, 1, 1);
pvm_pkint( &Alphabet_type, 1, 1);
pvm_pkint( &seed, 1, 1);
if (! PVMPackHMM(hmm)) Die("Failed to pack the HMM");
pvm_mcast(slave_tid, nslaves, HMMPVM_INIT);
SQD_DPRINTF1(("Initialized %d slaves\n", nslaves));
/* Confirm slaves' OK status.
*/
PVMConfirmSlaves(slave_tid, nslaves);
SQD_DPRINTF1(("Slaves confirm that they're ok...\n"));
/* Load the slaves
*/
nsent = ndone = 0;
for (slaveidx = 0; slaveidx < nslaves; slaveidx++)
{
packet = (nsample - nsent > lumpsize ? lumpsize : nsample - nsent);
pvm_initsend(PvmDataDefault);
pvm_pkint(&packet, 1, 1);
pvm_pkint(&slaveidx, 1, 1);
pvm_send(slave_tid[slaveidx], HMMPVM_WORK);
nsent += packet;
}
SQD_DPRINTF1(("Loaded %d slaves\n", nslaves));
/* Receive/send loop
*/
sc = MallocOrDie(sizeof(float) * lumpsize);
while (nsent < nsample)
{
/* integrity check of slaves */
PVMCheckSlaves(slave_tid, nslaves);
/* receive results */
SQD_DPRINTF2(("Waiting for results...\n"));
pvm_recv(-1, HMMPVM_RESULTS);
pvm_upkint(&slaveidx, 1, 1);
pvm_upkint(&packet, 1, 1);
pvm_upkfloat(sc, packet, 1);
SQD_DPRINTF2(("Got results.\n"));
ndone += packet;
/* store results */
for (i = 0; i < packet; i++) {
AddToHistogram(hist, sc[i]);
if (sc[i] > max) max = sc[i];
}
/* send new work */
packet = (nsample - nsent > lumpsize ? lumpsize : nsample - nsent);
pvm_initsend(PvmDataDefault);
pvm_pkint(&packet, 1, 1);
pvm_pkint(&slaveidx, 1, 1);
pvm_send(slave_tid[slaveidx], HMMPVM_WORK);
SQD_DPRINTF2(("Told slave %d to do %d more seqs.\n", slaveidx, packet));
nsent += packet;
}
/* Wait for the last output to come in.
*/
while (ndone < nsample)
{
/* integrity check of slaves */
PVMCheckSlaves(slave_tid, nslaves);
/* receive results */
SQD_DPRINTF1(("Waiting for final results...\n"));
pvm_recv(-1, HMMPVM_RESULTS);
pvm_upkint(&slaveidx, 1, 1);
pvm_upkint(&packet, 1, 1);
pvm_upkfloat(sc, packet, 1);
SQD_DPRINTF2(("Got some final results.\n"));
ndone += packet;
/* store results */
for (i = 0; i < packet; i++) {
AddToHistogram(hist, sc[i]);
if (sc[i] > max) max = sc[i];
}
}
/* Shut down the slaves: send -1,-1,-1.
*/
pvm_initsend(PvmDataDefault);
packet = -1;
pvm_pkint(&packet, 1, 1);
pvm_pkint(&packet, 1, 1);
pvm_pkint(&packet, 1, 1);
pvm_mcast(slave_tid, nslaves, HMMPVM_WORK);
/* Collect stopwatch results; quit the VM; return.
*/
for (i = 0; i < nslaves; i++)
{
pvm_recv(-1, HMMPVM_RESULTS);
pvm_upkint(&slaveidx, 1, 1);
StopwatchPVMUnpack(&slavewatch);
SQD_DPRINTF1(("Slave %d finished; says it used %.2f cpu, %.2f sys\n",
slaveidx, slavewatch.user, slavewatch.sys));
StopwatchInclude(extrawatch, &slavewatch);
}
free(slave_tid);
free(sc);
pvm_exit();
*ret_hist = hist;
*ret_max = max;
*ret_nslaves = nslaves;
return;
}
