static void
utest_oprofileSendRecv(int my_rank, int nproc)
{
ESL_RANDOMNESS *r = esl_randomness_CreateFast(42);
ESL_ALPHABET *abc = esl_alphabet_Create(eslAMINO);
P7_HMM *hmm = NULL;
P7_BG *bg = NULL;
P7_PROFILE *gm = NULL;
P7_OPROFILE *om = NULL;
P7_OPROFILE *om2 = NULL;
int M = 200;
int L = 400;
char *wbuf = NULL;
int wn = 0;
int i;
char errbuf[eslERRBUFSIZE];
p7_hmm_Sample(r, M, abc, &hmm); /* master and worker's sampled profiles are identical */
bg = p7_bg_Create(abc);
gm = p7_profile_Create(hmm->M, abc);
om = p7_oprofile_Create(hmm->M, abc);
p7_ProfileConfig(hmm, bg, gm, L, p7_LOCAL);
p7_oprofile_Convert(gm, om);
p7_bg_SetLength (bg, L);
if (my_rank == 0)
{
for (i = 1; i < nproc; i++)
{
ESL_DPRINTF1(("Master: receiving test profile\n"));
p7_oprofile_MPIRecv(MPI_ANY_SOURCE, 0, MPI_COMM_WORLD, &wbuf, &wn, &abc, &om2);
ESL_DPRINTF1(("Master: test profile received\n"));
if (p7_oprofile_Compare(om, om2, 0.001, errbuf) != eslOK)
p7_Die("Received profile not identical to what was sent\n%s", errbuf);
p7_oprofile_Destroy(om2);
}
}
else
{
ESL_DPRINTF1(("Worker %d: sending test profile\n", my_rank));
p7_oprofile_MPISend(om, 0, 0, MPI_COMM_WORLD, &wbuf, &wn);
ESL_DPRINTF1(("Worker %d: test profile sent\n", my_rank));
}
free(wbuf);
p7_profile_Destroy(gm);
p7_oprofile_Destroy(om);
p7_bg_Destroy(bg);
p7_hmm_Destroy(hmm);
esl_alphabet_Destroy(abc);
esl_randomness_Destroy(r);
return;
}
/* mpi_master()
* The MPI version of hmmbuild.
* Follows standard pattern for a master/worker load-balanced MPI program (J1/78-79).
*
* A master can only return if it's successful.
* Errors in an MPI master come in two classes: recoverable and nonrecoverable.
*
* Recoverable errors include all worker-side errors, and any
* master-side error that do not affect MPI communication. Error
* messages from recoverable messages are delayed until we've cleanly
* shut down the workers.
*
* Unrecoverable errors are master-side errors that may affect MPI
* communication, meaning we cannot count on being able to reach the
* workers and shut them down. Unrecoverable errors result in immediate
* p7_Fail()'s, which will cause MPI to shut down the worker processes
* uncleanly.
*/
static void
mpi_master(const ESL_GETOPTS *go, struct cfg_s *cfg)
{
int xstatus = eslOK; /* changes from OK on recoverable error */
int status;
int have_work = TRUE; /* TRUE while alignments remain */
int nproc_working = 0; /* number of worker processes working, up to nproc-1 */
int wi; /* rank of next worker to get an alignment to work on */
char *buf = NULL; /* input/output buffer, for packed MPI messages */
int bn = 0;
ESL_MSA *msa = NULL;
P7_HMM *hmm = NULL;
P7_BG *bg = NULL;
ESL_MSA **msalist = NULL;
ESL_MSA *postmsa = NULL;
int *msaidx = NULL;
char errmsg[eslERRBUFSIZE];
MPI_Status mpistatus;
int n;
int pos;
double entropy;
/* Master initialization: including, figure out the alphabet type.
* If any failure occurs, delay printing error message until we've shut down workers.
*/
if (xstatus == eslOK) { if ((status = init_master_cfg(go, cfg, errmsg)) != eslOK) xstatus = status; }
if (xstatus == eslOK) { bn = 4096; if ((buf = malloc(sizeof(char) * bn)) == NULL) { sprintf(errmsg, "allocation failed"); xstatus = eslEMEM; } }
if (xstatus == eslOK) { if ((msalist = malloc(sizeof(ESL_MSA *) * cfg->nproc)) == NULL) { sprintf(errmsg, "allocation failed"); xstatus = eslEMEM; } }
if (xstatus == eslOK) { if ((msaidx = malloc(sizeof(int) * cfg->nproc)) == NULL) { sprintf(errmsg, "allocation failed"); xstatus = eslEMEM; } }
MPI_Bcast(&xstatus, 1, MPI_INT, 0, MPI_COMM_WORLD);
if (xstatus != eslOK) { MPI_Finalize(); p7_Fail(errmsg); }
ESL_DPRINTF1(("MPI master is initialized\n"));
bg = p7_bg_Create(cfg->abc);
for (wi = 0; wi < cfg->nproc; wi++) { msalist[wi] = NULL; msaidx[wi] = 0; }
/* Worker initialization:
* Because we've already successfully initialized the master before we start
* initializing the workers, we don't expect worker initialization to fail;
* so we just receive a quick OK/error code reply from each worker to be sure,
* and don't worry about an informative message.
*/
MPI_Bcast(&(cfg->abc->type), 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Reduce(&xstatus, &status, 1, MPI_INT, MPI_MAX, 0, MPI_COMM_WORLD);
if (status != eslOK) { MPI_Finalize(); p7_Fail("One or more MPI worker processes failed to initialize."); }
ESL_DPRINTF1(("%d workers are initialized\n", cfg->nproc-1));
/* Main loop: combining load workers, send/receive, clear workers loops;
* also, catch error states and die later, after clean shutdown of workers.
*
* When a recoverable error occurs, have_work = FALSE, xstatus !=
* eslOK, and errmsg is set to an informative message. No more
* errmsg's can be received after the first one. We wait for all the
* workers to clear their work units, then send them shutdown signals,
* then finally print our errmsg and exit.
*
* Unrecoverable errors just crash us out with p7_Fail().
*/
wi = 1;
while (have_work || nproc_working)
{
if (have_work)
{
if ((status = esl_msa_Read(cfg->afp, &msa)) == eslOK)
{
cfg->nali++;
ESL_DPRINTF1(("MPI master read MSA %s\n", msa->name == NULL? "" : msa->name));
}
else
{
have_work = FALSE;
if (status == eslEFORMAT) { xstatus = eslEFORMAT; snprintf(errmsg, eslERRBUFSIZE, "Alignment file parse error:\n%s\n", cfg->afp->errbuf); }
else if (status == eslEINVAL) { xstatus = eslEFORMAT; snprintf(errmsg, eslERRBUFSIZE, "Alignment file parse error:\n%s\n", cfg->afp->errbuf); }
else if (status != eslEOF) { xstatus = status; snprintf(errmsg, eslERRBUFSIZE, "Alignment file read unexpectedly failed with code %d\n", status); }
ESL_DPRINTF1(("MPI master has run out of MSAs (having read %d)\n", cfg->nali));
}
}
if ((have_work && nproc_working == cfg->nproc-1) || (!have_work && nproc_working > 0))
{
if (MPI_Probe(MPI_ANY_SOURCE, 0, MPI_COMM_WORLD, &mpistatus) != 0) { MPI_Finalize(); p7_Fail("mpi probe failed"); }
if (MPI_Get_count(&mpistatus, MPI_PACKED, &n) != 0) { MPI_Finalize(); p7_Fail("mpi get count failed"); }
wi = mpistatus.MPI_SOURCE;
ESL_DPRINTF1(("MPI master sees a result of %d bytes from worker %d\n", n, wi));
if (n > bn) {
if ((buf = realloc(buf, sizeof(char) * n)) == NULL) p7_Fail("reallocation failed");
bn = n;
}
if (MPI_Recv(buf, bn, MPI_PACKED, wi, 0, MPI_COMM_WORLD, &mpistatus) != 0) { MPI_Finalize(); p7_Fail("mpi recv failed"); }
ESL_DPRINTF1(("MPI master has received the buffer\n"));
/* If we're in a recoverable error state, we're only clearing worker results;
* just receive them, don't unpack them or print them.
* But if our xstatus is OK, go ahead and process the result buffer.
*/
if (xstatus == eslOK)
{
pos = 0;
if (MPI_Unpack(buf, bn, &pos, &xstatus, 1, MPI_INT, MPI_COMM_WORLD) != 0) { MPI_Finalize(); p7_Fail("mpi unpack failed");}
if (xstatus == eslOK) /* worker reported success. Get the HMM. */
{
ESL_DPRINTF1(("MPI master sees that the result buffer contains an HMM\n"));
if (p7_hmm_MPIUnpack(buf, bn, &pos, MPI_COMM_WORLD, &(cfg->abc), &hmm) != eslOK) { MPI_Finalize(); p7_Fail("HMM unpack failed"); }
ESL_DPRINTF1(("MPI master has unpacked the HMM\n"));
if (cfg->postmsafile != NULL) {
if (esl_msa_MPIUnpack(cfg->abc, buf, bn, &pos, MPI_COMM_WORLD, &postmsa) != eslOK) { MPI_Finalize(); p7_Fail("postmsa unpack failed");}
}
entropy = p7_MeanMatchRelativeEntropy(hmm, bg);
if ((status = output_result(cfg, errmsg, msaidx[wi], msalist[wi], hmm, postmsa, entropy)) != eslOK) xstatus = status;
esl_msa_Destroy(postmsa); postmsa = NULL;
p7_hmm_Destroy(hmm); hmm = NULL;
}
else /* worker reported an error. Get the errmsg. */
{
if (MPI_Unpack(buf, bn, &pos, errmsg, eslERRBUFSIZE, MPI_CHAR, MPI_COMM_WORLD) != 0) { MPI_Finalize(); p7_Fail("mpi unpack of errmsg failed"); }
ESL_DPRINTF1(("MPI master sees that the result buffer contains an error message\n"));
}
}
esl_msa_Destroy(msalist[wi]);
msalist[wi] = NULL;
msaidx[wi] = 0;
nproc_working--;
}
if (have_work)
{
ESL_DPRINTF1(("MPI master is sending MSA %s to worker %d\n", msa->name == NULL ? "":msa->name, wi));
if (esl_msa_MPISend(msa, wi, 0, MPI_COMM_WORLD, &buf, &bn) != eslOK) p7_Fail("MPI msa send failed");
msalist[wi] = msa;
msaidx[wi] = cfg->nali; /* 1..N for N alignments in the MSA database */
msa = NULL;
wi++;
nproc_working++;
}
}
/* On success or recoverable errors:
* Shut down workers cleanly.
*/
ESL_DPRINTF1(("MPI master is done. Shutting down all the workers cleanly\n"));
for (wi = 1; wi < cfg->nproc; wi++)
if (esl_msa_MPISend(NULL, wi, 0, MPI_COMM_WORLD, &buf, &bn) != eslOK) p7_Fail("MPI msa send failed");
free(buf);
free(msaidx);
free(msalist);
p7_bg_Destroy(bg);
if (xstatus != eslOK) { MPI_Finalize(); p7_Fail(errmsg); }
else return;
}
/* Function: esl_gumbel_FitCensored()
* Synopsis: Estimates $\mu$, $\lambda$ from censored data.
* Date: SRE, Mon Nov 17 10:01:05 1997 [St. Louis]
*
* Purpose: Given a left-censored array of Gumbel-distributed samples
* <x[0]..x[n-1]>, the number of censored samples <z>, and the
* censoring value <phi> (all <x[i]> $>$ <phi>).
* Find maximum likelihood parameters <mu> and <lambda>.
*
* Algorithm: Uses approach described in [Lawless82]. Solves
* for lambda using Newton/Raphson iterations;
* then substitutes lambda into Lawless' equation 4.2.3
* to get mu.
*
* Args: x - array of Gumbel-distributed samples, 0..n-1
* n - number of observed samples
* z - number of censored samples
* phi - censoring value (all x_i >= phi)
* ret_mu : RETURN: ML estimate of mu
* ret_lambda : RETURN: ML estimate of lambda
*
* Returns: <eslOK> on success.
*
* Throws: <eslENOHALT> if the fit doesn't converge.
*/
int
esl_gumbel_FitCensored(double *x, int n, int z, double phi,
double *ret_mu, double *ret_lambda)
{
double variance;
double lambda, mu;
double fx; /* f(x) */
double dfx; /* f'(x) */
double esum; /* \sum e^(-lambda xi) */
double tol = 1e-5;
int i;
/* 1. Find an initial guess at lambda
* (Evans/Hastings/Peacock, Statistical Distributions, 2000, p.86)
*/
esl_stats_DMean(x, n, NULL, &variance);
lambda = eslCONST_PI / sqrt(6.*variance);
/* 2. Use Newton/Raphson to solve Lawless 4.2.2 and find ML lambda
*/
for (i = 0; i < 100; i++)
{
lawless422(x, n, z, phi, lambda, &fx, &dfx);
if (fabs(fx) < tol) break; /* success */
lambda = lambda - fx / dfx; /* Newton/Raphson is simple */
if (lambda <= 0.) lambda = 0.001; /* but be a little careful */
}
/* 2.5: If we did 100 iterations but didn't converge, Newton/Raphson failed.
* Resort to a bisection search. Worse convergence speed
* but guaranteed to converge (unlike Newton/Raphson).
* We assume (!?) that fx is a monotonically decreasing function of x;
* i.e. fx > 0 if we are left of the root, fx < 0 if we
* are right of the root.
*/
if (i == 100)
{
double left, right, mid;
ESL_DPRINTF1(("esl_gumbel_FitCensored(): Newton/Raphson failed; switched to bisection"));
/* First bracket the root */
left = 0.; /* we know that's the left bound */
right = eslCONST_PI / sqrt(6.*variance); /* start from here, move "right"... */
lawless422(x, n, z, phi, right, &fx, &dfx);
while (fx > 0.)
{
right *= 2.;
if (right > 100.) /* no reasonable lambda should be > 100, we assert */
ESL_EXCEPTION(eslENOHALT, "Failed to bracket root in esl_gumbel_FitCensored().");
lawless422(x, n, z, phi, right, &fx, &dfx);
}
/* Now we bisection search in left/right interval */
for (i = 0; i < 100; i++)
{
mid = (left + right) / 2.;
lawless422(x, n, z, phi, mid, &fx, &dfx);
if (fabs(fx) < tol) break; /* success */
if (fx > 0.) left = mid;
else right = mid;
}
if (i == 100)
ESL_EXCEPTION(eslENOHALT, "Even bisection search failed in esl_gumbel_FitCensored().");
lambda = mid;
}
/* 3. Substitute into Lawless 4.2.3 to find mu
*/
esum = 0.;
for (i = 0; i < n; i++)
esum += exp(-lambda * x[i]);
esum += z * exp(-1. * lambda * phi); /* term from censored data */
mu = -log(esum / n) / lambda;
*ret_lambda = lambda;
*ret_mu = mu;
return eslOK;
}
