int collect_pstree(pid_t pid)
{
int ret;
timing_start(TIME_FREEZING);
if (opts.freeze_cgroup && freeze_processes())
return -1;
root_item = alloc_pstree_item();
if (root_item == NULL)
return -1;
root_item->pid.real = pid;
if (!opts.freeze_cgroup && seize_catch_task(pid)) {
set_cr_errno(ESRCH);
goto err;
}
ret = seize_wait_task(pid, -1, &dmpi(root_item)->pi_creds);
if (ret < 0)
goto err;
pr_info("Seized task %d, state %d\n", pid, ret);
root_item->state = ret;
ret = collect_task(root_item);
if (ret < 0)
goto err;
timing_stop(TIME_FREEZING);
timing_start(TIME_FROZEN);
return 0;
err:
pstree_switch_state(root_item, TASK_ALIVE);
return -1;
}
int
main(int argc, char *argv[])
{
lexicon_t *lex;
model_def_t *omdef;
model_def_t *dmdef;
uint32 n_stream;
const uint32 *veclen;
uint32 ts_off;
uint32 ts_cnt;
FILE *fp;
timing_t *all_timer= NULL;
timing_t *km_timer= NULL;
timing_t *var_timer= NULL;
timing_t *em_timer= NULL;
if (main_initialize(argc, argv, &lex, &omdef, &dmdef) != S3_SUCCESS) {
return -1;
}
km_timer = timing_get("km");
var_timer = timing_get("var");
em_timer = timing_get("em");
all_timer = timing_get("all");
n_stream = feat_n_stream();
veclen = feat_vecsize();
if (strcmp((const char *)cmd_ln_access("-gthobj"), "state") == 0) {
ts_off = *(uint32 *)cmd_ln_access("-tsoff");
if (cmd_ln_access("-tscnt") == NULL) {
ts_cnt = omdef->n_tied_state - ts_off;
}
else {
ts_cnt = *(uint32 *)cmd_ln_access("-tscnt");
}
if (ts_off + ts_cnt > omdef->n_tied_state) {
E_FATAL("Too many tied states specified\n");
}
n_tot_frame = 0;
if (all_timer)
timing_reset(all_timer);
if (km_timer)
timing_reset(km_timer);
if (var_timer)
timing_reset(var_timer);
if (em_timer)
timing_reset(em_timer);
if (all_timer)
timing_start(all_timer);
if (init_state((const char *)cmd_ln_access("-segdmpfn"),
(const char *)cmd_ln_access("-segidxfn"),
*(int32 *)cmd_ln_access("-ndensity"),
n_stream,
veclen,
*(int32 *)cmd_ln_access("-reest"),
(const char *)cmd_ln_access("-mixwfn"),
(const char *)cmd_ln_access("-meanfn"),
(const char *)cmd_ln_access("-varfn"),
ts_off,
ts_cnt,
omdef->n_tied_state,
(dmdef != NULL ? dmdef->n_tied_state : omdef->n_tied_state))
!= S3_SUCCESS) {
E_ERROR("Unable to train [%u %u]\n", ts_off, ts_off+ts_cnt-1);
}
if (all_timer)
timing_stop(all_timer);
if (n_tot_frame > 0) {
E_INFO("TOTALS:");
if (km_timer) {
E_INFOCONT(" km %4.3fx %4.3e",
km_timer->t_cpu / (n_tot_frame * 0.01),
(km_timer->t_cpu > 0 ?
km_timer->t_elapsed / km_timer->t_cpu : 0.0));
}
if (var_timer) {
E_INFOCONT(" var %4.3fx %4.3e",
var_timer->t_cpu / (n_tot_frame * 0.01),
(var_timer->t_cpu > 0 ?
var_timer->t_elapsed / var_timer->t_cpu : 0.0));
}
if (em_timer) {
E_INFOCONT(" em %4.3fx %4.3e",
em_timer->t_cpu / (n_tot_frame * 0.01),
(em_timer->t_cpu > 0 ?
em_timer->t_elapsed / em_timer->t_cpu : 0.0));
}
if (all_timer) {
E_INFOCONT(" all %4.3fx %4.3e",
all_timer->t_cpu / (n_tot_frame * 0.01),
(all_timer->t_cpu > 0 ?
all_timer->t_elapsed / all_timer->t_cpu : 0.0));
}
E_INFOCONT("\n");
}
if (cmd_ln_access("-tsrngfn") != NULL) {
fp = fopen((const char *)cmd_ln_access("-tsrngfn"),
"w");
if (fp == NULL) {
E_FATAL_SYSTEM("Unable to open %s for reading",
(const char *)cmd_ln_access("-tsrngfn"));
}
fprintf(fp, "%d %d\n", ts_off, ts_cnt);
}
else if (ts_cnt != omdef->n_tied_state) {
E_WARN("Subset of tied states specified, but no -tsrngfn arg");
}
}
else if (strcmp((const char *)cmd_ln_access("-gthobj"), "single") == 0) {
n_tot_frame = 0;
if (all_timer)
timing_reset(all_timer);
if (km_timer)
timing_reset(km_timer);
if (var_timer)
timing_reset(var_timer);
if (em_timer)
timing_reset(em_timer);
if (all_timer)
timing_start(all_timer);
if (init_state((const char *)cmd_ln_access("-segdmpfn"),
NULL, /* No index -> single class dump file */
*(int32 *)cmd_ln_access("-ndensity"),
n_stream,
veclen,
*(int32 *)cmd_ln_access("-reest"),
(const char *)cmd_ln_access("-mixwfn"),
(const char *)cmd_ln_access("-meanfn"),
(const char *)cmd_ln_access("-varfn"),
0,
1,
1,
1) != S3_SUCCESS) {
E_ERROR("Unable to train\n");
}
if (all_timer)
timing_stop(all_timer);
if (n_tot_frame > 0) {
E_INFO("TOTALS:");
if (km_timer) {
E_INFOCONT(" km %4.3fx %4.3e",
km_timer->t_cpu / (n_tot_frame * 0.01),
(km_timer->t_cpu > 0 ?
km_timer->t_elapsed / km_timer->t_cpu : 0.0));
}
if (var_timer) {
E_INFOCONT(" var %4.3fx %4.3e",
var_timer->t_cpu / (n_tot_frame * 0.01),
(var_timer->t_cpu > 0 ?
var_timer->t_elapsed / var_timer->t_cpu : 0.0));
}
if (em_timer) {
E_INFOCONT(" em %4.3fx %4.3e",
em_timer->t_cpu / (n_tot_frame * 0.01),
(em_timer->t_cpu > 0 ?
em_timer->t_elapsed / em_timer->t_cpu : 0.0));
}
if (all_timer) {
E_INFOCONT(" all %4.3fx %4.3e",
all_timer->t_cpu / (n_tot_frame * 0.01),
(all_timer->t_cpu > 0 ?
all_timer->t_elapsed / all_timer->t_cpu : 0.0));
}
E_INFOCONT("\n");
}
}
return 0;
}
static void process_reffile (char *reffile)
{
FILE *rfp, *hfp;
char line[16384], uttid[4096], file[4096], lc_uttid[4096];
int32 i, k;
dagnode_t ref[MAX_UTT_LEN];
int32 nref, noov, nhyp;
int32 tot_err, tot_ref, tot_corr, tot_oov, tot_hyp;
dag_t *dag;
dpnode_t retval;
timing_t *tm;
char *latdir, *hypfile;
if ((rfp = fopen(reffile, "r")) == NULL)
E_FATAL("fopen(%s,r) failed\n", reffile);
latdir = (char *) cmd_ln_access ("-latdir");
hypfile = (char *) cmd_ln_access ("-hyp");
if ((! latdir) && (! hypfile))
E_FATAL("Both -latdir and -hyp arguments missing\n");
if (latdir && hypfile)
E_FATAL("-latdir and -hyp arguments are mutually exclusive\n");
hfp = NULL;
if (hypfile) {
if ((hfp = fopen(hypfile, "r")) == NULL)
E_FATAL("fopen(%s,r) failed\n", hypfile);
}
tot_err = 0;
tot_ref = 0;
tot_hyp = 0;
tot_corr = 0;
tot_oov = 0;
tm = timing_new ("Utt");
while (fgets(line, sizeof(line), rfp) != NULL) {
timing_reset (tm);
timing_start (tm);
if ((nref = refline2wds (line, ref, &noov, uttid)) < 0)
E_FATAL("Bad line in file %s: %s\n", reffile, line);
/* Read lattice or hypfile, whichever is specified */
if (latdir) {
sprintf (file, "%s/%s.lat", latdir, uttid);
dag = dag_load (file);
if (! dag) {
/* Try lower casing uttid */
strcpy (lc_uttid, uttid);
lcase (lc_uttid);
sprintf (file, "%s/%s.lat", latdir, lc_uttid);
dag = dag_load (file);
}
} else {
if (fgets(line, sizeof(line), hfp) == NULL)
E_FATAL("Premature EOF(%s) at uttid %s\n", hypfile, uttid);
dag = hypline2dag (uttid, line);
}
if (dag) {
/* Append sentinel silwid node to end of DAG */
dag_append_sentinel (dag, silwid);
/* Find best path (returns #errors/#correct and updates *nhyp) */
retval = dp (uttid, dict, oovbegin, ref, nref, dag, &nhyp, 0);
dag_destroy (dag);
} else {
retval.c = 0;
retval.e = nref-1;
nhyp = 0;
}
timing_stop (tm);
tot_ref += nref-1;
tot_hyp += nhyp;
tot_err += retval.e;
tot_corr += retval.c;
tot_oov += noov;
printf("(%s) << %d ref; %d %.1f%% oov; %d hyp; %d %.1f%% corr; %d %.1f%% err; %.1fs CPU >>\n",
uttid, nref-1,
noov, (nref > 1) ? (noov * 100.0) / (nref-1) : 0.0,
nhyp,
retval.c, (nref > 1) ? (retval.c * 100.0) / (nref-1) : 0.0,
retval.e, (nref > 1) ? (retval.e * 100.0) / (nref-1) : 0.0,
tm->t_cpu);
printf("== %7d ref; %5d %5.1f%% oov; %7d hyp; %7d %5.1f%% corr; %6d %5.1f%% err; %5.1fs CPU; %s\n",
tot_ref,
tot_oov, (tot_ref > 0) ? (tot_oov * 100.0) / tot_ref : 0.0,
tot_hyp,
tot_corr, (tot_ref > 0) ? (tot_corr * 100.0) / tot_ref : 0.0,
tot_err, (tot_ref > 0) ? (tot_err * 100.0) / tot_ref : 0.0,
tm->t_tot_cpu,
uttid);
fflush (stderr);
fflush (stdout);
}
fclose (rfp);
if (hfp)
fclose (hfp);
printf("SUMMARY: %d ref; %d %.3f%% oov; %d hyp; %d %.3f%% corr; %d %.3f%% err; %.1fs CPU\n",
tot_ref,
tot_oov, (tot_ref > 0) ? (tot_oov * 100.0) / tot_ref : 0.0,
tot_hyp,
tot_corr, (tot_ref > 0) ? (tot_corr * 100.0) / tot_ref : 0.0,
tot_err, (tot_ref > 0) ? (tot_err * 100.0) / tot_ref : 0.0,
tm->t_tot_cpu);
}
/*
* Find Viterbi alignment.
*/
static void align_utt (char *sent, /* In: Reference transcript */
float32 **mfc, /* In: MFC cepstra for input utterance */
int32 nfr, /* In: #frames of input */
char *ctlspec, /* In: Utt specifiction from control file */
char *uttid) /* In: Utterance id, for logging and other use */
{
static float32 **feat = NULL;
static int32 w;
static int32 topn;
static gauden_dist_t ***dist;
static int32 *senscr;
static s3senid_t *sen_active;
static int8 *mgau_active;
static char *s2stsegdir;
static char *stsegdir;
static char *phsegdir;
static char *wdsegdir;
int32 i, s, sid, gid, n_sen_active, best;
char *arg;
align_stseg_t *stseg;
align_phseg_t *phseg;
align_wdseg_t *wdseg;
if (! feat) {
/* One-time allocation of necessary intermediate variables */
/* Allocate space for a feature vector */
feat = (float32 **) ckd_calloc (n_feat, sizeof(float32 *));
for (i = 0; i < n_feat; i++)
feat[i] = (float32 *) ckd_calloc (featlen[i], sizeof(float32));
/* Allocate space for top-N codeword density values in a codebook */
w = feat_window_size (); /* #MFC vectors needed on either side of current
frame to compute one feature vector */
topn = *((int32 *) cmd_ln_access("-topn"));
if (topn > g->n_density) {
E_ERROR("-topn argument (%d) > #density codewords (%d); set to latter\n",
topn, g->n_density);
topn = g->n_density;
}
dist = (gauden_dist_t ***) ckd_calloc_3d (g->n_mgau, n_feat, topn,
sizeof(gauden_dist_t));
/* Space for one frame of senone scores, and per frame active flags */
senscr = (int32 *) ckd_calloc (sen->n_sen, sizeof(int32));
sen_active = (s3senid_t *) ckd_calloc (sen->n_sen, sizeof(s3senid_t));
mgau_active = (int8 *) ckd_calloc (g->n_mgau, sizeof(int8));
/* Note various output directories */
s2stsegdir = NULL;
stsegdir = NULL;
phsegdir = NULL;
wdsegdir = NULL;
if ((arg = (char *) cmd_ln_access ("-s2stsegdir")) != NULL)
s2stsegdir = (char *) ckd_salloc (arg);
if ((arg = (char *) cmd_ln_access ("-stsegdir")) != NULL)
stsegdir = (char *) ckd_salloc (arg);
if ((arg = (char *) cmd_ln_access ("-phsegdir")) != NULL)
phsegdir = (char *) ckd_salloc (arg);
if ((arg = (char *) cmd_ln_access ("-wdsegdir")) != NULL)
wdsegdir = (char *) ckd_salloc (arg);
}
/* HACK HACKA HACK BHIKSHA
if (nfr <= (w<<1)) {
E_ERROR("Utterance %s < %d frames (%d); ignored\n", uttid, (w<<1)+1, nfr);
return;
}
END HACK HACKA HACK */
cyctimer_reset_all ();
counter_reset_all ();
timing_reset (tm_utt);
timing_start (tm_utt);
cyctimer_resume (tmr_utt);
/* AGC and CMN */
arg = (char *) cmd_ln_access ("-cmn");
if (strcmp (arg, "current") == 0)
norm_mean (mfc-4, nfr+8, cepsize); /* -4 HACKA HACK */
arg = (char *) cmd_ln_access ("-agc");
if (strcmp (arg, "max") == 0)
agc_max (mfc, nfr);
if (align_build_sent_hmm (sent) != 0) {
align_destroy_sent_hmm ();
cyctimer_pause (tmr_utt);
E_ERROR("No sentence HMM; no alignment for %s\n", uttid);
return;
}
align_start_utt (uttid);
/*
* A feature vector for frame f depends on input MFC vectors [f-w..f+w]. Hence
* the feature vector corresponding to the first w and last w input frames is
* undefined. We define them by simply replicating the first and last true
* feature vectors (presumably silence regions).
*/
for (i = 0; i < nfr; i++) {
cyctimer_resume (tmr_utt);
/* Compute feature vector for current frame from input speech cepstra */
/* HACK HACKA HACK BHIKSHA
if (i < w)
feat_cep2feat (mfc+w, feat);
else if (i >= nfr-w)
feat_cep2feat (mfc+(nfr-w-1), feat);
else
END HACK HACKA HACK */
feat_cep2feat (mfc+i, feat);
/*
* Evaluate gaussian density codebooks and senone scores for input codeword.
* Evaluate only active codebooks and senones.
*/
/* Obtain active senone flags */
cyctimer_resume (tmr_senone);
align_sen_active (sen_active, sen->n_sen);
/* Flag all CI senones to active if interpolating */
if (interp) {
for (s = 0; s < mdef->n_ci_sen; s++)
sen_active[s] = 1;
}
/* Turn active flags into list (for faster access) */
n_sen_active = 0;
for (s = 0; s < mdef->n_sen; s++) {
if (sen_active[s])
sen_active[n_sen_active++] = s;
}
cyctimer_pause (tmr_senone);
/* Flag all active mixture-gaussian codebooks */
cyctimer_resume (tmr_gauden);
for (gid = 0; gid < g->n_mgau; gid++)
mgau_active[gid] = 0;
for (s = 0; s < n_sen_active; s++) {
sid = sen_active[s];
mgau_active[sen->mgau[sid]] = 1;
}
/* Compute topn gaussian density values (for active codebooks) */
for (gid = 0; gid < g->n_mgau; gid++)
if (mgau_active[gid])
gauden_dist (g, gid, topn, feat, dist[gid]);
cyctimer_pause (tmr_gauden);
/* Evaluate active senones */
cyctimer_resume (tmr_senone);
best = (int32) 0x80000000;
for (s = 0; s < n_sen_active; s++) {
sid = sen_active[s];
senscr[sid] = senone_eval (sen, sid, dist[sen->mgau[sid]], topn);
if (best < senscr[sid])
best = senscr[sid];
}
if (interp) {
for (s = 0; s < n_sen_active; s++) {
if ((sid = sen_active[s]) >= mdef->n_ci_sen)
interp_cd_ci (interp, senscr, sid, mdef->cd2cisen[sid]);
}
}
/* Normalize senone scores (interpolation above can only lower best score) */
for (s = 0; s < n_sen_active; s++) {
sid = sen_active[s];
senscr[sid] -= best;
}
senscale[i] = best;
cyctimer_pause (tmr_senone);
/* Step alignment one frame forward */
cyctimer_resume (tmr_align);
align_frame (senscr);
cyctimer_pause (tmr_align);
cyctimer_pause (tmr_utt);
}
timing_stop (tm_utt);
printf ("\n");
/* Wind up alignment for this utterance */
if (align_end_utt (&stseg, &phseg, &wdseg) < 0)
E_ERROR("Final state not reached; no alignment for %s\n\n", uttid);
else {
if (s2stsegdir)
write_s2stseg (s2stsegdir, stseg, uttid, ctlspec);
if (stsegdir)
write_stseg (stsegdir, stseg, uttid, ctlspec);
if (phsegdir)
write_phseg (phsegdir, phseg, uttid, ctlspec);
if (wdsegdir)
write_wdseg (wdsegdir, wdseg, uttid, ctlspec);
if (outsentfp)
write_outsent (outsentfp, wdseg, uttid);
}
align_destroy_sent_hmm ();
cyctimer_print_all_norm (stdout, nfr*0.01, tmr_utt);
counter_print_all (stdout);
printf("EXECTIME: %5d frames, %7.2f sec CPU, %6.2f xRT; %7.2f sec elapsed, %6.2f xRT\n",
nfr,
tm_utt->t_cpu, tm_utt->t_cpu * 100.0 / nfr,
tm_utt->t_elapsed, tm_utt->t_elapsed * 100.0 / nfr);
tot_nfr += nfr;
}
static int
init_state(const char *obsdmp,
const char *obsidx,
uint32 n_density,
uint32 n_stream,
const uint32 *veclen,
int reest,
const char *mixwfn,
const char *meanfn,
const char *varfn,
uint32 ts_off,
uint32 ts_cnt,
uint32 n_ts,
uint32 n_d_ts)
{
uint32 blksz;
vector_t ***mean;
vector_t ***var = NULL;
vector_t ****fullvar = NULL;
float32 ***mixw = NULL;
uint32 n_frame;
uint32 ignore = 0;
codew_t *label;
uint32 n_corpus = 0;
float64 sqerr;
float64 tot_sqerr;
segdmp_type_t t;
uint32 i, j, ts, n;
timing_t *km_timer;
timing_t *var_timer;
timing_t *em_timer;
int32 full_covar;
km_timer = timing_get("km");
var_timer = timing_get("var");
em_timer = timing_get("em");
blksz = feat_blksize();
full_covar = cmd_ln_int32("-fullvar");
/* fully-continuous for now */
mean = gauden_alloc_param(ts_cnt, n_stream, n_density, veclen);
if (full_covar)
fullvar = gauden_alloc_param_full(ts_cnt, n_stream, n_density, veclen);
else
var = gauden_alloc_param(ts_cnt, n_stream, n_density, veclen);
if (mixwfn)
mixw = (float32 ***)ckd_calloc_3d(ts_cnt,
n_stream,
n_density,
sizeof(float32));
if ((const char *)cmd_ln_access("-segidxfn")) {
E_INFO("Multi-class dump\n");
if (segdmp_open_read((const char **)cmd_ln_access("-segdmpdirs"),
(const char *)cmd_ln_access("-segdmpfn"),
(const char *)cmd_ln_access("-segidxfn"),
&n,
&t) != S3_SUCCESS) {
E_FATAL("Unable to open dumps\n");
}
if (n != n_d_ts) {
E_FATAL("Expected %u tied-states in dump, but apparently %u\n",
n_d_ts, n);
}
if (t != SEGDMP_TYPE_FEAT) {
E_FATAL("Expected feature dump, but instead saw %u\n", t);
}
multiclass = TRUE;
}
else {
E_INFO("1-class dump file\n");
multiclass = FALSE;
dmp_fp = s3open((const char *)cmd_ln_access("-segdmpfn"), "rb",
&dmp_swp);
if (dmp_fp == NULL) {
E_ERROR_SYSTEM("Unable to open dump file %s for reading\n",
(const char *)cmd_ln_access("-segdmpfn"));
return S3_ERROR;
}
if (s3read(&n_frame, sizeof(uint32), 1, dmp_fp, dmp_swp, &ignore) != 1) {
E_ERROR_SYSTEM("Unable to open dump file %s for reading\n",
(const char *)cmd_ln_access("-segdmpfn"));
return S3_ERROR;
}
data_offset = ftell(dmp_fp);
}
tot_sqerr = 0;
for (i = 0; i < ts_cnt; i++) {
ts = ts_off + i;
/* stride not accounted for yet */
if (o2d == NULL) {
if (multiclass)
n_frame = segdmp_n_seg(ts);
}
else {
for (j = 0, n_frame = 0; j < n_o2d[ts]; j++) {
n_frame += segdmp_n_seg(o2d[ts][j]);
}
}
E_INFO("Corpus %u: sz==%u frames%s\n",
ts, n_frame,
(n_frame > *(uint32 *)cmd_ln_access("-vartiethr") ? "" : " tied var"));
if (n_frame == 0) {
continue;
}
E_INFO("Convergence ratios are abs(cur - prior) / abs(prior)\n");
/* Do some variety of k-means clustering */
if (km_timer)
timing_start(km_timer);
sqerr = cluster(ts, n_stream, n_frame, veclen, mean[i], n_density, &label);
if (km_timer)
timing_stop(km_timer);
if (sqerr < 0) {
E_ERROR("Unable to do k-means for state %u; skipping...\n", ts);
continue;
}
/* Given the k-means and assuming equal prior liklihoods
* compute the variances */
if (var_timer)
timing_start(var_timer);
if (full_covar)
full_variances(ts, mean[i], fullvar[i], n_density, veclen,
n_frame, n_stream, label);
else
variances(ts, mean[i], var[i], n_density, veclen, n_frame, n_stream, label);
if (var_timer)
timing_stop(var_timer);
if (mixwfn) {
/* initialize the mixing weights by counting # of occurrances
* of the top codeword over the corpus and normalizing */
init_mixw(mixw[i], mean[i], n_density, veclen, n_frame, n_stream, label);
ckd_free(label);
if (reest == TRUE && full_covar)
E_ERROR("EM re-estimation is not yet supported for full covariances\n");
else if (reest == TRUE) {
if (em_timer)
timing_start(em_timer);
/* Do iterations of EM to estimate the mixture densities */
reest_sum(ts, mean[i], var[i], mixw[i], n_density, n_stream,
n_frame, veclen,
*(uint32 *)cmd_ln_access("-niter"),
FALSE,
*(uint32 *)cmd_ln_access("-vartiethr"));
if (em_timer)
timing_stop(em_timer);
}
}
++n_corpus;
tot_sqerr += sqerr;
E_INFO("sqerr [%u] == %e\n", ts, sqerr);
}
if (n_corpus > 0) {
E_INFO("sqerr = %e tot %e rms\n", tot_sqerr, sqrt(tot_sqerr/n_corpus));
}
if (!multiclass)
s3close(dmp_fp);
if (meanfn) {
if (s3gau_write(meanfn,
(const vector_t ***)mean,
ts_cnt,
n_stream,
n_density,
veclen) != S3_SUCCESS) {
return S3_ERROR;
}
}
else {
E_INFO("No mean file given; none written\n");
}
if (varfn) {
if (full_covar) {
if (s3gau_write_full(varfn,
(const vector_t ****)fullvar,
ts_cnt,
n_stream,
n_density,
veclen) != S3_SUCCESS)
return S3_ERROR;
}
else {
if (s3gau_write(varfn,
(const vector_t ***)var,
ts_cnt,
n_stream,
n_density,
veclen) != S3_SUCCESS)
return S3_ERROR;
}
}
else {
E_INFO("No variance file given; none written\n");
}
if (mixwfn) {
if (s3mixw_write(mixwfn,
mixw,
ts_cnt,
n_stream,
n_density) != S3_SUCCESS) {
return S3_ERROR;
}
}
else {
E_INFO("No mixing weight file given; none written\n");
}
return S3_SUCCESS;
}
int32
viterbi_update(float64 *log_forw_prob,
vector_t **feature,
uint32 n_obs,
state_t *state_seq,
uint32 n_state,
model_inventory_t *inv,
float64 a_beam,
float32 spthresh,
s3phseg_t *phseg,
int32 mixw_reest,
int32 tmat_reest,
int32 mean_reest,
int32 var_reest,
int32 pass2var,
int32 var_is_full,
FILE *pdumpfh,
feat_t *fcb)
{
float64 *scale = NULL;
float64 **dscale = NULL;
float64 **active_alpha;
uint32 **active_astate;
uint32 **bp;
uint32 *n_active_astate;
gauden_t *g; /* Gaussian density parameters and
reestimation sums */
float32 ***mixw; /* all mixing weights */
float64 ***now_den = NULL; /* Short for den[t] */
uint32 ***now_den_idx = NULL;/* Short for den_idx[t] */
uint32 *active_cb;
uint32 n_active_cb;
float32 **tacc; /* Transition matrix reestimation sum accumulators
for the utterance. */
float32 ***wacc; /* mixing weight reestimation sum accumulators
for the utterance. */
float32 ***denacc = NULL; /* mean/var reestimation accumulators for time t */
size_t denacc_size; /* Total size of data references in denacc. Allows
for quick clears between time frames */
uint32 n_lcl_cb;
uint32 *cb_inv;
uint32 i, j, q;
int32 t;
uint32 n_feat;
uint32 n_density;
uint32 n_top;
int ret;
timing_t *fwd_timer = NULL;
timing_t *rstu_timer = NULL;
timing_t *gau_timer = NULL;
timing_t *rsts_timer = NULL;
timing_t *rstf_timer = NULL;
float64 log_fp; /* accumulator for the log of the probability
* of observing the input given the model */
uint32 max_n_next = 0;
uint32 n_cb;
static float64 *p_op = NULL;
static float64 *p_ci_op = NULL;
static float64 **d_term = NULL;
static float64 **d_term_ci = NULL;
/* caller must ensure that there is some non-zero amount
of work to be done here */
assert(n_obs > 0);
assert(n_state > 0);
/* Get the forward estimation CPU timer */
fwd_timer = timing_get("fwd");
/* Get the per utterance reestimation CPU timer */
rstu_timer = timing_get("rstu");
/* Get the Gaussian density evaluation CPU timer */
gau_timer = timing_get("gau");
/* Get the per state reestimation CPU timer */
rsts_timer = timing_get("rsts");
/* Get the per frame reestimation CPU timer */
rstf_timer = timing_get("rstf");
g = inv->gauden;
n_feat = gauden_n_feat(g);
n_density = gauden_n_density(g);
n_top = gauden_n_top(g);
n_cb = gauden_n_mgau(g);
if (p_op == NULL) {
p_op = ckd_calloc(n_feat, sizeof(float64));
p_ci_op = ckd_calloc(n_feat, sizeof(float64));
}
if (d_term == NULL) {
d_term = (float64 **)ckd_calloc_2d(n_feat, n_top, sizeof(float64));
d_term_ci = (float64 **)ckd_calloc_2d(n_feat, n_top, sizeof(float64));
}
scale = (float64 *)ckd_calloc(n_obs, sizeof(float64));
dscale = (float64 **)ckd_calloc(n_obs, sizeof(float64 *));
n_active_astate = (uint32 *)ckd_calloc(n_obs, sizeof(uint32));
active_alpha = (float64 **)ckd_calloc(n_obs, sizeof(float64 *));
active_astate = (uint32 **)ckd_calloc(n_obs, sizeof(uint32 *));
active_cb = ckd_calloc(2*n_state, sizeof(uint32));
bp = (uint32 **)ckd_calloc(n_obs, sizeof(uint32 *));
/* Run forward algorithm, which has embedded Viterbi. */
if (fwd_timer)
timing_start(fwd_timer);
ret = forward(active_alpha, active_astate, n_active_astate, bp,
scale, dscale,
feature, n_obs, state_seq, n_state,
inv, a_beam, phseg, 0);
/* Dump a phoneme segmentation if requested */
if (cmd_ln_str("-outphsegdir")) {
const char *phsegdir;
char *segfn, *uttid;
phsegdir = cmd_ln_str("-outphsegdir");
uttid = (cmd_ln_int32("-outputfullpath")
? corpus_utt_full_name() : corpus_utt());
segfn = ckd_calloc(strlen(phsegdir) + 1
+ strlen(uttid)
+ strlen(".phseg") + 1, 1);
strcpy(segfn, phsegdir);
strcat(segfn, "/");
strcat(segfn, uttid);
strcat(segfn, ".phseg");
write_phseg(segfn, inv, state_seq, active_astate, n_active_astate,
n_state, n_obs, active_alpha, scale, bp);
ckd_free(segfn);
}
if (fwd_timer)
timing_stop(fwd_timer);
if (ret != S3_SUCCESS) {
/* Some problem with the utterance, release per utterance storage and
* forget about adding the utterance accumulators to the global accumulators */
goto all_done;
}
mixw = inv->mixw;
if (mixw_reest) {
/* Need to reallocate mixing accumulators for utt */
if (inv->l_mixw_acc) {
ckd_free_3d((void ***)inv->l_mixw_acc);
inv->l_mixw_acc = NULL;
}
inv->l_mixw_acc = (float32 ***)ckd_calloc_3d(inv->n_mixw_inverse,
n_feat,
n_density,
sizeof(float32));
}
wacc = inv->l_mixw_acc;
n_lcl_cb = inv->n_cb_inverse;
cb_inv = inv->cb_inverse;
/* Allocate local accumulators for mean, variance reestimation
sums if necessary */
gauden_alloc_l_acc(g, n_lcl_cb,
mean_reest, var_reest,
var_is_full);
if (tmat_reest) {
if (inv->l_tmat_acc) {
ckd_free_2d((void **)inv->l_tmat_acc);
inv->l_tmat_acc = NULL;
}
for (i = 0; i < n_state; i++) {
if (state_seq[i].n_next > max_n_next)
max_n_next = state_seq[i].n_next;
}
inv->l_tmat_acc = (float32 **)ckd_calloc_2d(n_state,
max_n_next,
sizeof(float32));
}
/* transition matrix reestimation sum accumulators
for the utterance */
tacc = inv->l_tmat_acc;
n_active_cb = 0;
now_den = (float64 ***)ckd_calloc_3d(n_lcl_cb,
n_feat,
n_top,
sizeof(float64));
now_den_idx = (uint32 ***)ckd_calloc_3d(n_lcl_cb,
n_feat,
n_top,
sizeof(uint32));
if (mean_reest || var_reest) {
/* allocate space for the per frame density counts */
denacc = (float32 ***)ckd_calloc_3d(n_lcl_cb,
n_feat,
n_density,
sizeof(float32));
/* # of bytes required to store all weighted vectors */
denacc_size = n_lcl_cb * n_feat * n_density * sizeof(float32);
}
else {
denacc = NULL;
denacc_size = 0;
}
/* Okay now run through the backtrace and accumulate counts. */
/* Find the non-emitting ending state */
for (q = 0; q < n_active_astate[n_obs-1]; ++q) {
if (active_astate[n_obs-1][q] == n_state-1)
break;
}
if (q == n_active_astate[n_obs-1]) {
E_ERROR("Failed to align audio to trancript: final state of the search is not reached\n");
ret = S3_ERROR;
goto all_done;
}
for (t = n_obs-1; t >= 0; --t) {
uint32 l_cb;
uint32 l_ci_cb;
float64 op, p_reest_term;
uint32 prev;
j = active_astate[t][q];
/* Follow any non-emitting states at time t first. */
while (state_seq[j].mixw == TYING_NON_EMITTING) {
prev = active_astate[t][bp[t][q]];
#if VITERBI_DEBUG
printf("Following non-emitting state at time %d, %u => %u\n",
t, j, prev);
#endif
/* Backtrace and accumulate transition counts. */
if (tmat_reest) {
assert(tacc != NULL);
tacc[prev][j - prev] += 1.0;
}
q = bp[t][q];
j = prev;
}
/* Now accumulate statistics for the real state. */
l_cb = state_seq[j].l_cb;
l_ci_cb = state_seq[j].l_ci_cb;
n_active_cb = 0;
if (gau_timer)
timing_start(gau_timer);
gauden_compute_log(now_den[l_cb],
now_den_idx[l_cb],
feature[t],
g,
state_seq[j].cb,
NULL);
active_cb[n_active_cb++] = l_cb;
if (l_cb != l_ci_cb) {
gauden_compute_log(now_den[l_ci_cb],
now_den_idx[l_ci_cb],
feature[t],
g,
state_seq[j].ci_cb,
NULL);
active_cb[n_active_cb++] = l_ci_cb;
}
gauden_scale_densities_bwd(now_den, now_den_idx,
&dscale[t],
active_cb, n_active_cb, g);
assert(state_seq[j].mixw != TYING_NON_EMITTING);
/* Now calculate mixture densities. */
/* This is the normalizer sum_m c_{jm} p(o_t|\lambda_{jm}) */
op = gauden_mixture(now_den[l_cb], now_den_idx[l_cb],
mixw[state_seq[j].mixw], g);
if (gau_timer)
timing_stop(gau_timer);
if (rsts_timer)
timing_start(rsts_timer);
/* Make up this bogus value to be consistent with backward.c */
p_reest_term = 1.0 / op;
/* Compute the output probability excluding the contribution
* of each feature stream. i.e. p_op[0] is the output
* probability excluding feature stream 0 */
partial_op(p_op,
op,
now_den[l_cb],
now_den_idx[l_cb],
mixw[state_seq[j].mixw],
n_feat,
n_top);
/* compute the probability of each (of possibly topn) density */
den_terms(d_term,
p_reest_term,
p_op,
now_den[l_cb],
now_den_idx[l_cb],
mixw[state_seq[j].mixw],
n_feat,
n_top);
if (l_cb != l_ci_cb) {
/* For each feature stream f, compute:
* sum_k(mixw[f][k] den[f][k])
* and store the results in p_ci_op */
partial_ci_op(p_ci_op,
now_den[l_ci_cb],
now_den_idx[l_ci_cb],
mixw[state_seq[j].ci_mixw],
n_feat,
n_top);
/* For each feature stream and density compute the terms:
* w[f][k] den[f][k] / sum_k(w[f][k] den[f][k]) * post_j
* and store results in d_term_ci */
den_terms_ci(d_term_ci,
1.0, /* post_j = 1.0 */
p_ci_op,
now_den[l_ci_cb],
now_den_idx[l_ci_cb],
mixw[state_seq[j].ci_mixw],
n_feat,
n_top);
}
/* accumulate the probability for each density in the mixing
* weight reestimation accumulators */
if (mixw_reest) {
accum_den_terms(wacc[state_seq[j].l_mixw], d_term,
now_den_idx[l_cb], n_feat, n_top);
/* check if mixw and ci_mixw are different to avoid
* doubling the EM counts in a CI run. */
if (state_seq[j].mixw != state_seq[j].ci_mixw) {
if (n_cb < inv->n_mixw) {
/* semi-continuous, tied mixture, and discrete case */
accum_den_terms(wacc[state_seq[j].l_ci_mixw], d_term,
now_den_idx[l_cb], n_feat, n_top);
}
else {
/* continuous case */
accum_den_terms(wacc[state_seq[j].l_ci_mixw], d_term_ci,
now_den_idx[l_ci_cb], n_feat, n_top);
}
}
}
/* accumulate the probability for each density in the
* density reestimation accumulators */
if (mean_reest || var_reest) {
accum_den_terms(denacc[l_cb], d_term,
now_den_idx[l_cb], n_feat, n_top);
if (l_cb != l_ci_cb) {
accum_den_terms(denacc[l_ci_cb], d_term_ci,
now_den_idx[l_ci_cb], n_feat, n_top);
}
}
if (rsts_timer)
timing_stop(rsts_timer);
/* Note that there is only one state/frame so this is kind of
redundant */
if (rstf_timer)
timing_start(rstf_timer);
if (mean_reest || var_reest) {
/* Update the mean and variance reestimation accumulators */
if (pdumpfh)
fprintf(pdumpfh, "time %d:\n", t);
accum_gauden(denacc,
cb_inv,
n_lcl_cb,
feature[t],
now_den_idx,
g,
mean_reest,
var_reest,
pass2var,
inv->l_mixw_acc,
var_is_full,
pdumpfh,
fcb);
memset(&denacc[0][0][0], 0, denacc_size);
}
if (rstf_timer)
timing_stop(rstf_timer);
if (t > 0) {
prev = active_astate[t-1][bp[t][q]];
#if VITERBI_DEBUG
printf("Backtrace at time %d, %u => %u\n",
t, j, prev);
#endif
/* Backtrace and accumulate transition counts. */
if (tmat_reest) {
assert(tacc != NULL);
tacc[prev][j-prev] += 1.0;
}
q = bp[t][q];
j = prev;
}
}
/* If no error was found, add the resulting utterance reestimation
* accumulators to the global reestimation accumulators */
if (rstu_timer)
timing_start(rstu_timer);
accum_global(inv, state_seq, n_state,
mixw_reest, tmat_reest, mean_reest, var_reest,
var_is_full);
if (rstu_timer)
timing_stop(rstu_timer);
/* Find the final state */
for (i = 0; i < n_active_astate[n_obs-1]; ++i) {
if (active_astate[n_obs-1][i] == n_state-1)
break;
}
/* Calculate log[ p( O | \lambda ) ] */
assert(active_alpha[n_obs-1][i] > 0);
log_fp = log(active_alpha[n_obs-1][i]);
for (t = 0; t < n_obs; t++) {
assert(scale[t] > 0);
log_fp -= log(scale[t]);
for (j = 0; j < inv->gauden->n_feat; j++) {
log_fp += dscale[t][j];
}
}
*log_forw_prob = log_fp;
all_done:
ckd_free((void *)scale);
for (i = 0; i < n_obs; i++) {
if (dscale[i])
ckd_free((void *)dscale[i]);
}
ckd_free((void **)dscale);
ckd_free(n_active_astate);
for (i = 0; i < n_obs; i++) {
ckd_free((void *)active_alpha[i]);
ckd_free((void *)active_astate[i]);
ckd_free((void *)bp[i]);
}
ckd_free((void *)active_alpha);
ckd_free((void *)active_astate);
ckd_free((void *)active_cb);
if (denacc)
ckd_free_3d((void ***)denacc);
if (now_den)
ckd_free_3d((void ***)now_den);
if (now_den_idx)
ckd_free_3d((void ***)now_den_idx);
if (ret != S3_SUCCESS)
E_ERROR("%s ignored\n", corpus_utt_brief_name());
return ret;
}
/*
* Quantize pt[0..n_pt-1][0..veclen-1] into cb[0..vqsize-1][0..veclen-1] (where
* vqsize < n_pt, presumably). Do this with the following iterative procedure:
* 1. Choose an initial VQ codebook by selecting vqsize random points from pt.
* 2. Map each point in pt to the "nearest" codebook entry (currently based on
* Euclidean distance.
* 3. Re-estimate each VQ entry by taking the centroid of all pt entries mapped
* to it in step 2.
* 4. Repeat steps 2 and 3 until the "total error stabilizes".
* In the end, replace each point in pt with the nearest VQ value.
* Return value: final total error.
*/
static float64 vq (float32 **pt, float32 **cb, int32 n_pt, int32 vqsize, int32 veclen)
{
int32 p, c, i, iter, bestc, *pt2cb, *n_newcb;
float64 d, bestdist, err, prev_err;
float32 **newcb;
E_INFO("Clustering %d points into %d\n", n_pt, vqsize);
/* Allocate some temporary variables */
pt2cb = (int32 *) ckd_calloc (n_pt, sizeof(int32));
newcb = (float32 **)ckd_calloc_2d (vqsize, veclen, sizeof(float32));
n_newcb = (int32 *) ckd_calloc (vqsize, sizeof(int32));
/* Choose an initial codebook */
vq_init (pt, cb, n_pt, vqsize, veclen);
for (iter = 0;; iter++) {
timing_start (tmg);
/* Map each point to closest codebook entry (using Euclidean distance metric) */
err = 0.0;
for (p = 0; p < n_pt; p++) {
bestdist = 1e+200;
for (c = 0; c < vqsize; c++) {
d = vecdist (pt[p], cb[c], veclen);
if (d < bestdist) {
bestdist = d;
bestc = c;
}
}
pt2cb[p] = bestc;
err += bestdist;
}
/* Update codebook entries with centroid of mapped points */
for (c = 0; c < vqsize; c++) {
for (i = 0; i < veclen; i++)
newcb[c][i] = 0.0;
n_newcb[c] = 0;
}
for (p = 0; p < n_pt; p++) {
c = pt2cb[p];
for (i = 0; i < veclen; i++)
newcb[c][i] += pt[p][i];
n_newcb[c]++;
}
for (c = 0; c < vqsize; c++) {
if (n_newcb[c] == 0)
E_ERROR("Nothing mapped to codebook entry %d; entry not updated\n", c);
else {
float64 t;
t = 1.0 / n_newcb[c];
for (i = 0; i < veclen; i++)
cb[c][i] = newcb[c][i] * t;
}
}
timing_stop (tmg);
E_INFO("%4d: Error = %e, %.2f sec CPU, %.2f sec elapsed\n",
iter, err, tmg->t_cpu, tmg->t_elapsed);
timing_reset (tmg);
/* Check if VQ codebook converged */
if (iter > 10) {
if ((err == 0.0) || ((prev_err - err)/prev_err < 0.002))
break;
}
prev_err = err;
}
/* Replace points with nearest VQ entries created */
for (p = 0; p < n_pt; p++) {
c = pt2cb[p];
for (i = 0; i < veclen; i++)
pt[p][i] = cb[c][i];
}
ckd_free (pt2cb);
ckd_free_2d ((void **) newcb);
ckd_free (n_newcb);
return err;
}
int main(int argc, char *argv[])
{
//////////////////////////////////////////////////////////////////////
// Handel argument
if (argc != 4) {
usage(argv[0]);
exit(1);
}
const unsigned int shuffle_time = atof(argv[1]);
const char* inputX = argv[2];
const char* inputY = argv[3];
debug_level_set(SILENT);
printf("begin allocating memory:\n");
vec_p x, y;
x = vector_new(4);
y = vector_new(4);
printf("begin read:\n");
vector_read(x, inputX);
vector_read(y, inputY);
// vector_print(x);
double* ret = malloc(sizeof(double)*shuffle_time);
assert(ret);
//////////////////////////////////////////////////////////////////////
// Method 1
// parallel this part
timing_start();
for (int i = 0; i < shuffle_time; i++) {
inplace_shuffle(y->value, y->len);
ret[i] = vector_corr(x,y);
}
timing_stop();
timing_diff();
//////////////////////////////////////////////////////////////////////
// Method 2
timing_start();
vector_corr_permutation(x, y, ret, shuffle_time);
timing_stop();
timing_diff();
//////////////////////////////////////////////////////////////////////
// Method 3
timing_start();
mkl_vector_corr_permutation(x, y, ret, shuffle_time);
timing_stop();
timing_diff();
exit(0);
// output result
printf("Result\n");
for (int i = 0; i < shuffle_time; i++) {
printf("%f\n", ret[i]);
}
vector_delete(&x);
vector_delete(&y);
free(ret);
return 0;
}
/* Decode the given mfc file and write result to matchfp and matchsegfp */
static void decode_utt (char *uttid, FILE *matchfp, FILE *matchsegfp)
{
char dagfile[1024];
hyp_t *h, *hyp;
char *latdir, *latext;
int32 nfrm, ascr, lscr;
timing_reset (tm_utt);
timing_start (tm_utt);
latdir = (char *) cmd_ln_access ("-inlatdir");
latext = (char *) cmd_ln_access ("-latext");
if (latdir)
sprintf (dagfile, "%s/%s.%s", latdir, uttid, latext);
else
sprintf (dagfile, "%s.%s", uttid, latext);
if ((nfrm = dag_load (dagfile)) >= 0) {
hyp = dag_search (uttid);
if ( *((int32 *) cmd_ln_access("-backtrace")) )
log_hyp_detailed (stdout, hyp, uttid, "BP", "bp");
/* Total scaled acoustic score and LM score */
ascr = lscr = 0;
for (h = hyp; h; h = h->next) {
ascr += h->ascr;
lscr += h->lscr;
}
printf ("BSTPTH: ");
log_hypstr (stdout, hyp, uttid, ascr+lscr);
printf ("BSTXCT: ");
log_hypseg (uttid, stdout, hyp, nfrm);
lm_cache_stats_dump ();
lm_cache_reset ();
} else {
E_ERROR("DAG search (%s) failed\n", uttid);
hyp = NULL;
}
/* Log recognition output to the standard match and matchseg files */
if (matchfp)
log_hypstr (matchfp, hyp, uttid, 0);
if (matchsegfp)
log_hypseg (uttid, matchsegfp, hyp, nfrm);
dag_destroy ();
timing_stop (tm_utt);
printf ("%s: TMR: %5d Frm", uttid, nfrm);
if (nfrm > 0) {
printf (" %6.2f xEl", tm_utt->t_elapsed * 100.0 / nfrm);
printf (" %6.2f xCPU", tm_utt->t_cpu * 100.0 / nfrm);
}
printf ("\n");
fflush (stdout);
tot_nfr += nfrm;
}
void region_seeker_server(region_seeker_input_t *input_p){
printf("region_seeker_server(%d): START\n", omp_get_thread_num());
list_item_t *item_p = NULL;
list_item_t *cal_item_p = NULL;
fastq_batch_t *unmapped_batch_p;
size_t num_reads;
array_list_t **allocate_mapping_p;
cal_batch_t *cal_batch_p;
size_t num_mappings, total_mappings = 0, num_batches = 0;
size_t num_threads = input_p->region_threads;
size_t chunk;
size_t total_reads = 0;
omp_set_num_threads(num_threads);
while ( (item_p = list_remove_item(input_p->unmapped_read_list_p)) != NULL ) {
//printf("Region Seeker Processing batch...\n");
num_batches++;
if (time_on) { timing_start(REGION_SEEKER, 0, timing_p); }
unmapped_batch_p = (fastq_batch_t *)item_p->data_p;
num_reads = unmapped_batch_p->num_reads;
total_reads += num_reads;
allocate_mapping_p = (array_list_t **)malloc(sizeof(array_list_t *)*num_reads);
if (input_p->gpu_enable) {
//******************************* GPU PROCESS *********************************//
for (size_t i = 0; i < num_reads; i++) {
allocate_mapping_p[i] = array_list_new(1000,
1.25f,
COLLECTION_MODE_ASYNCHRONIZED);
}
#ifdef HPG_GPU
num_mappings = bwt_map_exact_seed_batch_gpu(unmapped_batch_p,
input_p->bwt_optarg_p,
input_p->cal_optarg_p,
input_p->bwt_index_p,
input_p->gpu_context,
allocate_mapping_p);
#endif
//****************************************************************************//
} else {
//******************************* CPU PROCESS *********************************//
//printf("Region Seeker :: Process Batch with %d reads\n", num_reads);
chunk = MAX(1, num_reads/(num_threads*10));
//printf("Region Seeker :: Process Batch with %d reads\n", num_reads);
#pragma omp parallel for private(num_mappings) reduction(+:total_mappings) schedule(dynamic, chunk)
//#pragma omp parallel for private(num_mappings) reduction(+:total_mappings) schedule(static)
for (size_t i = 0; i < num_reads; i++) {
//printf("Threads region zone: %d\n", omp_get_num_threads());
allocate_mapping_p[i] = array_list_new(1000,
1.25f,
COLLECTION_MODE_ASYNCHRONIZED);
num_mappings = bwt_map_exact_seeds_seq(&(unmapped_batch_p->seq[unmapped_batch_p->data_indices[i]]),
input_p->cal_optarg_p->seed_size,
input_p->cal_optarg_p->min_seed_size,
input_p->bwt_optarg_p, input_p->bwt_index_p, allocate_mapping_p[i]);
total_mappings += num_mappings;
//printf("----------------->>>>>>>>>>>Regions found %d\n", num_mappings);
}
//****************************************************************************//
}
cal_batch_p = cal_batch_new(allocate_mapping_p, unmapped_batch_p);
list_item_free(item_p);
cal_item_p = list_item_new(0, 0, cal_batch_p);
//region_batch_free(region_batch_p);
if (time_on) { timing_stop(REGION_SEEKER, 0, timing_p); }
list_insert_item(cal_item_p, input_p->region_list_p);
//printf("Region Seeker Processing batch finish!\n");
} //End of while
list_decr_writers(input_p->region_list_p);
if (statistics_on) {
statistics_set(REGION_SEEKER_ST, 0, num_batches, statistics_p);
statistics_set(REGION_SEEKER_ST, 1, total_reads, statistics_p);
}
printf("region_seeker_server: END\n");
}
/* Main program. */
int main(void)
{
int i, j, n_samples, max_n, step_n;
int array_size;
int radix;
test_item_t a[N_ITEMS], test_array[N_ITEMS];
test_item_t *timing_array, *copy_array;
timing_t *t;
/* Assign random values to the key of each array element. */
rand_array(a, N_ITEMS, 100);
/* Now test quicksort(). */
memcpy(test_array, a, sizeof(test_array));
printf("array before quicksort\n = ");
print_array(test_array, N_ITEMS);
printf("\n\n");
quicksort(test_array, N_ITEMS, sizeof(test_item_t), item_cmp);
printf("array after quicksort\n = ");
print_array(test_array, N_ITEMS);
printf("\n\n");
/* Now test mergesort0(). */
memcpy(test_array, a, sizeof(test_array));
printf("array before mergesort0\n = ");
print_array(test_array, N_ITEMS);
printf("\n\n");
mergesort0(test_array, N_ITEMS, sizeof(test_item_t), item_cmp);
printf("array after mergesort0\n = ");
print_array(test_array, N_ITEMS);
printf("\n\n");
/* Now test mergesort(). */
memcpy(test_array, a, sizeof(test_array));
printf("array before mergesort\n = ");
print_array(test_array, N_ITEMS);
printf("\n\n");
mergesort(test_array, N_ITEMS, sizeof(test_item_t), item_cmp);
printf("array after mergesort\n = ");
print_array(test_array, N_ITEMS);
printf("\n\n");
/* Now test radix sort. */
memcpy(test_array, a, sizeof(test_array));
printf("array before radixsort\n = ");
print_array(test_array, N_ITEMS);
printf("\n\n");
radixsort(test_array, N_ITEMS, sizeof(test_item_t), get_value, 10);
printf("array after radixsort\n = ");
print_array(test_array, N_ITEMS);
printf("\n\n");
/* Now test heapsort. */
memcpy(test_array, a, sizeof(test_array));
printf("array before heapsort\n = ");
print_array(test_array, N_ITEMS);
printf("\n\n");
heapsort(test_array, N_ITEMS, sizeof(test_item_t), item_cmp);
printf("array after heapsort\n = ");
print_array(test_array, N_ITEMS);
printf("\n\n");
/* Time the quicksort and mergesort sorting functions. */
printf("Enter the number of samples to use: ");
scanf("%d", &n_samples);
printf("Enter the maximum array length to sort: ");
scanf("%d", &max_n);
printf("Enter the step size for array lengths: ");
scanf("%d", &step_n);
t = timing_alloc(5); /* Five different sorting algorithms. */
printf("\nResults (n, qsort, quicksort, mergesort, mergesort0, heapsort) (msec)\n"
);
for(i = step_n; i <= max_n; i += step_n) {
array_size = i * sizeof(test_item_t);
timing_array = malloc(array_size);
copy_array = malloc(array_size);
rand_array(copy_array, i, MAX_VALUE);
timing_reset(t);
for(j = 0; j < n_samples; j++) {
memcpy(timing_array, copy_array, array_size);
timing_start();
qsort(timing_array, i, sizeof(test_item_t), item_cmp);
timing_stop(t,0);
memcpy(timing_array, copy_array, array_size);
timing_start();
quicksort(timing_array, i, sizeof(test_item_t), item_cmp);
timing_stop(t,1);
memcpy(timing_array, copy_array, array_size);
timing_start();
mergesort(timing_array, i, sizeof(test_item_t), item_cmp);
timing_stop(t,2);
memcpy(timing_array, copy_array, array_size);
timing_start();
mergesort0(timing_array, i, sizeof(test_item_t), item_cmp);
timing_stop(t,3);
memcpy(timing_array, copy_array, array_size);
timing_start();
heapsort(timing_array, i, sizeof(test_item_t), item_cmp);
timing_stop(t,4);
}
printf("%d", i);
timing_print(t,"\t%.2f",n_samples);
putchar('\n');
free(timing_array);
free(copy_array);
}
timing_free(t);
/* Time radix sort on the largest array, using different radix sizes. */
printf("\nRadix Sort Results. Using n = %d\n", max_n);
printf("(radix, time)\n");
array_size = max_n * sizeof(test_item_t);
timing_array = malloc(array_size);
copy_array = malloc(array_size);
rand_array(copy_array, max_n, MAX_VALUE);
for(radix = 2; radix <= max_n; radix <<= 1) {
timer_reset();
for(j = 0; j < n_samples; j++) {
memcpy(timing_array, copy_array, array_size);
timer_start();
radixsort(timing_array, max_n, sizeof(test_item_t), get_value,
radix);
timer_stop();
}
printf("%d", radix);
timer_print("\t%.2f", n_samples);
putchar('\n');
}
free(timing_array);
free(copy_array);
return 0;
}
int32
baum_welch_update(float64 *log_forw_prob,
vector_t **feature,
uint32 n_obs,
state_t *state,
uint32 n_state,
model_inventory_t *inv,
float64 a_beam,
float64 b_beam,
float32 spthresh,
s3phseg_t *phseg,
int32 mixw_reest,
int32 tmat_reest,
int32 mean_reest,
int32 var_reest,
int32 pass2var,
int32 var_is_full,
FILE *pdumpfh,
float32 ***lda)
{
float64 *scale = NULL;
float64 **dscale = NULL;
float64 **active_alpha;
uint32 **active_astate;
uint32 **bp;
uint32 *n_active_astate;
float64 log_fp; /* accumulator for the log of the probability
* of observing the input given the model */
uint32 t; /* time */
int ret;
timing_t *fwd_timer = NULL;
timing_t *bwd_timer = NULL;
timing_t *rstu_timer = NULL;
uint32 i,j;
/* caller must ensure that there is some non-zero amount
of work to be done here */
assert(n_obs > 0);
assert(n_state > 0);
fwd_timer = timing_get("fwd");
bwd_timer = timing_get("bwd");
rstu_timer = timing_get("rstu");
scale = (float64 *)ckd_calloc(n_obs, sizeof(float64));
dscale = (float64 **)ckd_calloc(n_obs, sizeof(float64 *));
n_active_astate = (uint32 *)ckd_calloc(n_obs, sizeof(uint32));
active_alpha = (float64 **)ckd_calloc(n_obs, sizeof(float64 *));
active_astate = (uint32 **)ckd_calloc(n_obs, sizeof(uint32 *));
bp = (uint32 **)ckd_calloc(n_obs, sizeof(uint32 *));
/* Compute the scaled alpha variable and scale factors
* for all states and time subject to the pruning constraints */
if (fwd_timer)
timing_start(fwd_timer);
/*
* Debug?
* E_INFO("Before Forward search\n");
*/
ret = forward(active_alpha, active_astate, n_active_astate, bp,
scale, dscale,
feature, n_obs, state, n_state,
inv, a_beam, phseg);
#if BW_DEBUG
for (i=0 ; i < n_obs;i++){
E_INFO("Number of active states %d at time %d\n",n_active_astate[i],i);
E_INFO("Scale of time %d is %e \n",i,scale[i]);
for(j=0 ; j < n_active_astate[i];j++){
E_INFO("Active state: %d Active alpha: %e\n",active_astate[i][j], active_alpha[i][j]);
}
}
i=0;
j=0;
#endif
/* Dump a phoneme segmentation if requested */
if (cmd_ln_str("-outphsegdir")) {
const char *phsegdir;
char *segfn, *uttid;
phsegdir = cmd_ln_str("-outphsegdir");
uttid = (cmd_ln_int32("-outputfullpath")
? corpus_utt_full_name() : corpus_utt());
segfn = ckd_calloc(strlen(phsegdir) + 1
+ strlen(uttid)
+ strlen(".phseg") + 1, 1);
strcpy(segfn, phsegdir);
strcat(segfn, "/");
strcat(segfn, uttid);
strcat(segfn, ".phseg");
write_phseg(segfn, inv, state, active_astate, n_active_astate,
n_state, n_obs, active_alpha, scale, bp);
ckd_free(segfn);
}
if (fwd_timer)
timing_stop(fwd_timer);
if (ret != S3_SUCCESS) {
/* Some problem with the utterance, release per utterance storage and
* forget about adding the utterance accumulators to the global accumulators */
goto error;
}
/* Compute the scaled beta variable and update the reestimation
* sums */
if (bwd_timer)
timing_start(bwd_timer);
#if BW_DEBUG
E_INFO("Before Backward search\n");
#endif
ret = backward_update(active_alpha, active_astate, n_active_astate, scale, dscale,
feature, n_obs,
state, n_state,
inv, b_beam, spthresh,
mixw_reest, tmat_reest, mean_reest, var_reest, pass2var,
var_is_full, pdumpfh, lda);
if (bwd_timer)
timing_stop(bwd_timer);
if (ret != S3_SUCCESS) {
/* Some problem with the utterance, release per utterance storage and
* forget about adding the utterance accumulators to the global accumulators */
goto error;
}
#if BW_DEBUG
E_INFO("Before Global Accumulation\n");
#endif
/* If no error was found in the forward or backward procedures,
* add the resulting utterance reestimation accumulators to the
* global reestimation accumulators */
if (rstu_timer)
timing_start(rstu_timer);
accum_global(inv, state, n_state,
mixw_reest, tmat_reest, mean_reest, var_reest,
var_is_full);
if (rstu_timer)
timing_stop(rstu_timer);
for (i = 0; i < n_active_astate[n_obs-1] && active_astate[n_obs-1][i] != (n_state-1); i++);
assert(i < n_active_astate[n_obs-1]);
/* Calculate log[ p( O | \lambda ) ] */
assert(active_alpha[n_obs-1][i] > 0);
log_fp = log(active_alpha[n_obs-1][i]);
for (t = 0; t < n_obs; t++) {
assert(scale[t] > 0);
log_fp -= log(scale[t]);
for (j = 0; j < inv->gauden->n_feat; j++) {
log_fp += dscale[t][j];
}
}
*log_forw_prob = log_fp;
ckd_free((void *)scale);
ckd_free(n_active_astate);
for (i = 0; i < n_obs; i++) {
ckd_free((void *)active_alpha[i]);
ckd_free((void *)active_astate[i]);
ckd_free((void *)dscale[i]);
ckd_free((void *)bp[i]);
}
ckd_free((void *)active_alpha);
ckd_free((void *)active_astate);
ckd_free((void **)dscale);
return S3_SUCCESS;
error:
ckd_free((void *)scale);
for (i = 0; i < n_obs; i++) {
if (dscale[i])
ckd_free((void *)dscale[i]);
}
ckd_free((void **)dscale);
ckd_free(n_active_astate);
for (i = 0; i < n_obs; i++) {
ckd_free((void *)active_alpha[i]);
ckd_free((void *)active_astate[i]);
ckd_free((void *)bp[i]);
}
ckd_free((void *)active_alpha);
ckd_free((void *)active_astate);
E_ERROR("%s ignored\n", corpus_utt_brief_name());
return S3_ERROR;
}
void batch_writer(batch_writer_input_t* input_p) {
struct timespec ts;
ts.tv_sec = 1;
ts.tv_nsec = 0;
alignment_t **buffer_p;
bam1_t* bam1_p;
bam_header_t* bam_header_p;
bam_file_t* bam_file_p;
char* match_filename = input_p->match_filename;
//char* mismatch_filename = input_p->mismatch_filename;
char* splice_exact_filename = input_p->splice_exact_filename;
char* splice_extend_filename = input_p->splice_extend_filename;
list_t* list_p = input_p->list_p;
printf("batch_writer (%i): START\n", omp_get_thread_num());
list_item_t *item_p = NULL;
write_batch_t* batch_p;
FILE* fd;
FILE* splice_exact_fd = fopen(splice_exact_filename, "w");
FILE* splice_extend_fd = fopen(splice_extend_filename, "w");
//printf("HEADER FROM WRITE: %s\n", input_p->header_filename);
bam_header_p = bam_header_new(HUMAN, NCBI37, input_p->header_filename);
//bam_file_p = bam_fopen(match_filename);
bam_file_p = bam_fopen_mode(match_filename, bam_header_p, "w");
bam_fwrite_header(bam_header_p, bam_file_p);
// main loop
while ( (item_p = list_remove_item(list_p)) != NULL ) {
if (time_on) {
timing_start(BATCH_WRITER, 0, timing_p);
}
batch_p = (write_batch_t*) item_p->data_p;
//printf("*********************************Extract one item*********************************\n");
if (batch_p->flag == MATCH_FLAG || batch_p->flag == MISMATCH_FLAG) { //fd = match_fd;
//printf("start write alignment. Total %d\n", batch_p->size);
buffer_p = (alignment_t **)batch_p->buffer_p;
for(int i = 0; i < batch_p->size; i++) {
//alignment_print(buffer_p[i]);
bam1_p = convert_to_bam(buffer_p[i], 33);
bam_fwrite(bam1_p, bam_file_p);
bam_destroy1(bam1_p);
alignment_free(buffer_p[i]);
}
} else {
if (batch_p->flag == SPLICE_EXACT_FLAG) {
fd = splice_exact_fd;
}
else if (batch_p->flag == SPLICE_EXTEND_FLAG) {
fd = splice_extend_fd;
}
else {
fd = NULL;
}
if (fd != NULL) {
//printf("start write batch, %i bytes...\n", batch_p->size);
fwrite((char *)batch_p->buffer_p, batch_p->size, 1, fd);
//printf("write done !!\n");
//if (time_on) { stop_timer(t1_write, t2_write, write_time); }
}
}
//printf("Free batch\n");
write_batch_free(batch_p);
list_item_free(item_p);
if (time_on) {
timing_stop(BATCH_WRITER, 0, timing_p);
}
} // end of batch loop
//fclose(match_fd);
//fclose(mismatch_fd);
fclose(splice_exact_fd);
fclose(splice_extend_fd);
bam_fclose(bam_file_p);
//bam_header_free(bam_header_p);
printf("batch_writer: END\n");
}
void batch_writer2(batch_writer_input_t* input) {
printf("START: batch_writer (%i): START, for file %s\n",
omp_get_thread_num(), input->match_filename);
bam1_t *bam1;
bam_header_t *bam_header;
bam_file_t *bam_file;
alignment_t *alig;
char* match_filename = input->match_filename;
// char* splice_filename = input->splice_filename;
list_t *write_list = input->list_p;
array_list_t *array_list;
list_item_t *item = NULL;
aligner_batch_t *batch = NULL;
fastq_batch_t *fq_batch = NULL;
FILE* fd;
static char aux[10];
size_t read_len;
bam_header = bam_header_new(HUMAN, NCBI37, input->header_filename);
bam_file = bam_fopen_mode(match_filename, bam_header, "w");
bam_fwrite_header(bam_header, bam_file);
size_t num_reads = 0, num_items = 0, total_mappings = 0;
// main loop
while ( (item = list_remove_item(write_list)) != NULL ) {
// if (array_list == NULL) printf("batch_writer.c...\n");
batch = (aligner_batch_t *) item->data_p;
fq_batch = batch->fq_batch;
num_reads = batch->num_mapping_lists;
for (size_t i = 0; i < num_reads; i++) {
array_list = batch->mapping_lists[i];
// if (array_list == NULL) printf("READ %d, writer, list is NULL\n", i);
// printf("----> list == NULL ? %d\n", (array_list == NULL));
num_items = (array_list == NULL ? 0 : array_list_size(array_list));
// printf("----> number of items = %d, num_items <= 0 ? %d\n", num_items, num_items <= 0);
read_len = fq_batch->data_indices[i + 1] - fq_batch->data_indices[i] - 1;
// mapped or not mapped ?
if (num_items == 0) {
//printf("\tWRITE : read %i (%d items): unmapped...\n", i, num_items);
// calculating cigar
sprintf(aux, "%luX", read_len);
alig = alignment_new();
alignment_init_single_end(&(fq_batch->header[fq_batch->header_indices[i]])+1,
&(fq_batch->seq[fq_batch->data_indices[i]]),
&(fq_batch->quality[fq_batch->data_indices[i]]),
0,
0,
0,
aux, 1, 255, 0, 0, alig);
bam1 = convert_to_bam(alig, 33);
bam_fwrite(bam1, bam_file);
bam_destroy1(bam1);
// some cosmetic stuff before freeing the alignment,
// (in order to not free twice some fields)
alig->query_name = NULL;
alig->sequence = NULL;
alig->quality = NULL;
alig->cigar = NULL;
alignment_free(alig);
// printf("\tWRITE : read %i (%d items): unmapped...done !!\n", i, num_items);
} else {
// printf("\tWRITE : read %d (%d items): mapped...\n", i, num_items);
for (size_t j = 0; j < num_items; j++) {
alig = (alignment_t *) array_list_get(j, array_list);
if (alig != NULL) {
bam1 = convert_to_bam(alig, 33);
bam_fwrite(bam1, bam_file);
bam_destroy1(bam1);
alignment_free(alig);
}
}
// printf("\tWRITE : read %d (%d items): mapped...done !!\n", i, num_items);
}
if (array_list != NULL) array_list_free(array_list, NULL);
}
if (batch != NULL) aligner_batch_free(batch);
if (item != NULL) list_item_free(item);
if (time_on) {
timing_stop(BATCH_WRITER, 0, timing_p);
}
} // end of batch loop
bam_fclose(bam_file);
printf("END: batch_writer (total mappings %lu)\n", total_mappings);
}
