static void process_reffile (char *reffile)
{
FILE *rfp, *hfp;
char line[16384], uttid[4096], file[4096], lc_uttid[4096];
int32 i, k;
dagnode_t ref[MAX_UTT_LEN];
int32 nref, noov, nhyp;
int32 tot_err, tot_ref, tot_corr, tot_oov, tot_hyp;
dag_t *dag;
dpnode_t retval;
timing_t *tm;
char *latdir, *hypfile;
if ((rfp = fopen(reffile, "r")) == NULL)
E_FATAL("fopen(%s,r) failed\n", reffile);
latdir = (char *) cmd_ln_access ("-latdir");
hypfile = (char *) cmd_ln_access ("-hyp");
if ((! latdir) && (! hypfile))
E_FATAL("Both -latdir and -hyp arguments missing\n");
if (latdir && hypfile)
E_FATAL("-latdir and -hyp arguments are mutually exclusive\n");
hfp = NULL;
if (hypfile) {
if ((hfp = fopen(hypfile, "r")) == NULL)
E_FATAL("fopen(%s,r) failed\n", hypfile);
}
tot_err = 0;
tot_ref = 0;
tot_hyp = 0;
tot_corr = 0;
tot_oov = 0;
tm = timing_new ("Utt");
while (fgets(line, sizeof(line), rfp) != NULL) {
timing_reset (tm);
timing_start (tm);
if ((nref = refline2wds (line, ref, &noov, uttid)) < 0)
E_FATAL("Bad line in file %s: %s\n", reffile, line);
/* Read lattice or hypfile, whichever is specified */
if (latdir) {
sprintf (file, "%s/%s.lat", latdir, uttid);
dag = dag_load (file);
if (! dag) {
/* Try lower casing uttid */
strcpy (lc_uttid, uttid);
lcase (lc_uttid);
sprintf (file, "%s/%s.lat", latdir, lc_uttid);
dag = dag_load (file);
}
} else {
if (fgets(line, sizeof(line), hfp) == NULL)
E_FATAL("Premature EOF(%s) at uttid %s\n", hypfile, uttid);
dag = hypline2dag (uttid, line);
}
if (dag) {
/* Append sentinel silwid node to end of DAG */
dag_append_sentinel (dag, silwid);
/* Find best path (returns #errors/#correct and updates *nhyp) */
retval = dp (uttid, dict, oovbegin, ref, nref, dag, &nhyp, 0);
dag_destroy (dag);
} else {
retval.c = 0;
retval.e = nref-1;
nhyp = 0;
}
timing_stop (tm);
tot_ref += nref-1;
tot_hyp += nhyp;
tot_err += retval.e;
tot_corr += retval.c;
tot_oov += noov;
printf("(%s) << %d ref; %d %.1f%% oov; %d hyp; %d %.1f%% corr; %d %.1f%% err; %.1fs CPU >>\n",
uttid, nref-1,
noov, (nref > 1) ? (noov * 100.0) / (nref-1) : 0.0,
nhyp,
retval.c, (nref > 1) ? (retval.c * 100.0) / (nref-1) : 0.0,
retval.e, (nref > 1) ? (retval.e * 100.0) / (nref-1) : 0.0,
tm->t_cpu);
printf("== %7d ref; %5d %5.1f%% oov; %7d hyp; %7d %5.1f%% corr; %6d %5.1f%% err; %5.1fs CPU; %s\n",
tot_ref,
tot_oov, (tot_ref > 0) ? (tot_oov * 100.0) / tot_ref : 0.0,
tot_hyp,
tot_corr, (tot_ref > 0) ? (tot_corr * 100.0) / tot_ref : 0.0,
tot_err, (tot_ref > 0) ? (tot_err * 100.0) / tot_ref : 0.0,
tm->t_tot_cpu,
uttid);
fflush (stderr);
fflush (stdout);
}
fclose (rfp);
if (hfp)
fclose (hfp);
printf("SUMMARY: %d ref; %d %.3f%% oov; %d hyp; %d %.3f%% corr; %d %.3f%% err; %.1fs CPU\n",
tot_ref,
tot_oov, (tot_ref > 0) ? (tot_oov * 100.0) / tot_ref : 0.0,
tot_hyp,
tot_corr, (tot_ref > 0) ? (tot_corr * 100.0) / tot_ref : 0.0,
tot_err, (tot_ref > 0) ? (tot_err * 100.0) / tot_ref : 0.0,
tm->t_tot_cpu);
}
int
main_initialize(int argc,
char *argv[],
lexicon_t **out_lex,
model_def_t **out_omdef,
model_def_t **out_dmdef)
{
model_def_t *dmdef = NULL;
model_def_t *omdef = NULL;
lexicon_t *lex = NULL;
const char *fn;
uint32 n_ts;
uint32 n_cb;
const char *ts2cbfn;
parse_cmd_ln(argc, argv);
timing_bind_name("km", timing_new());
timing_bind_name("var", timing_new());
timing_bind_name("em", timing_new());
timing_bind_name("all", timing_new());
if (cmd_ln_access("-feat") != NULL) {
feat_set(cmd_ln_str("-feat"));
feat_set_in_veclen(cmd_ln_int32("-ceplen"));
feat_set_subvecs(cmd_ln_str("-svspec"));
}
else {
E_FATAL("You need to set a feature extraction config using -feat\n");
}
if (cmd_ln_access("-ldafn") != NULL) {
if (feat_read_lda(cmd_ln_access("-ldafn"), cmd_ln_int32("-ldadim"))) {
E_FATAL("Failed to read LDA matrix\n");
}
}
if (cmd_ln_access("-omoddeffn")) {
E_INFO("Reading output model definitions: %s\n", cmd_ln_access("-omoddeffn"));
/* Read in the model definitions. Defines the set of
CI phones and context dependent phones. Defines the
transition matrix tying and state level tying. */
if (model_def_read(&omdef,
cmd_ln_access("-omoddeffn")) != S3_SUCCESS) {
return S3_ERROR;
}
if (cmd_ln_access("-dmoddeffn")) {
E_INFO("Reading dump model definitions: %s\n", cmd_ln_access("-dmoddeffn"));
if (model_def_read(&dmdef,
cmd_ln_access("-dmoddeffn")) != S3_SUCCESS) {
return S3_ERROR;
}
setup_d2o_map(dmdef, omdef);
}
else {
E_INFO("Assuming dump and output model definitions are identical\n");
}
ts2cbfn = cmd_ln_access("-ts2cbfn");
if (ts2cbfn) {
if (strcmp(SEMI_LABEL, ts2cbfn) == 0) {
omdef->cb = semi_ts2cb(omdef->n_tied_state);
n_ts = omdef->n_tied_state;
n_cb = 1;
}
else if (strcmp(CONT_LABEL, ts2cbfn) == 0) {
omdef->cb = cont_ts2cb(omdef->n_tied_state);
n_ts = omdef->n_tied_state;
n_cb = omdef->n_tied_state;
}
else if (strcmp(PTM_LABEL, ts2cbfn) == 0) {
omdef->cb = ptm_ts2cb(omdef);
n_ts = omdef->n_tied_state;
n_cb = omdef->acmod_set->n_ci;
}
else if (s3ts2cb_read(cmd_ln_access("-ts2cbfn"),
&omdef->cb,
&n_ts,
&n_cb) != S3_SUCCESS) {
return S3_ERROR;
}
if (omdef->n_tied_state != n_ts) {
E_FATAL("Model definition file n_tied_state = %u, but %u mappings in ts2cb\n",
omdef->n_tied_state, n_ts);
}
}
}
else {
E_INFO("No mdef files. Assuming 1-class init\n");
}
*out_omdef = omdef;
*out_dmdef = dmdef;
fn = cmd_ln_access("-dictfn");
if (fn) {
E_INFO("Reading main lexicon: %s\n", fn);
lex = lexicon_read(NULL,
fn,
omdef->acmod_set);
if (lex == NULL)
return S3_ERROR;
}
fn = cmd_ln_access("-fdictfn");
if (fn) {
E_INFO("Reading filler lexicon: %s\n", fn);
(void)lexicon_read(lex,
fn,
omdef->acmod_set);
}
*out_lex = lex;
stride = *(int32 *)cmd_ln_access("-stride");
return S3_SUCCESS;
}
/*
* Read mean and variance S3-format files and cluster them as follows:
* assume single feature vector
* for each dimension of feature vector {
* create list of <mean,var> pairs in the entire codebook set (for that dimension);
* // HACK!! 0 vectors omitted from the above list
* VQ cluster this list into the given number of vqsize points;
* replace codebook entries with nearest clustered values (for that dimension);
* // HACK!! 0 vectors remain untouched
* }
* write remapped codebooks to files "mean-vq" and "var-vq";
*/
main (int32 argc, char *argv[])
{
float32 ****mean, ****var;
int32 n_cb, n_feat, n_den, *featlen;
int32 i, j, c, w, dim, p, n_pt, vqsize;
float32 **pt, **cb;
float64 err;
FILE *fp;
if ((argc < 4) || (sscanf (argv[3], "%d", &vqsize) != 1))
E_FATAL("Usage: %s meanfile varfile vqsize\n", argv[0]);
if (argc > 4) {
if ((fp = fopen(argv[4], "w")) != NULL) {
*stdout = *fp;
*stderr = *fp;
} else
E_ERROR("fopen(%s,w) failed\n", argv[4]);
}
gauden_param_read (&mean, &n_cb, &n_feat, &n_den, &featlen, argv[1]);
E_INFO("%s: %d x %d x %d ", argv[1], n_cb, n_feat, n_den);
for (i = 0; i < n_feat; i++)
printf (" %d", featlen[i]);
printf ("\n");
gauden_param_read (&var, &n_cb, &n_feat, &n_den, &featlen, argv[2]);
E_INFO("%s: %d x %d x %d ", argv[2], n_cb, n_feat, n_den);
for (i = 0; i < n_feat; i++)
printf (" %d", featlen[i]);
printf ("\n");
assert (n_feat == 1);
n_pt = n_cb * n_den;
pt = (float32 **) ckd_calloc_2d (n_pt, 2, sizeof(float32));
cb = (float32 **) ckd_calloc_2d (vqsize, 2, sizeof(float32));
tmg = timing_new ();
/* VQ each column of mean,var pair (assuming just one feature vector) */
for (dim = 0; dim < featlen[0]; dim++) {
j = 0;
for (c = 0; c < n_cb; c++) {
for (w = 0; w < n_den; w++) {
if ((mean[c][0][w][dim] != 0.0) || (var[c][0][w][dim] != 0.0)) {
pt[j][0] = mean[c][0][w][dim];
pt[j][1] = var[c][0][w][dim];
j++;
}
}
}
err = vq (pt, cb, j, vqsize, 2);
vq_dump (cb, vqsize, 2);
j = 0;
for (c = 0; c < n_cb; c++) {
for (w = 0; w < n_den; w++) {
if ((mean[c][0][w][dim] != 0.0) || (var[c][0][w][dim] != 0.0)) {
mean[c][0][w][dim] = pt[j][0];
var[c][0][w][dim] = pt[j][1];
j++;
}
}
}
E_INFO("%d values quantized for dimension %d; error = %e\n", j, dim, err);
}
fflush (fp);
gauden_param_write (mean, n_cb, n_feat, n_den, featlen, "mean-vq");
gauden_param_write (var, n_cb, n_feat, n_den, featlen, "var-vq");
fflush (fp);
exit(0);
}
main (int32 argc, char *argv[])
{
char *str;
#if 0
ckd_debug(100000);
#endif
E_INFO("%s COMPILED ON: %s, AT: %s\n\n", argv[0], __DATE__, __TIME__);
/* Digest command line argument definitions */
cmd_ln_define (defn);
if ((argc == 2) && (strcmp (argv[1], "help") == 0)) {
cmd_ln_print_definitions();
exit(1);
}
/* Look for default or specified arguments file */
str = NULL;
if ((argc == 2) && (argv[1][0] != '-'))
str = argv[1];
else if (argc == 1) {
str = "s3align.arg";
E_INFO("Looking for default argument file: %s\n", str);
}
if (str) {
/* Build command line argument list from file */
if ((argc = load_argfile (str, argv[0], &argv)) < 0) {
fprintf (stderr, "Usage:\n");
fprintf (stderr, "\t%s argument-list, or\n", argv[0]);
fprintf (stderr, "\t%s [argument-file] (default file: s3align.arg)\n\n",
argv[0]);
cmd_ln_print_definitions();
exit(1);
}
}
cmdline_parse (argc, argv);
if ((cmd_ln_access("-mdeffn") == NULL) ||
(cmd_ln_access("-meanfn") == NULL) ||
(cmd_ln_access("-varfn") == NULL) ||
(cmd_ln_access("-mixwfn") == NULL) ||
(cmd_ln_access("-tmatfn") == NULL) ||
(cmd_ln_access("-dictfn") == NULL))
E_FATAL("Missing -mdeffn, -meanfn, -varfn, -mixwfn, -tmatfn, or -dictfn argument\n");
if ((cmd_ln_access("-ctlfn") == NULL) || (cmd_ln_access("-insentfn") == NULL))
E_FATAL("Missing -ctlfn or -insentfn argument\n");
if ((cmd_ln_access ("-s2stsegdir") == NULL) &&
(cmd_ln_access ("-stsegdir") == NULL) &&
(cmd_ln_access ("-phsegdir") == NULL) &&
(cmd_ln_access ("-wdsegdir") == NULL) &&
(cmd_ln_access ("-outsentfn") == NULL))
E_FATAL("Missing output file/directory argument(s)\n");
tm_utt = timing_new ();
/*
* Initialize log(S3-base). All scores (probs...) computed in log domain to avoid
* underflow. At the same time, log base = 1.0001 (1+epsilon) to allow log values
* to be maintained in int32 variables without significant loss of precision.
*/
if (cmd_ln_access("-logbase") == NULL)
logs3_init (1.0001);
else {
float32 logbase;
logbase = *((float32 *) cmd_ln_access("-logbase"));
if (logbase <= 1.0)
E_FATAL("Illegal log-base: %e; must be > 1.0\n", logbase);
if (logbase > 1.1)
E_WARN("Logbase %e perhaps too large??\n", logbase);
logs3_init ((float64) logbase);
}
/* Initialize feature stream type */
feat_init ((char *) cmd_ln_access ("-feat"));
/* BHIKSHA: PASS CEPSIZE TO FEAT_CEPSIZE, 6 Jan 98 */
cepsize = *((int32 *) cmd_ln_access("-ceplen"));
cepsize = feat_cepsize (cepsize);
/* END CHANGES BY BHIKSHA */
/* Read in input databases */
models_init ();
senscale = (int32 *) ckd_calloc (S3_MAX_FRAMES, sizeof(int32));
tmr_utt = cyctimer_new ("U");
tmr_gauden = cyctimer_new ("G");
tmr_senone = cyctimer_new ("S");
tmr_align = cyctimer_new ("A");
/* Initialize align module */
align_init ();
printf ("\n");
tot_nfr = 0;
process_ctlfile ();
if (tot_nfr > 0) {
printf ("\n");
printf("TOTAL FRAMES: %8d\n", tot_nfr);
printf("TOTAL CPU TIME: %11.2f sec, %7.2f xRT\n",
tm_utt->t_tot_cpu, tm_utt->t_tot_cpu/(tot_nfr*0.01));
printf("TOTAL ELAPSED TIME: %11.2f sec, %7.2f xRT\n",
tm_utt->t_tot_elapsed, tm_utt->t_tot_elapsed/(tot_nfr*0.01));
}
#if (! WIN32)
system ("ps aguxwww | grep s3align");
#endif
/* Hack!! To avoid hanging problem under Linux */
if (logfp) {
fclose (logfp);
*stdout = orig_stdout;
*stderr = orig_stderr;
}
exit(0);
}
main (int32 argc, char *argv[])
{
char *str;
#if 0
ckd_debug(100000);
#endif
/* Digest command line argument definitions */
cmd_ln_define (defn);
if ((argc == 2) && (strcmp (argv[1], "help") == 0)) {
cmd_ln_print_definitions();
exit(1);
}
/* Look for default or specified arguments file */
str = NULL;
if ((argc == 2) && (argv[1][0] != '-'))
str = argv[1];
else if (argc == 1) {
str = "s3decode.arg";
E_INFO("Looking for default argument file: %s\n", str);
}
if (str) {
/* Build command line argument list from file */
if ((argc = load_argfile (str, argv[0], &argv)) < 0) {
fprintf (stderr, "Usage:\n");
fprintf (stderr, "\t%s argument-list, or\n", argv[0]);
fprintf (stderr, "\t%s [argument-file] (default file: s3decode.arg)\n\n",
argv[0]);
cmd_ln_print_definitions();
exit(1);
}
}
cmdline_parse (argc, argv);
/* Remove memory allocation restrictions */
unlimit ();
#if (! WIN32)
{
char buf[1024];
gethostname (buf, 1024);
buf[1023] = '\0';
E_INFO ("Executing on: %s\n", buf);
}
#endif
E_INFO("%s COMPILED ON: %s, AT: %s\n\n", argv[0], __DATE__, __TIME__);
if ((cmd_ln_access("-mdeffn") == NULL) ||
(cmd_ln_access("-dictfn") == NULL) ||
(cmd_ln_access("-lmfn") == NULL))
E_FATAL("Missing -mdeffn, -dictfn, or -lmfn argument\n");
/*
* Initialize log(S3-base). All scores (probs...) computed in log domain to avoid
* underflow. At the same time, log base = 1.0001 (1+epsilon) to allow log values
* to be maintained in int32 variables without significant loss of precision.
*/
if (cmd_ln_access("-logbase") == NULL)
logs3_init (1.0001);
else {
float32 logbase;
logbase = *((float32 *) cmd_ln_access("-logbase"));
if (logbase <= 1.0)
E_FATAL("Illegal log-base: %e; must be > 1.0\n", logbase);
if (logbase > 1.1)
E_WARN("Logbase %e perhaps too large??\n", logbase);
logs3_init ((float64) logbase);
}
/* Read in input databases */
models_init ();
/* Allocate timing object */
tm_utt = timing_new ();
tot_nfr = 0;
/* Initialize forward Viterbi search module */
dag_init ();
printf ("\n");
process_ctlfile ();
printf ("\n");
printf("TOTAL FRAMES: %8d\n", tot_nfr);
if (tot_nfr > 0) {
printf("TOTAL CPU TIME: %11.2f sec, %7.2f xRT\n",
tm_utt->t_tot_cpu, tm_utt->t_tot_cpu/(tot_nfr*0.01));
printf("TOTAL ELAPSED TIME: %11.2f sec, %7.2f xRT\n",
tm_utt->t_tot_elapsed, tm_utt->t_tot_elapsed/(tot_nfr*0.01));
}
fflush (stdout);
#if (! WIN32)
system ("ps auxwww | grep s3dag");
#endif
/* Hack!! To avoid hanging problem under Linux */
if (logfp) {
fclose (logfp);
*stdout = orig_stdout;
*stderr = orig_stderr;
}
exit(0);
}
