int
dict_free(dict_t * d)
{
int i;
dictword_t *word;
if (d == NULL)
return 0;
if (--d->refcnt > 0)
return d->refcnt;
/* First Step, free all memory allocated for each word */
for (i = 0; i < d->n_word; i++) {
word = (dictword_t *) & (d->word[i]);
if (word->word)
ckd_free((void *) word->word);
if (word->ciphone)
ckd_free((void *) word->ciphone);
}
if (d->word)
ckd_free((void *) d->word);
if (d->ht)
hash_table_free(d->ht);
if (d->mdef)
bin_mdef_free(d->mdef);
ckd_free((void *) d);
return 0;
}
int
dict2pid_free(dict2pid_t * d2p)
{
if (d2p == NULL)
return 0;
if (--d2p->refcount > 0)
return d2p->refcount;
if (d2p->ldiph_lc)
ckd_free_3d((void ***) d2p->ldiph_lc);
if (d2p->lrdiph_rc)
ckd_free_3d((void ***) d2p->lrdiph_rc);
if (d2p->rssid)
free_compress_map(d2p->rssid, bin_mdef_n_ciphone(d2p->mdef));
if (d2p->lrssid)
free_compress_map(d2p->lrssid, bin_mdef_n_ciphone(d2p->mdef));
bin_mdef_free(d2p->mdef);
dict_free(d2p->dict);
ckd_free(d2p);
return 0;
}
int
acmod_free(acmod_t *acmod)
{
if (acmod == NULL)
return 0;
if (--acmod->refcount > 0)
return acmod->refcount;
ckd_free(acmod->senone_scores);
ckd_free(acmod->senone_active_vec);
ckd_free(acmod->senone_active);
if (acmod->mdef)
bin_mdef_free(acmod->mdef);
if (acmod->tmat)
tmat_free(acmod->tmat);
if (acmod->mgau)
ps_mgau_free(acmod->mgau);
featbuf_free(acmod->fb);
feat_array_free(acmod->feat_buf);
logmath_free(acmod->lmath);
cmd_ln_free_r(acmod->config);
ckd_free(acmod);
return 0;
}
int
main(int argc, char *argv[])
{
bin_mdef_t *mdef;
dict_t *dict;
cmd_ln_t *config;
int i;
char buf[100];
TEST_ASSERT(config = cmd_ln_init(NULL, NULL, FALSE,
"-dict", MODELDIR "/en-us/cmudict-en-us.dict",
"-fdict", MODELDIR "/en-us/en-us/noisedict",
NULL));
/* Test dictionary in standard fashion. */
TEST_ASSERT(mdef = bin_mdef_read(NULL, MODELDIR "/en-us/en-us/mdef"));
TEST_ASSERT(dict = dict_init(config, mdef, NULL));
printf("Word ID (CARNEGIE) = %d\n",
dict_wordid(dict, "CARNEGIE"));
printf("Word ID (ASDFASFASSD) = %d\n",
dict_wordid(dict, "ASDFASFASSD"));
TEST_EQUAL(0, dict_write(dict, "_cmu07a.dic", NULL));
TEST_EQUAL(0, system("diff -uw " MODELDIR "/en-us/cmudict-en-us.dict _cmu07a.dic"));
dict_free(dict);
bin_mdef_free(mdef);
/* Now test an empty dictionary. */
TEST_ASSERT(dict = dict_init(NULL, NULL, NULL));
printf("Word ID(<s>) = %d\n", dict_wordid(dict, "<s>"));
TEST_ASSERT(BAD_S3WID != dict_add_word(dict, "FOOBIE", NULL, 0));
TEST_ASSERT(BAD_S3WID != dict_add_word(dict, "BLETCH", NULL, 0));
printf("Word ID(FOOBIE) = %d\n", dict_wordid(dict, "FOOBIE"));
printf("Word ID(BLETCH) = %d\n", dict_wordid(dict, "BLETCH"));
TEST_ASSERT(dict_real_word(dict, dict_wordid(dict, "FOOBIE")));
TEST_ASSERT(dict_real_word(dict, dict_wordid(dict, "BLETCH")));
TEST_ASSERT(!dict_real_word(dict, dict_wordid(dict, "</s>")));
dict_free(dict);
/* Test to add 500k words. */
TEST_ASSERT(dict = dict_init(NULL, NULL, NULL));
for (i = 0; i < 500000; i++) {
sprintf(buf, "word_%d", i);
TEST_ASSERT(BAD_S3WID != dict_add_word(dict, buf, NULL, 0));
}
dict_free(dict);
cmd_ln_free_r(config);
return 0;
}
int
main(int argc, char *argv[])
{
ngram_trie_t *t;
dict_t *dict;
bin_mdef_t *mdef;
logmath_t *lmath;
cmd_ln_t *config;
FILE *arpafh;
config = cmd_ln_init(NULL, ps_args(), TRUE,
"-hmm", TESTDATADIR "/hub4wsj_sc_8k",
"-dict", TESTDATADIR "/bn10000.homos.dic",
NULL);
ps_init_defaults(config);
lmath = logmath_init(cmd_ln_float32_r(config, "-logbase"),
0, FALSE);
mdef = bin_mdef_read(config, cmd_ln_str_r(config, "-mdef"));
dict = dict_init(config, mdef);
t = ngram_trie_init(dict, lmath);
arpafh = fopen(TESTDATADIR "/bn10000.3g.arpa", "r");
ngram_trie_read_arpa(t, arpafh);
fclose(arpafh);
/* Test 1, 2, 3-gram probs without backoff. */
test_lookups(t, lmath);
arpafh = fopen("tmp.bn10000.3g.arpa", "w");
ngram_trie_write_arpa(t, arpafh);
fclose(arpafh);
ngram_trie_free(t);
t = ngram_trie_init(dict, lmath);
arpafh = fopen("tmp.bn10000.3g.arpa", "r");
ngram_trie_read_arpa(t, arpafh);
fclose(arpafh);
/* Test 1, 2, 3-gram probs without backoff. */
test_lookups(t, lmath);
/* Test adding nodes. */
test_add_nodes(t, lmath);
ngram_trie_free(t);
dict_free(dict);
logmath_free(lmath);
bin_mdef_free(mdef);
cmd_ln_free_r(config);
return 0;
}
void
acmod_free(acmod_t *acmod)
{
if (acmod == NULL)
return;
feat_free(acmod->fcb);
fe_free(acmod->fe);
cmd_ln_free_r(acmod->config);
if (acmod->mfc_buf)
ckd_free_2d((void **)acmod->mfc_buf);
if (acmod->feat_buf)
feat_array_free(acmod->feat_buf);
if (acmod->mfcfh)
fclose(acmod->mfcfh);
if (acmod->rawfh)
fclose(acmod->rawfh);
if (acmod->senfh)
fclose(acmod->senfh);
ckd_free(acmod->framepos);
ckd_free(acmod->senone_scores);
ckd_free(acmod->senone_active_vec);
ckd_free(acmod->senone_active);
ckd_free(acmod->rawdata);
if (acmod->mdef)
bin_mdef_free(acmod->mdef);
if (acmod->tmat)
tmat_free(acmod->tmat);
if (acmod->mgau)
ps_mgau_free(acmod->mgau);
if (acmod->mllr)
ps_mllr_free(acmod->mllr);
ckd_free(acmod);
}
int
main(int argc, char *argv[])
{
bin_mdef_t *mdef;
dict_t *dict;
dict2pid_t *d2p;
cmd_ln_t *config;
TEST_ASSERT(config = cmd_ln_init(NULL, NULL, FALSE,
"-dict", MODELDIR "/lm/en_US/cmu07a.dic",
"-fdict", MODELDIR "/hmm/en_US/hub4wsj_sc_8k/noisedict",
NULL));
TEST_ASSERT(mdef = bin_mdef_read(NULL, MODELDIR "/hmm/en_US/hub4wsj_sc_8k/mdef"));
TEST_ASSERT(dict = dict_init(config, mdef));
TEST_ASSERT(d2p = dict2pid_build(mdef, dict));
dict_free(dict);
dict2pid_free(d2p);
bin_mdef_free(mdef);
cmd_ln_free_r(config);
return 0;
}
int
main(int argc, char *argv[])
{
dict_t *dict;
dict2pid_t *d2p;
bin_mdef_t *mdef;
glextree_t *tree;
hmm_context_t *ctx;
logmath_t *lmath;
tmat_t *tmat;
int i;
TEST_ASSERT(mdef = bin_mdef_read(NULL, MODELDIR "/hmm/en_US/hub4wsj_sc_8k/mdef"));
TEST_ASSERT(dict = dict_init(cmd_ln_init(NULL, NULL, FALSE,
"-dict", DATADIR "/turtle.dic",
"-dictcase", "no", NULL),
mdef));
TEST_ASSERT(d2p = dict2pid_build(mdef, dict));
lmath = logmath_init(1.0001, 0, TRUE);
tmat = tmat_init(MODELDIR "/hmm/en_US/hub4wsj_sc_8k/transition_matrices",
lmath, 1e-5, TRUE);
ctx = hmm_context_init(bin_mdef_n_emit_state(mdef), tmat->tp, NULL, mdef->sseq);
TEST_ASSERT(tree = glextree_build(ctx, dict, d2p, NULL, NULL));
/* Check that a path exists for all dictionary words. */
for (i = 0; i < dict_size(dict); ++i)
TEST_ASSERT(glextree_has_word(tree, i));
dict_free(dict);
dict2pid_free(d2p);
bin_mdef_free(mdef);
tmat_free(tmat);
hmm_context_free(ctx);
glextree_free(tree);
return 0;
}
bin_mdef_t *
bin_mdef_read_text(cmd_ln_t *config, const char *filename)
{
bin_mdef_t *bmdef;
mdef_t *mdef;
int i, nodes, ci_idx, lc_idx, rc_idx;
int nchars;
if ((mdef = mdef_init((char *) filename, TRUE)) == NULL)
return NULL;
/* Enforce some limits. */
if (mdef->n_sen > BAD_SENID) {
E_ERROR("Number of senones exceeds limit: %d > %d\n",
mdef->n_sen, BAD_SENID);
mdef_free(mdef);
return NULL;
}
if (mdef->n_sseq > BAD_SSID) {
E_ERROR("Number of senone sequences exceeds limit: %d > %d\n",
mdef->n_sseq, BAD_SSID);
mdef_free(mdef);
return NULL;
}
/* We use uint8 for ciphones */
if (mdef->n_ciphone > 255) {
E_ERROR("Number of phones exceeds limit: %d > %d\n",
mdef->n_ciphone, 255);
mdef_free(mdef);
return NULL;
}
bmdef = ckd_calloc(1, sizeof(*bmdef));
bmdef->refcnt = 1;
/* Easy stuff. The mdef.c code has done the heavy lifting for us. */
bmdef->n_ciphone = mdef->n_ciphone;
bmdef->n_phone = mdef->n_phone;
bmdef->n_emit_state = mdef->n_emit_state;
bmdef->n_ci_sen = mdef->n_ci_sen;
bmdef->n_sen = mdef->n_sen;
bmdef->n_tmat = mdef->n_tmat;
bmdef->n_sseq = mdef->n_sseq;
bmdef->sseq = mdef->sseq;
bmdef->cd2cisen = mdef->cd2cisen;
bmdef->sen2cimap = mdef->sen2cimap;
bmdef->n_ctx = 3; /* Triphones only. */
bmdef->sil = mdef->sil;
mdef->sseq = NULL; /* We are taking over this one. */
mdef->cd2cisen = NULL; /* And this one. */
mdef->sen2cimap = NULL; /* And this one. */
/* Get the phone names. If they are not sorted
* ASCII-betically then we are in a world of hurt and
* therefore will simply refuse to continue. */
bmdef->ciname = ckd_calloc(bmdef->n_ciphone, sizeof(*bmdef->ciname));
nchars = 0;
for (i = 0; i < bmdef->n_ciphone; ++i)
nchars += strlen(mdef->ciphone[i].name) + 1;
bmdef->ciname[0] = ckd_calloc(nchars, 1);
strcpy(bmdef->ciname[0], mdef->ciphone[0].name);
for (i = 1; i < bmdef->n_ciphone; ++i) {
bmdef->ciname[i] =
bmdef->ciname[i - 1] + strlen(bmdef->ciname[i - 1]) + 1;
strcpy(bmdef->ciname[i], mdef->ciphone[i].name);
if (i > 0 && strcmp(bmdef->ciname[i - 1], bmdef->ciname[i]) > 0) {
/* FIXME: there should be a solution to this, actually. */
E_ERROR("Phone names are not in sorted order, sorry.");
bin_mdef_free(bmdef);
return NULL;
}
}
/* Copy over phone information. */
bmdef->phone = ckd_calloc(bmdef->n_phone, sizeof(*bmdef->phone));
for (i = 0; i < mdef->n_phone; ++i) {
bmdef->phone[i].ssid = mdef->phone[i].ssid;
bmdef->phone[i].tmat = mdef->phone[i].tmat;
if (i < bmdef->n_ciphone) {
bmdef->phone[i].info.ci.filler = mdef->ciphone[i].filler;
}
else {
bmdef->phone[i].info.cd.wpos = mdef->phone[i].wpos;
bmdef->phone[i].info.cd.ctx[0] = mdef->phone[i].ci;
bmdef->phone[i].info.cd.ctx[1] = mdef->phone[i].lc;
bmdef->phone[i].info.cd.ctx[2] = mdef->phone[i].rc;
}
}
/* Walk the wpos_ci_lclist once to find the total number of
* nodes and the starting locations for each level. */
nodes = lc_idx = ci_idx = rc_idx = 0;
for (i = 0; i < N_WORD_POSN; ++i) {
int j;
for (j = 0; j < mdef->n_ciphone; ++j) {
ph_lc_t *lc;
for (lc = mdef->wpos_ci_lclist[i][j]; lc; lc = lc->next) {
ph_rc_t *rc;
for (rc = lc->rclist; rc; rc = rc->next) {
++nodes; /* RC node */
}
++nodes; /* LC node */
++rc_idx; /* Start of RC nodes (after LC nodes) */
}
++nodes; /* CI node */
++lc_idx; /* Start of LC nodes (after CI nodes) */
++rc_idx; /* Start of RC nodes (after CI and LC nodes) */
}
++nodes; /* wpos node */
++ci_idx; /* Start of CI nodes (after wpos nodes) */
++lc_idx; /* Start of LC nodes (after CI nodes) */
++rc_idx; /* STart of RC nodes (after wpos, CI, and LC nodes) */
}
E_INFO("Allocating %d * %d bytes (%d KiB) for CD tree\n",
nodes, sizeof(*bmdef->cd_tree),
nodes * sizeof(*bmdef->cd_tree) / 1024);
bmdef->n_cd_tree = nodes;
bmdef->cd_tree = ckd_calloc(nodes, sizeof(*bmdef->cd_tree));
for (i = 0; i < N_WORD_POSN; ++i) {
int j;
bmdef->cd_tree[i].ctx = i;
bmdef->cd_tree[i].n_down = mdef->n_ciphone;
bmdef->cd_tree[i].c.down = ci_idx;
#if 0
E_INFO("%d => %c (%d@%d)\n",
i, (WPOS_NAME)[i],
bmdef->cd_tree[i].n_down, bmdef->cd_tree[i].c.down);
#endif
/* Now we can build the rest of the tree. */
for (j = 0; j < mdef->n_ciphone; ++j) {
ph_lc_t *lc;
bmdef->cd_tree[ci_idx].ctx = j;
bmdef->cd_tree[ci_idx].c.down = lc_idx;
for (lc = mdef->wpos_ci_lclist[i][j]; lc; lc = lc->next) {
ph_rc_t *rc;
bmdef->cd_tree[lc_idx].ctx = lc->lc;
bmdef->cd_tree[lc_idx].c.down = rc_idx;
for (rc = lc->rclist; rc; rc = rc->next) {
bmdef->cd_tree[rc_idx].ctx = rc->rc;
bmdef->cd_tree[rc_idx].n_down = 0;
bmdef->cd_tree[rc_idx].c.pid = rc->pid;
#if 0
E_INFO("%d => %s %s %s %c (%d@%d)\n",
rc_idx,
bmdef->ciname[j],
bmdef->ciname[lc->lc],
bmdef->ciname[rc->rc],
(WPOS_NAME)[i],
bmdef->cd_tree[rc_idx].n_down,
bmdef->cd_tree[rc_idx].c.down);
#endif
++bmdef->cd_tree[lc_idx].n_down;
++rc_idx;
}
/* If there are no triphones here,
* this is considered a leafnode, so
* set the pid to -1. */
if (bmdef->cd_tree[lc_idx].n_down == 0)
bmdef->cd_tree[lc_idx].c.pid = -1;
#if 0
E_INFO("%d => %s %s %c (%d@%d)\n",
lc_idx,
bmdef->ciname[j],
bmdef->ciname[lc->lc],
(WPOS_NAME)[i],
bmdef->cd_tree[lc_idx].n_down,
bmdef->cd_tree[lc_idx].c.down);
#endif
++bmdef->cd_tree[ci_idx].n_down;
++lc_idx;
}
/* As above, so below. */
if (bmdef->cd_tree[ci_idx].n_down == 0)
bmdef->cd_tree[ci_idx].c.pid = -1;
#if 0
E_INFO("%d => %d=%s (%d@%d)\n",
ci_idx, j, bmdef->ciname[j],
bmdef->cd_tree[ci_idx].n_down,
bmdef->cd_tree[ci_idx].c.down);
#endif
++ci_idx;
}
}
mdef_free(mdef);
bmdef->alloc_mode = BIN_MDEF_FROM_TEXT;
return bmdef;
}
