static void
jsgf_set_search_path(jsgf_t * jsgf, const char *filename)
{
char *jsgf_path;
#if !defined(_WIN32_WCE)
if ((jsgf_path = getenv("JSGF_PATH")) != NULL) {
char *word, *c;
/* FIXME: This should be a function in libsphinxbase. */
word = jsgf_path = ckd_salloc(jsgf_path);
while ((c = strchr(word, ':'))) {
*c = '\0';
jsgf->searchpath = glist_add_ptr(jsgf->searchpath, word);
word = c + 1;
}
jsgf->searchpath = glist_add_ptr(jsgf->searchpath, word);
jsgf->searchpath = glist_reverse(jsgf->searchpath);
return;
}
#endif
if (!filename) {
jsgf->searchpath =
glist_add_ptr(jsgf->searchpath, ckd_salloc("."));
return;
}
jsgf_path = ckd_salloc(filename);
path2dirname(filename, jsgf_path);
jsgf->searchpath = glist_add_ptr(jsgf->searchpath, jsgf_path);
}
glist_t vithist_backtrace (vithist_t *vh, int32 id)
{
vithist_entry_t *ve;
int32 b, l;
glist_t hyp;
hyp_t *h;
hyp = NULL;
while (id > 0) {
b = VITHIST_ID2BLK(id);
l = VITHIST_ID2BLKOFFSET(id);
ve = vh->entry[b] + l;
h = (hyp_t *) ckd_calloc (1, sizeof(hyp_t));
h->id = ve->wid;
h->sf = ve->sf;
h->ef = ve->ef;
h->ascr = ve->ascr;
h->lscr = ve->lscr;
h->type = ve->type;
h->vhid = id;
hyp = glist_add_ptr (hyp, h);
id = ve->pred;
}
return hyp;
}
void
kws_detections_add(kws_detections_t *detections, const char* keyphrase, int sf, int ef, int prob, int ascr)
{
gnode_t *gn;
kws_detection_t* detection;
for (gn = detections->detect_list; gn; gn = gnode_next(gn)) {
kws_detection_t *det = (kws_detection_t *)gnode_ptr(gn);
if (strcmp(keyphrase, det->keyphrase) == 0 && det->sf < ef && det->ef > sf) {
if (det->prob < prob) {
det->sf = sf;
det->ef = ef;
det->prob = prob;
det->ascr = ascr;
}
return;
}
}
/* Nothing found */
detection = (kws_detection_t *)ckd_calloc(1, sizeof(*detection));
detection->sf = sf;
detection->ef = ef;
detection->keyphrase = keyphrase;
detection->prob = prob;
detection->ascr = ascr;
detections->detect_list = glist_add_ptr(detections->detect_list, detection);
}
int
fsg_model_add_alt(fsg_model_t * fsg, char const *baseword,
char const *altword)
{
int i, basewid, altwid;
int ntrans;
/* FIXME: This will get slow, eventually... */
for (basewid = 0; basewid < fsg->n_word; ++basewid)
if (0 == strcmp(fsg->vocab[basewid], baseword))
break;
if (basewid == fsg->n_word) {
E_ERROR("Base word %s not present in FSG vocabulary!\n", baseword);
return -1;
}
altwid = fsg_model_word_add(fsg, altword);
if (fsg->altwords == NULL)
fsg->altwords = bitvec_alloc(fsg->n_word_alloc);
bitvec_set(fsg->altwords, altwid);
E_DEBUG(2,("Adding alternate word transitions (%s,%s) to FSG\n",
baseword, altword));
/* Look for all transitions involving baseword and duplicate them. */
/* FIXME: This will also get slow, eventually... */
ntrans = 0;
for (i = 0; i < fsg->n_state; ++i) {
hash_iter_t *itor;
if (fsg->trans[i].trans == NULL)
continue;
for (itor = hash_table_iter(fsg->trans[i].trans); itor;
itor = hash_table_iter_next(itor)) {
glist_t trans;
gnode_t *gn;
trans = hash_entry_val(itor->ent);
for (gn = trans; gn; gn = gnode_next(gn)) {
fsg_link_t *fl = gnode_ptr(gn);
if (fl->wid == basewid) {
fsg_link_t *link;
/* Create transition object */
link = listelem_malloc(fsg->link_alloc);
link->from_state = fl->from_state;
link->to_state = fl->to_state;
link->logs2prob = fl->logs2prob; /* FIXME!!!??? */
link->wid = altwid;
trans =
glist_add_ptr(trans, (void *) link);
++ntrans;
}
}
hash_entry_val(itor->ent) = trans;
}
}
E_DEBUG(2,("Added %d alternate word transitions\n", ntrans));
return ntrans;
}
glist_t
srch_FLAT_FWD_gen_hyp(void *srch /**< a pointer of srch_t */
)
{
srch_t *s;
srch_FLAT_FWD_graph_t *fwg;
srch_hyp_t *tmph, *hyp;
glist_t ghyp, rhyp;
s = (srch_t *) srch;
fwg = (srch_FLAT_FWD_graph_t *) s->grh->graph_struct;
if (s->exit_id == -1)
s->exit_id = lat_final_entry(fwg->lathist, kbcore_dict(s->kbc), fwg->n_frm,
s->uttid);
if (NOT_S3LATID(s->exit_id)) {
E_INFO("lattice ID: %d\n", s->exit_id);
E_ERROR("%s: NO RECOGNITION\n", s->uttid);
return NULL;
}
else {
/* BAD_S3WID => Any right context */
lattice_backtrace(fwg->lathist, s->exit_id, BAD_S3WID, &hyp,
s->kbc->lmset->cur_lm, kbcore_dict(s->kbc),
fwg->ctxt, s->kbc->fillpen);
ghyp = NULL;
for (tmph = hyp; tmph; tmph = tmph->next) {
ghyp = glist_add_ptr(ghyp, (void *) tmph);
}
rhyp = glist_reverse(ghyp);
return rhyp;
}
}
/*
* Enter a new LMstate into the current frame LMstates trees; called ONLY IF not already
* present.
*/
static void
vithist_lmstate_enter(vithist_t * vh, int32 vhid, vithist_entry_t * ve)
{
vh_lms2vh_t *lms2vh, *child;
s3lmwid32_t lwid;
lwid = ve->lmstate.lm3g.lwid[0];
if ((lms2vh = vh->lms2vh_root[lwid]) == NULL) {
lms2vh = (vh_lms2vh_t *) ckd_calloc(1, sizeof(vh_lms2vh_t));
vh->lms2vh_root[lwid] = lms2vh;
lms2vh->state = lwid;
lms2vh->children = NULL;
vh->lwidlist = glist_add_int32(vh->lwidlist, (int32) lwid);
}
else {
assert(lms2vh->state == lwid);
}
child = (vh_lms2vh_t *) ckd_calloc(1, sizeof(vh_lms2vh_t));
child->state = ve->lmstate.lm3g.lwid[1];
child->children = NULL;
child->vhid = vhid;
child->ve = ve;
lms2vh->children = glist_add_ptr(lms2vh->children, (void *) child);
}
ngram_model_t *
ps_update_lmset(ps_decoder_t *ps, ngram_model_t *lmset)
{
ngram_search_t *ngs;
ps_search_t *search;
/* Look for N-Gram search. */
search = ps_find_search(ps, "ngram");
if (search == NULL) {
/* Initialize N-Gram search. */
search = ngram_search_init(ps->config, ps->acmod, ps->dict, ps->d2p);
if (search == NULL)
return NULL;
search->pls = ps->phone_loop;
ps->searches = glist_add_ptr(ps->searches, search);
ngs = (ngram_search_t *)search;
}
else {
ngs = (ngram_search_t *)search;
/* Free any previous lmset if this is a new one. */
if (ngs->lmset != NULL && ngs->lmset != lmset)
ngram_model_free(ngs->lmset);
ngs->lmset = lmset;
/* Tell N-Gram search to update its view of the world. */
if (ps_search_reinit(search, ps->dict, ps->d2p) < 0)
return NULL;
}
ps->search = search;
return ngs->lmset;
}
glist_t vithist_sort (glist_t vithist_list)
{
heap_t heap;
gnode_t *gn;
vithist_t *h;
glist_t vithist_new;
int32 ret, score;
vithist_new = NULL;
heap = heap_new();
for (gn = vithist_list; gn; gn = gnode_next(gn)) {
h = (vithist_t *) gnode_ptr(gn);
if (heap_insert (heap, (void *) h, h->scr) < 0) {
E_ERROR("Panic: heap_insert() failed\n");
return NULL;
}
}
/*
* Note: The heap returns nodes with ASCENDING values; and glist_add adds new nodes to the
* HEAD of the list. So we get a glist in the desired descending score order.
*/
while ((ret = heap_pop (heap, (void **)(&h), &score)) > 0)
vithist_new = glist_add_ptr (vithist_new, (void *)h);
if (ret < 0) {
E_ERROR("Panic: heap_pop() failed\n");
return NULL;
}
heap_destroy (heap);
return vithist_new;
}
glist_t vithist_backtrace (vithist_t *hist, int32 *senscale)
{
glist_t hyp;
hyp_t *hypnode;
hyp = NULL;
for (; hist; hist = hist->hist) {
hypnode = (hyp_t *) mymalloc (sizeof(hyp_t));
hypnode->id = hist->id;
hypnode->ef = hist->frm;
if (hist->hist) {
hypnode->sf = hist->hist->frm+1;
hypnode->ascr = hist->scr - hist->hist->scr; /* Still scaled */
} else {
hypnode->sf = 0;
hypnode->ascr = hist->scr;
}
/* Undo senone score scaling */
hypnode->ascr += senone_get_senscale (senscale, hypnode->sf, hypnode->ef);
hypnode->lscr = 0;
hypnode->scr = hypnode->ascr;
hyp = glist_add_ptr (hyp, (void *)hypnode);
}
return hyp;
}
/*
* Add the given transition to the FSG transition matrix. Duplicates (i.e.,
* two transitions between the same states, with the same word label) are
* flagged and only the highest prob retained.
*/
static void
word_fsg_trans_add(word_fsg_t * fsg,
int32 from, int32 to, int32 logp, int32 wid)
{
word_fsglink_t *link;
gnode_t *gn;
/* Check for duplicate link (i.e., link already exists with label=wid) */
for (gn = fsg->trans[from][to]; gn; gn = gnode_next(gn)) {
link = (word_fsglink_t *) gnode_ptr(gn);
if (link->wid == wid) {
#if 0
E_WARN
("Duplicate transition %d -> %d ('%s'); highest prob kept\n",
from, to, dict_wordstr(fsg->dict, wid));
#endif
if (link->logs2prob < logp)
link->logs2prob = logp;
return;
}
}
/* Create transition object */
link = (word_fsglink_t *) ckd_calloc(1, sizeof(word_fsglink_t));
link->from_state = from;
link->to_state = to;
link->logs2prob = logp;
link->wid = wid;
fsg->trans[from][to] =
glist_add_ptr(fsg->trans[from][to], (void *) link);
}
void
fsg_model_trans_add(fsg_model_t * fsg,
int32 from, int32 to, int32 logp, int32 wid)
{
fsg_link_t *link;
glist_t gl;
gnode_t *gn;
if (fsg->trans[from].trans == NULL)
fsg->trans[from].trans = hash_table_new(5, HASH_CASE_YES);
/* Check for duplicate link (i.e., link already exists with label=wid) */
for (gn = gl = fsg_model_trans(fsg, from, to); gn; gn = gnode_next(gn)) {
link = (fsg_link_t *) gnode_ptr(gn);
if (link->wid == wid) {
if (link->logs2prob < logp)
link->logs2prob = logp;
return;
}
}
/* Create transition object */
link = listelem_malloc(fsg->link_alloc);
link->from_state = from;
link->to_state = to;
link->logs2prob = logp;
link->wid = wid;
/* Add it to the list of transitions and update the hash table */
gl = glist_add_ptr(gl, (void *) link);
hash_table_replace_bkey(fsg->trans[from].trans,
(char const *) &link->to_state,
sizeof(link->to_state), gl);
}
jsgf_t *
jsgf_grammar_new(jsgf_t *parent)
{
jsgf_t *grammar;
grammar = ckd_calloc(1, sizeof(*grammar));
/* If this is an imported/subgrammar, then we will share a global
* namespace with the parent grammar. */
if (parent) {
grammar->rules = parent->rules;
grammar->imports = parent->imports;
grammar->searchpath = parent->searchpath;
grammar->parent = parent;
}
else {
char *jsgf_path;
grammar->rules = hash_table_new(64, 0);
grammar->imports = hash_table_new(16, 0);
/* Silvio Moioli: no getenv() in Windows CE */
#if !defined(_WIN32_WCE)
if ((jsgf_path = getenv("JSGF_PATH")) != NULL) {
char *word, *c;
/* FIXME: This should be a function in libsphinxbase. */
/* FIXME: Also nextword() is totally useless... */
word = jsgf_path = ckd_salloc(jsgf_path);
while ((c = strchr(word, ':'))) {
*c = '\0';
grammar->searchpath = glist_add_ptr(grammar->searchpath, word);
word = c + 1;
}
grammar->searchpath = glist_add_ptr(grammar->searchpath, word);
grammar->searchpath = glist_reverse(grammar->searchpath);
}
else {
/* Default to current directory. */
grammar->searchpath = glist_add_ptr(grammar->searchpath, ckd_salloc("."));
}
#endif
}
return grammar;
}
jsgf_rule_t *
jsgf_optional_new(jsgf_t *jsgf, jsgf_rhs_t *exp)
{
jsgf_rhs_t *rhs = ckd_calloc(1, sizeof(*rhs));
jsgf_atom_t *atom = jsgf_atom_new("<NULL>", 1.0);
rhs->alt = exp;
rhs->atoms = glist_add_ptr(NULL, atom);
return jsgf_define_rule(jsgf, NULL, rhs, 0);
}
glist_t
srch_FLAT_FWD_bestpath_impl(void *srch, /**< A void pointer to a search structure */
dag_t * dag)
{
srch_t *s;
srch_FLAT_FWD_graph_t *fwg;
float32 bestpathlw;
float64 lwf;
srch_hyp_t *tmph, *bph;
glist_t ghyp, rhyp;
s = (srch_t *) srch;
fwg = (srch_FLAT_FWD_graph_t *) s->grh->graph_struct;
assert(fwg->lathist);
bestpathlw = cmd_ln_float32_r(kbcore_config(fwg->kbcore), "-bestpathlw");
lwf = bestpathlw ? (bestpathlw / cmd_ln_float32_r(kbcore_config(fwg->kbcore), "-lw")) : 1.0;
flat_fwd_dag_add_fudge_edges(fwg,
dag,
cmd_ln_int32_r(kbcore_config(fwg->kbcore), "-dagfudge"),
cmd_ln_int32_r(kbcore_config(fwg->kbcore), "-min_endfr"),
(void *) fwg->lathist, s->kbc->dict);
/* Bypass filler nodes */
if (!dag->filler_removed) {
/* If Viterbi search terminated in filler word coerce final DAG node to FINISH_WORD */
if (dict_filler_word(s->kbc->dict, dag->end->wid))
dag->end->wid = s->kbc->dict->finishwid;
if (dag_bypass_filler_nodes(dag, lwf, s->kbc->dict, s->kbc->fillpen) < 0)
E_ERROR("maxedge limit (%d) exceeded\n", dag->maxedge);
else
dag->filler_removed = 1;
}
bph =
dag_search(dag, s->uttid, lwf, dag->end,
s->kbc->dict, s->kbc->lmset->cur_lm, s->kbc->fillpen);
if (bph != NULL) {
ghyp = NULL;
for (tmph = bph; tmph; tmph = tmph->next)
ghyp = glist_add_ptr(ghyp, (void *) tmph);
rhyp = glist_reverse(ghyp);
return rhyp;
}
else {
return NULL;
}
}
void
jsgf_add_link(jsgf_t *grammar, jsgf_atom_t *atom, int from, int to)
{
jsgf_link_t *link;
link = ckd_calloc(1, sizeof(*link));
link->from = from;
link->to = to;
link->atom = atom;
grammar->links = glist_add_ptr(grammar->links, link);
}
/*
* Obtain transitive closure of NULL transitions in the given FSG. (Initial
* list of such transitions is given.)
* Return value: Updated list of null transitions.
*/
static glist_t
word_fsg_null_trans_closure(word_fsg_t * fsg, glist_t nulls)
{
gnode_t *gn1, *gn2;
int updated;
word_fsglink_t *tl1, *tl2;
int32 k, n;
E_INFO("Computing transitive closure for null transitions\n");
/*
* Probably not the most efficient closure implementation, in general, but
* probably reasonably efficient for a sparse null transition matrix.
*/
n = 0;
do {
updated = FALSE;
for (gn1 = nulls; gn1; gn1 = gnode_next(gn1)) {
tl1 = (word_fsglink_t *) gnode_ptr(gn1);
assert(tl1->wid < 0);
for (gn2 = nulls; gn2; gn2 = gnode_next(gn2)) {
tl2 = (word_fsglink_t *) gnode_ptr(gn2);
if (tl1->to_state == tl2->from_state) {
k = word_fsg_null_trans_add(fsg,
tl1->from_state,
tl2->to_state,
tl1->logs2prob +
tl2->logs2prob);
if (k >= 0) {
updated = TRUE;
if (k > 0) {
nulls =
glist_add_ptr(nulls,
(void *) fsg->
null_trans[tl1->
from_state][tl2->
to_state]);
n++;
}
}
}
}
}
} while (updated);
E_INFO("%d null transitions added\n", n);
return nulls;
}
static void
renormalize_hmms(phone_loop_search_t *pls, int frame_idx, int32 norm)
{
phone_loop_renorm_t *rn = ckd_calloc(1, sizeof(*rn));
int i;
pls->renorm = glist_add_ptr(pls->renorm, rn);
rn->frame_idx = frame_idx;
rn->norm = norm;
for (i = 0; i < pls->n_phones; ++i) {
hmm_normalize((hmm_t *)&pls->phones[i], norm);
}
}
jsgf_atom_t *
jsgf_kleene_new(jsgf_t *jsgf, jsgf_atom_t *atom, int plus)
{
jsgf_rule_t *rule;
jsgf_atom_t *rule_atom;
jsgf_rhs_t *rhs;
/* Generate an "internal" rule of the form (<NULL> | <name> <g0006>) */
/* Or if plus is true, (<name> | <name> <g0006>) */
rhs = ckd_calloc(1, sizeof(*rhs));
if (plus)
rhs->atoms = glist_add_ptr(NULL, jsgf_atom_new(atom->name, 1.0));
else
rhs->atoms = glist_add_ptr(NULL, jsgf_atom_new("<NULL>", 1.0));
rule = jsgf_define_rule(jsgf, NULL, rhs, 0);
rule_atom = jsgf_atom_new(rule->name, 1.0);
rhs = ckd_calloc(1, sizeof(*rhs));
rhs->atoms = glist_add_ptr(NULL, rule_atom);
rhs->atoms = glist_add_ptr(rhs->atoms, atom);
rule->rhs->alt = rhs;
return jsgf_atom_new(rule->name, 1.0);
}
glist_t vithist_append (glist_t hlist, int32 id, int32 frm, int32 score,
vithist_t *hist, vithist_t *lmhist)
{
vithist_t *h;
h = (vithist_t *) mymalloc (sizeof(vithist_t));
h->id = id;
h->frm = frm;
h->scr = score;
h->hist = hist;
h->lmhist = lmhist ? lmhist : h; /* Self if none provided */
hlist = glist_add_ptr (hlist, (void *)h);
return hlist;
}
static int
expand_rule(jsgf_t * grammar, jsgf_rule_t * rule, int rule_entry,
int rule_exit)
{
jsgf_rule_stack_t *rule_stack_entry;
jsgf_rhs_t *rhs;
/* Push this rule onto the stack */
rule_stack_entry =
(jsgf_rule_stack_t *) ckd_calloc(1, sizeof(jsgf_rule_stack_t));
rule_stack_entry->rule = rule;
rule_stack_entry->entry = rule_entry;
grammar->rulestack = glist_add_ptr(grammar->rulestack,
rule_stack_entry);
for (rhs = rule->rhs; rhs; rhs = rhs->alt) {
int lastnode;
lastnode = expand_rhs(grammar, rule, rhs, rule_entry, rule_exit);
if (lastnode == NO_NODE) {
return NO_NODE;
}
else if (lastnode == RECURSIVE_NODE) {
/* The rhs ended with right-recursion, i.e. a transition to
an earlier state. Nothing needs to happen at this level. */
;
}
else if (rule_exit == NO_NODE) {
/* If this rule doesn't have an exit state yet, use the exit
state of its first right-hand-side.
All other right-hand-sides will use this exit state. */
assert(lastnode >= 0);
rule_exit = lastnode;
}
}
/* If no exit-state was created, use the entry-state. */
if (rule_exit == NO_NODE) {
rule_exit = rule_entry;
}
/* Pop this rule from the rule stack */
ckd_free(gnode_ptr(grammar->rulestack));
grammar->rulestack = gnode_free(grammar->rulestack, NULL);
return rule_exit;
}
static int
expand_rule(jsgf_t *grammar, jsgf_rule_t *rule)
{
jsgf_rhs_t *rhs;
float norm;
/* Push this rule onto the stack */
grammar->rulestack = glist_add_ptr(grammar->rulestack, rule);
/* Normalize weights for all alternatives exiting rule->entry */
norm = 0;
for (rhs = rule->rhs; rhs; rhs = rhs->alt) {
if (rhs->atoms) {
jsgf_atom_t *atom = gnode_ptr(rhs->atoms);
norm += atom->weight;
}
}
rule->entry = grammar->nstate++;
rule->exit = grammar->nstate++;
if (norm == 0) norm = 1;
for (rhs = rule->rhs; rhs; rhs = rhs->alt) {
int lastnode;
if (rhs->atoms) {
jsgf_atom_t *atom = gnode_ptr(rhs->atoms);
atom->weight /= norm;
}
lastnode = expand_rhs(grammar, rule, rhs);
if (lastnode == -1) {
return -1;
}
else {
jsgf_add_link(grammar, NULL, lastnode, rule->exit);
}
}
/* Pop this rule from the rule stack */
grammar->rulestack = gnode_free(grammar->rulestack, NULL);
return rule->exit;
}
fsg_set_t *
ps_update_fsgset(ps_decoder_t *ps)
{
ps_search_t *search;
/* Look for FSG search. */
search = ps_find_search(ps, "fsg");
if (search == NULL) {
/* Initialize FSG search. */
search = fsg_search_init(ps->config,
ps->acmod, ps->dict, ps->d2p);
search->pls = ps->phone_loop;
ps->searches = glist_add_ptr(ps->searches, search);
}
else {
/* Tell FSG search to update its view of the world. */
if (ps_search_reinit(search, ps->dict, ps->d2p) < 0)
return NULL;
}
ps->search = search;
return (fsg_set_t *)search;
}
glist_t
srch_FSG_gen_hyp(void *srch /**< a pointer of srch_t */
)
{
srch_t *s;
fsg_search_t *fsgsrch;
srch_hyp_t *tmph;
glist_t ghyp, rhyp;
s = (srch_t *) srch;
fsgsrch = (fsg_search_t *) s->grh->graph_struct;
fsg_search_history_backtrace(fsgsrch, TRUE);
ghyp = NULL;
for (tmph = fsgsrch->hyp; tmph; tmph = tmph->next) {
ghyp = glist_add_ptr(ghyp, (void *) tmph);
}
rhyp = glist_reverse(ghyp);
return rhyp;
}
fsg_model_t *
fsg_model_read(FILE * fp, logmath_t * lmath, float32 lw)
{
fsg_model_t *fsg;
hash_table_t *vocab;
hash_iter_t *itor;
int32 lastwid;
char **wordptr;
char *lineptr;
char *fsgname;
int32 lineno;
int32 n, i, j;
int n_state, n_trans, n_null_trans;
glist_t nulls;
float32 p;
lineno = 0;
vocab = hash_table_new(32, FALSE);
wordptr = NULL;
lineptr = NULL;
nulls = NULL;
fsgname = NULL;
fsg = NULL;
/* Scan upto FSG_BEGIN header */
for (;;) {
n = nextline_str2words(fp, &lineno, &lineptr, &wordptr);
if (n < 0) {
E_ERROR("%s declaration missing\n", FSG_MODEL_BEGIN_DECL);
goto parse_error;
}
if ((strcmp(wordptr[0], FSG_MODEL_BEGIN_DECL) == 0)) {
if (n > 2) {
E_ERROR("Line[%d]: malformed FSG_BEGIN declaration\n",
lineno);
goto parse_error;
}
break;
}
}
/* Save FSG name, or it will get clobbered below :(.
* If name is missing, try the default.
*/
if (n == 2) {
fsgname = ckd_salloc(wordptr[1]);
}
else {
E_WARN("FSG name is missing\n");
fsgname = ckd_salloc("unknown");
}
/* Read #states */
n = nextline_str2words(fp, &lineno, &lineptr, &wordptr);
if ((n != 2)
|| ((strcmp(wordptr[0], FSG_MODEL_N_DECL) != 0)
&& (strcmp(wordptr[0], FSG_MODEL_NUM_STATES_DECL) != 0))
|| (sscanf(wordptr[1], "%d", &n_state) != 1)
|| (n_state <= 0)) {
E_ERROR
("Line[%d]: #states declaration line missing or malformed\n",
lineno);
goto parse_error;
}
/* Now create the FSG. */
fsg = fsg_model_init(fsgname, lmath, lw, n_state);
ckd_free(fsgname);
fsgname = NULL;
/* Read start state */
n = nextline_str2words(fp, &lineno, &lineptr, &wordptr);
if ((n != 2)
|| ((strcmp(wordptr[0], FSG_MODEL_S_DECL) != 0)
&& (strcmp(wordptr[0], FSG_MODEL_START_STATE_DECL) != 0))
|| (sscanf(wordptr[1], "%d", &(fsg->start_state)) != 1)
|| (fsg->start_state < 0)
|| (fsg->start_state >= fsg->n_state)) {
E_ERROR
("Line[%d]: start state declaration line missing or malformed\n",
lineno);
goto parse_error;
}
/* Read final state */
n = nextline_str2words(fp, &lineno, &lineptr, &wordptr);
if ((n != 2)
|| ((strcmp(wordptr[0], FSG_MODEL_F_DECL) != 0)
&& (strcmp(wordptr[0], FSG_MODEL_FINAL_STATE_DECL) != 0))
|| (sscanf(wordptr[1], "%d", &(fsg->final_state)) != 1)
|| (fsg->final_state < 0)
|| (fsg->final_state >= fsg->n_state)) {
E_ERROR
("Line[%d]: final state declaration line missing or malformed\n",
lineno);
goto parse_error;
}
/* Read transitions */
lastwid = 0;
n_trans = n_null_trans = 0;
for (;;) {
int32 wid, tprob;
n = nextline_str2words(fp, &lineno, &lineptr, &wordptr);
if (n <= 0) {
E_ERROR("Line[%d]: transition or FSG_END statement expected\n",
lineno);
goto parse_error;
}
if ((strcmp(wordptr[0], FSG_MODEL_END_DECL) == 0)) {
break;
}
if ((strcmp(wordptr[0], FSG_MODEL_T_DECL) == 0)
|| (strcmp(wordptr[0], FSG_MODEL_TRANSITION_DECL) == 0)) {
if (((n != 4) && (n != 5))
|| (sscanf(wordptr[1], "%d", &i) != 1)
|| (sscanf(wordptr[2], "%d", &j) != 1)
|| (i < 0) || (i >= fsg->n_state)
|| (j < 0) || (j >= fsg->n_state)) {
E_ERROR
("Line[%d]: transition spec malformed; Expecting: from-state to-state trans-prob [word]\n",
lineno);
goto parse_error;
}
p = atof_c(wordptr[3]);
if ((p <= 0.0) || (p > 1.0)) {
E_ERROR
("Line[%d]: transition spec malformed; Expecting float as transition probability\n",
lineno);
goto parse_error;
}
}
else {
E_ERROR("Line[%d]: transition or FSG_END statement expected\n",
lineno);
goto parse_error;
}
tprob = (int32) (logmath_log(lmath, p) * fsg->lw);
/* Add word to "dictionary". */
if (n > 4) {
if (hash_table_lookup_int32(vocab, wordptr[4], &wid) < 0) {
(void) hash_table_enter_int32(vocab,
ckd_salloc(wordptr[4]),
lastwid);
wid = lastwid;
++lastwid;
}
fsg_model_trans_add(fsg, i, j, tprob, wid);
++n_trans;
}
else {
if (fsg_model_null_trans_add(fsg, i, j, tprob) == 1) {
++n_null_trans;
nulls =
glist_add_ptr(nulls, fsg_model_null_trans(fsg, i, j));
}
}
}
E_INFO("FSG: %d states, %d unique words, %d transitions (%d null)\n",
fsg->n_state, hash_table_inuse(vocab), n_trans, n_null_trans);
/* Now create a string table from the "dictionary" */
fsg->n_word = hash_table_inuse(vocab);
fsg->n_word_alloc = fsg->n_word + 10; /* Pad it a bit. */
fsg->vocab = ckd_calloc(fsg->n_word_alloc, sizeof(*fsg->vocab));
for (itor = hash_table_iter(vocab); itor;
itor = hash_table_iter_next(itor)) {
char const *word = hash_entry_key(itor->ent);
int32 wid = (int32) (long) hash_entry_val(itor->ent);
fsg->vocab[wid] = (char *) word;
}
hash_table_free(vocab);
/* Do transitive closure on null transitions */
nulls = fsg_model_null_trans_closure(fsg, nulls);
glist_free(nulls);
ckd_free(lineptr);
ckd_free(wordptr);
return fsg;
parse_error:
for (itor = hash_table_iter(vocab); itor;
itor = hash_table_iter_next(itor))
ckd_free((char *) hash_entry_key(itor->ent));
glist_free(nulls);
hash_table_free(vocab);
ckd_free(fsgname);
ckd_free(lineptr);
ckd_free(wordptr);
fsg_model_free(fsg);
return NULL;
}
glist_t
fsg_model_null_trans_closure(fsg_model_t * fsg, glist_t nulls)
{
gnode_t *gn1;
int updated;
fsg_link_t *tl1, *tl2;
int32 k, n;
E_INFO("Computing transitive closure for null transitions\n");
if (nulls == NULL) {
fsg_link_t *null;
int i, j;
for (i = 0; i < fsg->n_state; ++i) {
for (j = 0; j < fsg->n_state; ++j) {
if ((null = fsg_model_null_trans(fsg, i, j)))
nulls = glist_add_ptr(nulls, null);
}
}
}
/*
* Probably not the most efficient closure implementation, in general, but
* probably reasonably efficient for a sparse null transition matrix.
*/
n = 0;
do {
updated = FALSE;
for (gn1 = nulls; gn1; gn1 = gnode_next(gn1)) {
hash_iter_t *itor;
tl1 = (fsg_link_t *) gnode_ptr(gn1);
assert(tl1->wid < 0);
if (fsg->trans[tl1->to_state].null_trans == NULL)
continue;
for (itor = hash_table_iter(fsg->trans[tl1->to_state].null_trans);
itor; itor = hash_table_iter_next(itor)) {
tl2 = (fsg_link_t *) hash_entry_val(itor->ent);
k = fsg_model_null_trans_add(fsg,
tl1->from_state,
tl2->to_state,
tl1->logs2prob +
tl2->logs2prob);
if (k >= 0) {
updated = TRUE;
if (k > 0) {
nulls = glist_add_ptr(nulls, (void *)
fsg_model_null_trans
(fsg, tl1->from_state,
tl2->to_state));
n++;
}
}
}
}
} while (updated);
E_INFO("%d null transitions added\n", n);
return nulls;
}
/* Find the confidence score and dump the confidence output into the file */
static void
confidence_utt(char *uttid, FILE * _confmatchsegfp)
{
seg_hyp_line_t s_hypline;
char line[16384];
char dagfile[16384];
const char *fmt;
const char *latdir;
const char *latext;
E_INFO("Processing %s\n", uttid);
if (fgets(line, sizeof(line), _confmatchsegfp) == NULL)
E_FATAL("Fail to read a line in the matchsegfp for uttid %s\n",
uttid);
/* Read the hypseg */
if (read_s3hypseg_line(line, &s_hypline, lmset->cur_lm, dict) ==
HYPSEG_FAILURE)
E_FATAL("Fail to parse matchseg in utt ID %s\n", uttid);
E_INFO("Matchseg file name %s\n", s_hypline.seq);
if (strcmp(uttid, s_hypline.seq))
E_FATAL("Uttids in control file and matchseg file mismatches\n");
/* Read the lattice */
latdir = cmd_ln_str_r(config, "-inlatdir");
latext = cmd_ln_str_r(config, "-latext");
if (latdir)
sprintf(dagfile, "%s/%s.%s", latdir, uttid, latext);
else
sprintf(dagfile, "%s.%s", uttid, latext);
E_INFO("Reading DAG file: %s\n", dagfile);
if (confidence_word_posterior(dagfile,
&s_hypline,
uttid,
lmset->cur_lm,
dict, fpen) == CONFIDENCE_FAILURE) {
E_INFO("Fail to compute word posterior probability \n");
}
#if 0
if (ca_dag_load_lattice
(dagfile, &word_lattice, lmset->cur_lm, dict,
fpen) == CONFIDENCE_FAILURE)
E_FATAL("Unable to load dag %s for uttid %s\n", dagfile, uttid);
/* Compute Alpha-beta */
if (alpha_beta(&word_lattice, lmset->cur_lm, dict) ==
CONFIDENCE_FAILURE)
E_FATAL("Unable to compute alpha beta score for uttid %s\n",
uttid);
/* Compute Posterior WORD probability */
if (pwp(&s_hypline, &word_lattice) == CONFIDENCE_FAILURE)
E_FATAL("Unable to compute pwp for uttid %s\n", uttid);
#endif
/* Compute LM type */
if (compute_lmtype(&s_hypline, lmset->cur_lm, dict) ==
CONFIDENCE_FAILURE)
E_FATAL("Fail to compute lm type\n");
/* combined LM type */
if (compute_combined_lmtype(&s_hypline) == CONFIDENCE_FAILURE)
E_FATAL("Fail to compute lm type\n");
/* Dump pwp line */
fmt = cmd_ln_str_r(config, "-confoutputfmt");
if (!strcmp(fmt, "scores")) {
dump_line(stdout, &s_hypline, dict);
dump_line(outconfmatchsegfp, &s_hypline, dict);
}
else {
glist_t hyp;
srch_hyp_t *s;
conf_srch_hyp_t *h;
hyp = NULL;
for (h = (conf_srch_hyp_t *) s_hypline.wordlist; h; h = h->next) {
s = &(h->sh);
hyp = glist_add_ptr(hyp, (void *) s);
}
matchseg_write(stdout, hyp, uttid, NULL,
lmset->cur_lm, dict, 0, NULL, 0);
matchseg_write(outconfmatchsegfp, hyp, uttid, NULL,
lmset->cur_lm, dict, 0, NULL, 0);
}
#if 0
/* Delete lattice, delete hypsegline */
if (ca_dag_free_lattice(&word_lattice) == CONFIDENCE_FAILURE) {
E_WARN("Fail to free lattice.\n");
return CONFIDENCE_FAILURE;
}
#endif
if (free_seg_hyp_line(&s_hypline) != HYPSEG_SUCCESS)
E_FATAL("Fail to free the segment hypothesis line structure. \n");
}
glist_t
fsg_model_null_trans_closure(fsg_model_t * fsg, glist_t nulls)
{
gnode_t *gn1;
int updated;
fsg_link_t *tl1, *tl2;
int32 k, n;
E_INFO("Computing transitive closure for null transitions\n");
/* If our caller didn't give us a list of null-transitions,
make such a list. Just loop through all the FSG states,
and all the null-transitions in that state (which are kept in
their own hash table). */
if (nulls == NULL) {
int i;
for (i = 0; i < fsg->n_state; ++i) {
hash_iter_t *itor;
hash_table_t *null_trans = fsg->trans[i].null_trans;
if (null_trans == NULL)
continue;
for (itor = hash_table_iter(null_trans);
itor != NULL;
itor = hash_table_iter_next(itor)) {
nulls = glist_add_ptr(nulls, hash_entry_val(itor->ent));
}
}
}
/*
* Probably not the most efficient closure implementation, in general, but
* probably reasonably efficient for a sparse null transition matrix.
*/
n = 0;
do {
updated = FALSE;
for (gn1 = nulls; gn1; gn1 = gnode_next(gn1)) {
hash_iter_t *itor;
tl1 = (fsg_link_t *) gnode_ptr(gn1);
assert(tl1->wid < 0);
if (fsg->trans[tl1->to_state].null_trans == NULL)
continue;
for (itor = hash_table_iter(fsg->trans[tl1->to_state].null_trans);
itor; itor = hash_table_iter_next(itor)) {
tl2 = (fsg_link_t *) hash_entry_val(itor->ent);
k = fsg_model_null_trans_add(fsg,
tl1->from_state,
tl2->to_state,
tl1->logs2prob +
tl2->logs2prob);
if (k >= 0) {
updated = TRUE;
if (k > 0) {
nulls = glist_add_ptr(nulls, (void *)
fsg_model_null_trans
(fsg, tl1->from_state,
tl2->to_state));
n++;
}
}
}
}
} while (updated);
E_INFO("%d null transitions added\n", n);
return nulls;
}
static void
huff_code_canonicalize(huff_code_t *hc, huff_node_t *root)
{
glist_t agenda;
uint32 *nextcode;
int i, ncw;
hc->firstcode =(uint32*) ckd_calloc(hc->maxbits+1, sizeof(*hc->firstcode));
hc->syms = (huff_codeword_t**)ckd_calloc(hc->maxbits+1, sizeof(*hc->syms));
hc->numl =(uint32*) ckd_calloc(hc->maxbits+1, sizeof(*nextcode));
nextcode =(uint32*) ckd_calloc(hc->maxbits+1, sizeof(*nextcode));
/* Traverse the tree, annotating it with the actual bit
* lengths, and histogramming them in numl. */
root->nbits = 0;
ncw = 0;
agenda = glist_add_ptr(0, root);
while (agenda) {
huff_node_t *node = (huff_node_t*)gnode_ptr(agenda);
agenda = gnode_free(agenda, 0);
if (node->l) {
node->l->nbits = node->nbits + 1;
agenda = glist_add_ptr(agenda, node->l);
node->r.r->nbits = node->nbits + 1;
agenda = glist_add_ptr(agenda, node->r.r);
}
else {
hc->numl[node->nbits]++;
ncw++;
}
}
/* Create starting codes and symbol tables for each bit length. */
hc->syms[hc->maxbits] = (huff_codeword_t*)ckd_calloc(hc->numl[hc->maxbits], sizeof(**hc->syms));
for (i = hc->maxbits - 1; i > 0; --i) {
hc->firstcode[i] = (hc->firstcode[i+1] + hc->numl[i+1]) / 2;
hc->syms[i] = (huff_codeword_t*)ckd_calloc(hc->numl[i], sizeof(**hc->syms));
}
memcpy(nextcode, hc->firstcode, (hc->maxbits + 1) * sizeof(*nextcode));
/* Traverse the tree again to produce the codebook itself. */
hc->codewords = hash_table_new(ncw, HASH_CASE_YES);
agenda = glist_add_ptr(0, root);
while (agenda) {
huff_node_t *node = (huff_node_t*)gnode_ptr(agenda);
agenda = gnode_free(agenda, 0);
if (node->l) {
agenda = glist_add_ptr(agenda, node->l);
agenda = glist_add_ptr(agenda, node->r.r);
}
else {
/* Initialize codebook entry, which also retains symbol pointer. */
huff_codeword_t *cw;
uint32 codeword = nextcode[node->nbits] & ((1 << node->nbits) - 1);
cw = hc->syms[node->nbits] + (codeword - hc->firstcode[node->nbits]);
cw->nbits = node->nbits;
cw->r.sval = node->r.sval; /* Will copy ints too... */
cw->codeword = codeword;
if (hc->type == HUFF_CODE_INT) {
hash_table_enter_bkey(hc->codewords,
(char const *)&cw->r.ival,
sizeof(cw->r.ival),
(void *)cw);
}
else {
hash_table_enter(hc->codewords, cw->r.sval, (void *)cw);
}
++nextcode[node->nbits];
}
}
ckd_free(nextcode);
}
int32
read_classdef_file(hash_table_t * classes, const char *file_name)
{
FILE *fp;
int32 is_pipe;
int inclass; /**< Are we currently reading a list of class words? */
int32 rv = -1;
gnode_t *gn;
glist_t classwords = NULL;
glist_t classprobs = NULL;
char *classname = NULL;
if ((fp = fopen_comp(file_name, "r", &is_pipe)) == NULL) {
E_ERROR("File %s not found\n", file_name);
return -1;
}
inclass = FALSE;
while (!feof(fp)) {
char line[512];
char *wptr[2];
int n_words;
if (fgets(line, sizeof(line), fp) == NULL)
break;
n_words = str2words(line, wptr, 2);
if (n_words <= 0)
continue;
if (inclass) {
/* Look for an end of class marker. */
if (n_words == 2 && 0 == strcmp(wptr[0], "END")) {
classdef_t *classdef;
gnode_t *word, *weight;
int32 i;
if (classname == NULL || 0 != strcmp(wptr[1], classname))
goto error_out;
inclass = FALSE;
/* Construct a class from the list of words collected. */
classdef = ckd_calloc(1, sizeof(*classdef));
classwords = glist_reverse(classwords);
classprobs = glist_reverse(classprobs);
classdef->n_words = glist_count(classwords);
classdef->words = ckd_calloc(classdef->n_words,
sizeof(*classdef->words));
classdef->weights = ckd_calloc(classdef->n_words,
sizeof(*classdef->weights));
word = classwords;
weight = classprobs;
for (i = 0; i < classdef->n_words; ++i) {
classdef->words[i] = gnode_ptr(word);
classdef->weights[i] = gnode_float32(weight);
word = gnode_next(word);
weight = gnode_next(weight);
}
/* Add this class to the hash table. */
if (hash_table_enter(classes, classname, classdef) !=
classdef) {
classdef_free(classdef);
goto error_out;
}
/* Reset everything. */
glist_free(classwords);
glist_free(classprobs);
classwords = NULL;
classprobs = NULL;
classname = NULL;
}
else {
float32 fprob;
if (n_words == 2)
fprob = atof_c(wptr[1]);
else
fprob = 1.0f;
/* Add it to the list of words for this class. */
classwords =
glist_add_ptr(classwords, ckd_salloc(wptr[0]));
classprobs = glist_add_float32(classprobs, fprob);
}
}
else {
/* Start a new LM class if the LMCLASS marker is seen */
if (n_words == 2 && 0 == strcmp(wptr[0], "LMCLASS")) {
if (inclass)
goto error_out;
inclass = TRUE;
classname = ckd_salloc(wptr[1]);
}
/* Otherwise, just ignore whatever junk we got */
}
}
rv = 0; /* Success. */
error_out:
/* Free all the stuff we might have allocated. */
fclose_comp(fp, is_pipe);
for (gn = classwords; gn; gn = gnode_next(gn))
ckd_free(gnode_ptr(gn));
glist_free(classwords);
glist_free(classprobs);
ckd_free(classname);
return rv;
}
void fsg_history_entry_add (fsg_history_t *h,
word_fsglink_t *link,
int32 frame, int32 score, int32 pred,
int32 lc, fsg_pnode_ctxt_t rc)
{
fsg_hist_entry_t *entry, *new_entry;
int32 s;
gnode_t *gn, *prev_gn;
/* Skip the optimization for the initial dummy entries; always enter them */
if (frame < 0) {
new_entry = (fsg_hist_entry_t *) ckd_calloc(1, sizeof(fsg_hist_entry_t));
new_entry->fsglink = link;
new_entry->frame = frame;
new_entry->score = score;
new_entry->pred = pred;
new_entry->lc = lc;
new_entry->rc = rc;
blkarray_list_append(h->entries, (void *)new_entry);
return;
}
s = word_fsglink_to_state(link);
/* Locate where this entry should be inserted in frame_entries[s][lc] */
prev_gn = NULL;
for (gn = h->frame_entries[s][lc]; gn; gn = gnode_next(gn)) {
entry = (fsg_hist_entry_t *) gnode_ptr(gn);
if (entry->score < score)
break; /* Found where to insert new entry */
/* Existing entry score not worse than new score */
if (fsg_pnode_ctxt_sub(&rc, &(entry->rc)) == 0)
return; /* rc set reduced to 0; new entry can be ignored */
prev_gn = gn;
}
/* Create new entry after prev_gn (if prev_gn is NULL, at head) */
new_entry = (fsg_hist_entry_t *) ckd_calloc(1, sizeof(fsg_hist_entry_t));
new_entry->fsglink = link;
new_entry->frame = frame;
new_entry->score = score;
new_entry->pred = pred;
new_entry->lc = lc;
new_entry->rc = rc; /* Note: rc set must be non-empty at this point */
if (! prev_gn) {
h->frame_entries[s][lc] = glist_add_ptr (h->frame_entries[s][lc],
(void *) new_entry);
prev_gn = h->frame_entries[s][lc];
} else
prev_gn = glist_insert_ptr (prev_gn, (void *) new_entry);
/*
* Update the rc set of all the remaining entries in the list. At this
* point, gn is the entry, if any, immediately following new entry.
*/
while (gn) {
entry = (fsg_hist_entry_t *) gnode_ptr(gn);
if (fsg_pnode_ctxt_sub(&(entry->rc), &rc) == 0) {
/* rc set of entry reduced to 0; can prune this entry */
ckd_free ((void *)entry);
gn = gnode_free (gn, prev_gn);
} else {
prev_gn = gn;
gn = gnode_next(gn);
}
}
}