static void jsgf_set_search_path(jsgf_t * jsgf, const char *filename) { char *jsgf_path; #if !defined(_WIN32_WCE) if ((jsgf_path = getenv("JSGF_PATH")) != NULL) { char *word, *c; /* FIXME: This should be a function in libsphinxbase. */ word = jsgf_path = ckd_salloc(jsgf_path); while ((c = strchr(word, ':'))) { *c = '\0'; jsgf->searchpath = glist_add_ptr(jsgf->searchpath, word); word = c + 1; } jsgf->searchpath = glist_add_ptr(jsgf->searchpath, word); jsgf->searchpath = glist_reverse(jsgf->searchpath); return; } #endif if (!filename) { jsgf->searchpath = glist_add_ptr(jsgf->searchpath, ckd_salloc(".")); return; } jsgf_path = ckd_salloc(filename); path2dirname(filename, jsgf_path); jsgf->searchpath = glist_add_ptr(jsgf->searchpath, jsgf_path); }
glist_t vithist_backtrace (vithist_t *vh, int32 id) { vithist_entry_t *ve; int32 b, l; glist_t hyp; hyp_t *h; hyp = NULL; while (id > 0) { b = VITHIST_ID2BLK(id); l = VITHIST_ID2BLKOFFSET(id); ve = vh->entry[b] + l; h = (hyp_t *) ckd_calloc (1, sizeof(hyp_t)); h->id = ve->wid; h->sf = ve->sf; h->ef = ve->ef; h->ascr = ve->ascr; h->lscr = ve->lscr; h->type = ve->type; h->vhid = id; hyp = glist_add_ptr (hyp, h); id = ve->pred; } return hyp; }
void kws_detections_add(kws_detections_t *detections, const char* keyphrase, int sf, int ef, int prob, int ascr) { gnode_t *gn; kws_detection_t* detection; for (gn = detections->detect_list; gn; gn = gnode_next(gn)) { kws_detection_t *det = (kws_detection_t *)gnode_ptr(gn); if (strcmp(keyphrase, det->keyphrase) == 0 && det->sf < ef && det->ef > sf) { if (det->prob < prob) { det->sf = sf; det->ef = ef; det->prob = prob; det->ascr = ascr; } return; } } /* Nothing found */ detection = (kws_detection_t *)ckd_calloc(1, sizeof(*detection)); detection->sf = sf; detection->ef = ef; detection->keyphrase = keyphrase; detection->prob = prob; detection->ascr = ascr; detections->detect_list = glist_add_ptr(detections->detect_list, detection); }
int fsg_model_add_alt(fsg_model_t * fsg, char const *baseword, char const *altword) { int i, basewid, altwid; int ntrans; /* FIXME: This will get slow, eventually... */ for (basewid = 0; basewid < fsg->n_word; ++basewid) if (0 == strcmp(fsg->vocab[basewid], baseword)) break; if (basewid == fsg->n_word) { E_ERROR("Base word %s not present in FSG vocabulary!\n", baseword); return -1; } altwid = fsg_model_word_add(fsg, altword); if (fsg->altwords == NULL) fsg->altwords = bitvec_alloc(fsg->n_word_alloc); bitvec_set(fsg->altwords, altwid); E_DEBUG(2,("Adding alternate word transitions (%s,%s) to FSG\n", baseword, altword)); /* Look for all transitions involving baseword and duplicate them. */ /* FIXME: This will also get slow, eventually... */ ntrans = 0; for (i = 0; i < fsg->n_state; ++i) { hash_iter_t *itor; if (fsg->trans[i].trans == NULL) continue; for (itor = hash_table_iter(fsg->trans[i].trans); itor; itor = hash_table_iter_next(itor)) { glist_t trans; gnode_t *gn; trans = hash_entry_val(itor->ent); for (gn = trans; gn; gn = gnode_next(gn)) { fsg_link_t *fl = gnode_ptr(gn); if (fl->wid == basewid) { fsg_link_t *link; /* Create transition object */ link = listelem_malloc(fsg->link_alloc); link->from_state = fl->from_state; link->to_state = fl->to_state; link->logs2prob = fl->logs2prob; /* FIXME!!!??? */ link->wid = altwid; trans = glist_add_ptr(trans, (void *) link); ++ntrans; } } hash_entry_val(itor->ent) = trans; } } E_DEBUG(2,("Added %d alternate word transitions\n", ntrans)); return ntrans; }
glist_t srch_FLAT_FWD_gen_hyp(void *srch /**< a pointer of srch_t */ ) { srch_t *s; srch_FLAT_FWD_graph_t *fwg; srch_hyp_t *tmph, *hyp; glist_t ghyp, rhyp; s = (srch_t *) srch; fwg = (srch_FLAT_FWD_graph_t *) s->grh->graph_struct; if (s->exit_id == -1) s->exit_id = lat_final_entry(fwg->lathist, kbcore_dict(s->kbc), fwg->n_frm, s->uttid); if (NOT_S3LATID(s->exit_id)) { E_INFO("lattice ID: %d\n", s->exit_id); E_ERROR("%s: NO RECOGNITION\n", s->uttid); return NULL; } else { /* BAD_S3WID => Any right context */ lattice_backtrace(fwg->lathist, s->exit_id, BAD_S3WID, &hyp, s->kbc->lmset->cur_lm, kbcore_dict(s->kbc), fwg->ctxt, s->kbc->fillpen); ghyp = NULL; for (tmph = hyp; tmph; tmph = tmph->next) { ghyp = glist_add_ptr(ghyp, (void *) tmph); } rhyp = glist_reverse(ghyp); return rhyp; } }
/* * Enter a new LMstate into the current frame LMstates trees; called ONLY IF not already * present. */ static void vithist_lmstate_enter(vithist_t * vh, int32 vhid, vithist_entry_t * ve) { vh_lms2vh_t *lms2vh, *child; s3lmwid32_t lwid; lwid = ve->lmstate.lm3g.lwid[0]; if ((lms2vh = vh->lms2vh_root[lwid]) == NULL) { lms2vh = (vh_lms2vh_t *) ckd_calloc(1, sizeof(vh_lms2vh_t)); vh->lms2vh_root[lwid] = lms2vh; lms2vh->state = lwid; lms2vh->children = NULL; vh->lwidlist = glist_add_int32(vh->lwidlist, (int32) lwid); } else { assert(lms2vh->state == lwid); } child = (vh_lms2vh_t *) ckd_calloc(1, sizeof(vh_lms2vh_t)); child->state = ve->lmstate.lm3g.lwid[1]; child->children = NULL; child->vhid = vhid; child->ve = ve; lms2vh->children = glist_add_ptr(lms2vh->children, (void *) child); }
ngram_model_t * ps_update_lmset(ps_decoder_t *ps, ngram_model_t *lmset) { ngram_search_t *ngs; ps_search_t *search; /* Look for N-Gram search. */ search = ps_find_search(ps, "ngram"); if (search == NULL) { /* Initialize N-Gram search. */ search = ngram_search_init(ps->config, ps->acmod, ps->dict, ps->d2p); if (search == NULL) return NULL; search->pls = ps->phone_loop; ps->searches = glist_add_ptr(ps->searches, search); ngs = (ngram_search_t *)search; } else { ngs = (ngram_search_t *)search; /* Free any previous lmset if this is a new one. */ if (ngs->lmset != NULL && ngs->lmset != lmset) ngram_model_free(ngs->lmset); ngs->lmset = lmset; /* Tell N-Gram search to update its view of the world. */ if (ps_search_reinit(search, ps->dict, ps->d2p) < 0) return NULL; } ps->search = search; return ngs->lmset; }
glist_t vithist_sort (glist_t vithist_list) { heap_t heap; gnode_t *gn; vithist_t *h; glist_t vithist_new; int32 ret, score; vithist_new = NULL; heap = heap_new(); for (gn = vithist_list; gn; gn = gnode_next(gn)) { h = (vithist_t *) gnode_ptr(gn); if (heap_insert (heap, (void *) h, h->scr) < 0) { E_ERROR("Panic: heap_insert() failed\n"); return NULL; } } /* * Note: The heap returns nodes with ASCENDING values; and glist_add adds new nodes to the * HEAD of the list. So we get a glist in the desired descending score order. */ while ((ret = heap_pop (heap, (void **)(&h), &score)) > 0) vithist_new = glist_add_ptr (vithist_new, (void *)h); if (ret < 0) { E_ERROR("Panic: heap_pop() failed\n"); return NULL; } heap_destroy (heap); return vithist_new; }
glist_t vithist_backtrace (vithist_t *hist, int32 *senscale) { glist_t hyp; hyp_t *hypnode; hyp = NULL; for (; hist; hist = hist->hist) { hypnode = (hyp_t *) mymalloc (sizeof(hyp_t)); hypnode->id = hist->id; hypnode->ef = hist->frm; if (hist->hist) { hypnode->sf = hist->hist->frm+1; hypnode->ascr = hist->scr - hist->hist->scr; /* Still scaled */ } else { hypnode->sf = 0; hypnode->ascr = hist->scr; } /* Undo senone score scaling */ hypnode->ascr += senone_get_senscale (senscale, hypnode->sf, hypnode->ef); hypnode->lscr = 0; hypnode->scr = hypnode->ascr; hyp = glist_add_ptr (hyp, (void *)hypnode); } return hyp; }
/* * Add the given transition to the FSG transition matrix. Duplicates (i.e., * two transitions between the same states, with the same word label) are * flagged and only the highest prob retained. */ static void word_fsg_trans_add(word_fsg_t * fsg, int32 from, int32 to, int32 logp, int32 wid) { word_fsglink_t *link; gnode_t *gn; /* Check for duplicate link (i.e., link already exists with label=wid) */ for (gn = fsg->trans[from][to]; gn; gn = gnode_next(gn)) { link = (word_fsglink_t *) gnode_ptr(gn); if (link->wid == wid) { #if 0 E_WARN ("Duplicate transition %d -> %d ('%s'); highest prob kept\n", from, to, dict_wordstr(fsg->dict, wid)); #endif if (link->logs2prob < logp) link->logs2prob = logp; return; } } /* Create transition object */ link = (word_fsglink_t *) ckd_calloc(1, sizeof(word_fsglink_t)); link->from_state = from; link->to_state = to; link->logs2prob = logp; link->wid = wid; fsg->trans[from][to] = glist_add_ptr(fsg->trans[from][to], (void *) link); }
void fsg_model_trans_add(fsg_model_t * fsg, int32 from, int32 to, int32 logp, int32 wid) { fsg_link_t *link; glist_t gl; gnode_t *gn; if (fsg->trans[from].trans == NULL) fsg->trans[from].trans = hash_table_new(5, HASH_CASE_YES); /* Check for duplicate link (i.e., link already exists with label=wid) */ for (gn = gl = fsg_model_trans(fsg, from, to); gn; gn = gnode_next(gn)) { link = (fsg_link_t *) gnode_ptr(gn); if (link->wid == wid) { if (link->logs2prob < logp) link->logs2prob = logp; return; } } /* Create transition object */ link = listelem_malloc(fsg->link_alloc); link->from_state = from; link->to_state = to; link->logs2prob = logp; link->wid = wid; /* Add it to the list of transitions and update the hash table */ gl = glist_add_ptr(gl, (void *) link); hash_table_replace_bkey(fsg->trans[from].trans, (char const *) &link->to_state, sizeof(link->to_state), gl); }
jsgf_t * jsgf_grammar_new(jsgf_t *parent) { jsgf_t *grammar; grammar = ckd_calloc(1, sizeof(*grammar)); /* If this is an imported/subgrammar, then we will share a global * namespace with the parent grammar. */ if (parent) { grammar->rules = parent->rules; grammar->imports = parent->imports; grammar->searchpath = parent->searchpath; grammar->parent = parent; } else { char *jsgf_path; grammar->rules = hash_table_new(64, 0); grammar->imports = hash_table_new(16, 0); /* Silvio Moioli: no getenv() in Windows CE */ #if !defined(_WIN32_WCE) if ((jsgf_path = getenv("JSGF_PATH")) != NULL) { char *word, *c; /* FIXME: This should be a function in libsphinxbase. */ /* FIXME: Also nextword() is totally useless... */ word = jsgf_path = ckd_salloc(jsgf_path); while ((c = strchr(word, ':'))) { *c = '\0'; grammar->searchpath = glist_add_ptr(grammar->searchpath, word); word = c + 1; } grammar->searchpath = glist_add_ptr(grammar->searchpath, word); grammar->searchpath = glist_reverse(grammar->searchpath); } else { /* Default to current directory. */ grammar->searchpath = glist_add_ptr(grammar->searchpath, ckd_salloc(".")); } #endif } return grammar; }
jsgf_rule_t * jsgf_optional_new(jsgf_t *jsgf, jsgf_rhs_t *exp) { jsgf_rhs_t *rhs = ckd_calloc(1, sizeof(*rhs)); jsgf_atom_t *atom = jsgf_atom_new("<NULL>", 1.0); rhs->alt = exp; rhs->atoms = glist_add_ptr(NULL, atom); return jsgf_define_rule(jsgf, NULL, rhs, 0); }
glist_t srch_FLAT_FWD_bestpath_impl(void *srch, /**< A void pointer to a search structure */ dag_t * dag) { srch_t *s; srch_FLAT_FWD_graph_t *fwg; float32 bestpathlw; float64 lwf; srch_hyp_t *tmph, *bph; glist_t ghyp, rhyp; s = (srch_t *) srch; fwg = (srch_FLAT_FWD_graph_t *) s->grh->graph_struct; assert(fwg->lathist); bestpathlw = cmd_ln_float32_r(kbcore_config(fwg->kbcore), "-bestpathlw"); lwf = bestpathlw ? (bestpathlw / cmd_ln_float32_r(kbcore_config(fwg->kbcore), "-lw")) : 1.0; flat_fwd_dag_add_fudge_edges(fwg, dag, cmd_ln_int32_r(kbcore_config(fwg->kbcore), "-dagfudge"), cmd_ln_int32_r(kbcore_config(fwg->kbcore), "-min_endfr"), (void *) fwg->lathist, s->kbc->dict); /* Bypass filler nodes */ if (!dag->filler_removed) { /* If Viterbi search terminated in filler word coerce final DAG node to FINISH_WORD */ if (dict_filler_word(s->kbc->dict, dag->end->wid)) dag->end->wid = s->kbc->dict->finishwid; if (dag_bypass_filler_nodes(dag, lwf, s->kbc->dict, s->kbc->fillpen) < 0) E_ERROR("maxedge limit (%d) exceeded\n", dag->maxedge); else dag->filler_removed = 1; } bph = dag_search(dag, s->uttid, lwf, dag->end, s->kbc->dict, s->kbc->lmset->cur_lm, s->kbc->fillpen); if (bph != NULL) { ghyp = NULL; for (tmph = bph; tmph; tmph = tmph->next) ghyp = glist_add_ptr(ghyp, (void *) tmph); rhyp = glist_reverse(ghyp); return rhyp; } else { return NULL; } }
void jsgf_add_link(jsgf_t *grammar, jsgf_atom_t *atom, int from, int to) { jsgf_link_t *link; link = ckd_calloc(1, sizeof(*link)); link->from = from; link->to = to; link->atom = atom; grammar->links = glist_add_ptr(grammar->links, link); }
/* * Obtain transitive closure of NULL transitions in the given FSG. (Initial * list of such transitions is given.) * Return value: Updated list of null transitions. */ static glist_t word_fsg_null_trans_closure(word_fsg_t * fsg, glist_t nulls) { gnode_t *gn1, *gn2; int updated; word_fsglink_t *tl1, *tl2; int32 k, n; E_INFO("Computing transitive closure for null transitions\n"); /* * Probably not the most efficient closure implementation, in general, but * probably reasonably efficient for a sparse null transition matrix. */ n = 0; do { updated = FALSE; for (gn1 = nulls; gn1; gn1 = gnode_next(gn1)) { tl1 = (word_fsglink_t *) gnode_ptr(gn1); assert(tl1->wid < 0); for (gn2 = nulls; gn2; gn2 = gnode_next(gn2)) { tl2 = (word_fsglink_t *) gnode_ptr(gn2); if (tl1->to_state == tl2->from_state) { k = word_fsg_null_trans_add(fsg, tl1->from_state, tl2->to_state, tl1->logs2prob + tl2->logs2prob); if (k >= 0) { updated = TRUE; if (k > 0) { nulls = glist_add_ptr(nulls, (void *) fsg-> null_trans[tl1-> from_state][tl2-> to_state]); n++; } } } } } } while (updated); E_INFO("%d null transitions added\n", n); return nulls; }
static void renormalize_hmms(phone_loop_search_t *pls, int frame_idx, int32 norm) { phone_loop_renorm_t *rn = ckd_calloc(1, sizeof(*rn)); int i; pls->renorm = glist_add_ptr(pls->renorm, rn); rn->frame_idx = frame_idx; rn->norm = norm; for (i = 0; i < pls->n_phones; ++i) { hmm_normalize((hmm_t *)&pls->phones[i], norm); } }
jsgf_atom_t * jsgf_kleene_new(jsgf_t *jsgf, jsgf_atom_t *atom, int plus) { jsgf_rule_t *rule; jsgf_atom_t *rule_atom; jsgf_rhs_t *rhs; /* Generate an "internal" rule of the form (<NULL> | <name> <g0006>) */ /* Or if plus is true, (<name> | <name> <g0006>) */ rhs = ckd_calloc(1, sizeof(*rhs)); if (plus) rhs->atoms = glist_add_ptr(NULL, jsgf_atom_new(atom->name, 1.0)); else rhs->atoms = glist_add_ptr(NULL, jsgf_atom_new("<NULL>", 1.0)); rule = jsgf_define_rule(jsgf, NULL, rhs, 0); rule_atom = jsgf_atom_new(rule->name, 1.0); rhs = ckd_calloc(1, sizeof(*rhs)); rhs->atoms = glist_add_ptr(NULL, rule_atom); rhs->atoms = glist_add_ptr(rhs->atoms, atom); rule->rhs->alt = rhs; return jsgf_atom_new(rule->name, 1.0); }
glist_t vithist_append (glist_t hlist, int32 id, int32 frm, int32 score, vithist_t *hist, vithist_t *lmhist) { vithist_t *h; h = (vithist_t *) mymalloc (sizeof(vithist_t)); h->id = id; h->frm = frm; h->scr = score; h->hist = hist; h->lmhist = lmhist ? lmhist : h; /* Self if none provided */ hlist = glist_add_ptr (hlist, (void *)h); return hlist; }
static int expand_rule(jsgf_t * grammar, jsgf_rule_t * rule, int rule_entry, int rule_exit) { jsgf_rule_stack_t *rule_stack_entry; jsgf_rhs_t *rhs; /* Push this rule onto the stack */ rule_stack_entry = (jsgf_rule_stack_t *) ckd_calloc(1, sizeof(jsgf_rule_stack_t)); rule_stack_entry->rule = rule; rule_stack_entry->entry = rule_entry; grammar->rulestack = glist_add_ptr(grammar->rulestack, rule_stack_entry); for (rhs = rule->rhs; rhs; rhs = rhs->alt) { int lastnode; lastnode = expand_rhs(grammar, rule, rhs, rule_entry, rule_exit); if (lastnode == NO_NODE) { return NO_NODE; } else if (lastnode == RECURSIVE_NODE) { /* The rhs ended with right-recursion, i.e. a transition to an earlier state. Nothing needs to happen at this level. */ ; } else if (rule_exit == NO_NODE) { /* If this rule doesn't have an exit state yet, use the exit state of its first right-hand-side. All other right-hand-sides will use this exit state. */ assert(lastnode >= 0); rule_exit = lastnode; } } /* If no exit-state was created, use the entry-state. */ if (rule_exit == NO_NODE) { rule_exit = rule_entry; } /* Pop this rule from the rule stack */ ckd_free(gnode_ptr(grammar->rulestack)); grammar->rulestack = gnode_free(grammar->rulestack, NULL); return rule_exit; }
static int expand_rule(jsgf_t *grammar, jsgf_rule_t *rule) { jsgf_rhs_t *rhs; float norm; /* Push this rule onto the stack */ grammar->rulestack = glist_add_ptr(grammar->rulestack, rule); /* Normalize weights for all alternatives exiting rule->entry */ norm = 0; for (rhs = rule->rhs; rhs; rhs = rhs->alt) { if (rhs->atoms) { jsgf_atom_t *atom = gnode_ptr(rhs->atoms); norm += atom->weight; } } rule->entry = grammar->nstate++; rule->exit = grammar->nstate++; if (norm == 0) norm = 1; for (rhs = rule->rhs; rhs; rhs = rhs->alt) { int lastnode; if (rhs->atoms) { jsgf_atom_t *atom = gnode_ptr(rhs->atoms); atom->weight /= norm; } lastnode = expand_rhs(grammar, rule, rhs); if (lastnode == -1) { return -1; } else { jsgf_add_link(grammar, NULL, lastnode, rule->exit); } } /* Pop this rule from the rule stack */ grammar->rulestack = gnode_free(grammar->rulestack, NULL); return rule->exit; }
fsg_set_t * ps_update_fsgset(ps_decoder_t *ps) { ps_search_t *search; /* Look for FSG search. */ search = ps_find_search(ps, "fsg"); if (search == NULL) { /* Initialize FSG search. */ search = fsg_search_init(ps->config, ps->acmod, ps->dict, ps->d2p); search->pls = ps->phone_loop; ps->searches = glist_add_ptr(ps->searches, search); } else { /* Tell FSG search to update its view of the world. */ if (ps_search_reinit(search, ps->dict, ps->d2p) < 0) return NULL; } ps->search = search; return (fsg_set_t *)search; }
glist_t srch_FSG_gen_hyp(void *srch /**< a pointer of srch_t */ ) { srch_t *s; fsg_search_t *fsgsrch; srch_hyp_t *tmph; glist_t ghyp, rhyp; s = (srch_t *) srch; fsgsrch = (fsg_search_t *) s->grh->graph_struct; fsg_search_history_backtrace(fsgsrch, TRUE); ghyp = NULL; for (tmph = fsgsrch->hyp; tmph; tmph = tmph->next) { ghyp = glist_add_ptr(ghyp, (void *) tmph); } rhyp = glist_reverse(ghyp); return rhyp; }
fsg_model_t * fsg_model_read(FILE * fp, logmath_t * lmath, float32 lw) { fsg_model_t *fsg; hash_table_t *vocab; hash_iter_t *itor; int32 lastwid; char **wordptr; char *lineptr; char *fsgname; int32 lineno; int32 n, i, j; int n_state, n_trans, n_null_trans; glist_t nulls; float32 p; lineno = 0; vocab = hash_table_new(32, FALSE); wordptr = NULL; lineptr = NULL; nulls = NULL; fsgname = NULL; fsg = NULL; /* Scan upto FSG_BEGIN header */ for (;;) { n = nextline_str2words(fp, &lineno, &lineptr, &wordptr); if (n < 0) { E_ERROR("%s declaration missing\n", FSG_MODEL_BEGIN_DECL); goto parse_error; } if ((strcmp(wordptr[0], FSG_MODEL_BEGIN_DECL) == 0)) { if (n > 2) { E_ERROR("Line[%d]: malformed FSG_BEGIN declaration\n", lineno); goto parse_error; } break; } } /* Save FSG name, or it will get clobbered below :(. * If name is missing, try the default. */ if (n == 2) { fsgname = ckd_salloc(wordptr[1]); } else { E_WARN("FSG name is missing\n"); fsgname = ckd_salloc("unknown"); } /* Read #states */ n = nextline_str2words(fp, &lineno, &lineptr, &wordptr); if ((n != 2) || ((strcmp(wordptr[0], FSG_MODEL_N_DECL) != 0) && (strcmp(wordptr[0], FSG_MODEL_NUM_STATES_DECL) != 0)) || (sscanf(wordptr[1], "%d", &n_state) != 1) || (n_state <= 0)) { E_ERROR ("Line[%d]: #states declaration line missing or malformed\n", lineno); goto parse_error; } /* Now create the FSG. */ fsg = fsg_model_init(fsgname, lmath, lw, n_state); ckd_free(fsgname); fsgname = NULL; /* Read start state */ n = nextline_str2words(fp, &lineno, &lineptr, &wordptr); if ((n != 2) || ((strcmp(wordptr[0], FSG_MODEL_S_DECL) != 0) && (strcmp(wordptr[0], FSG_MODEL_START_STATE_DECL) != 0)) || (sscanf(wordptr[1], "%d", &(fsg->start_state)) != 1) || (fsg->start_state < 0) || (fsg->start_state >= fsg->n_state)) { E_ERROR ("Line[%d]: start state declaration line missing or malformed\n", lineno); goto parse_error; } /* Read final state */ n = nextline_str2words(fp, &lineno, &lineptr, &wordptr); if ((n != 2) || ((strcmp(wordptr[0], FSG_MODEL_F_DECL) != 0) && (strcmp(wordptr[0], FSG_MODEL_FINAL_STATE_DECL) != 0)) || (sscanf(wordptr[1], "%d", &(fsg->final_state)) != 1) || (fsg->final_state < 0) || (fsg->final_state >= fsg->n_state)) { E_ERROR ("Line[%d]: final state declaration line missing or malformed\n", lineno); goto parse_error; } /* Read transitions */ lastwid = 0; n_trans = n_null_trans = 0; for (;;) { int32 wid, tprob; n = nextline_str2words(fp, &lineno, &lineptr, &wordptr); if (n <= 0) { E_ERROR("Line[%d]: transition or FSG_END statement expected\n", lineno); goto parse_error; } if ((strcmp(wordptr[0], FSG_MODEL_END_DECL) == 0)) { break; } if ((strcmp(wordptr[0], FSG_MODEL_T_DECL) == 0) || (strcmp(wordptr[0], FSG_MODEL_TRANSITION_DECL) == 0)) { if (((n != 4) && (n != 5)) || (sscanf(wordptr[1], "%d", &i) != 1) || (sscanf(wordptr[2], "%d", &j) != 1) || (i < 0) || (i >= fsg->n_state) || (j < 0) || (j >= fsg->n_state)) { E_ERROR ("Line[%d]: transition spec malformed; Expecting: from-state to-state trans-prob [word]\n", lineno); goto parse_error; } p = atof_c(wordptr[3]); if ((p <= 0.0) || (p > 1.0)) { E_ERROR ("Line[%d]: transition spec malformed; Expecting float as transition probability\n", lineno); goto parse_error; } } else { E_ERROR("Line[%d]: transition or FSG_END statement expected\n", lineno); goto parse_error; } tprob = (int32) (logmath_log(lmath, p) * fsg->lw); /* Add word to "dictionary". */ if (n > 4) { if (hash_table_lookup_int32(vocab, wordptr[4], &wid) < 0) { (void) hash_table_enter_int32(vocab, ckd_salloc(wordptr[4]), lastwid); wid = lastwid; ++lastwid; } fsg_model_trans_add(fsg, i, j, tprob, wid); ++n_trans; } else { if (fsg_model_null_trans_add(fsg, i, j, tprob) == 1) { ++n_null_trans; nulls = glist_add_ptr(nulls, fsg_model_null_trans(fsg, i, j)); } } } E_INFO("FSG: %d states, %d unique words, %d transitions (%d null)\n", fsg->n_state, hash_table_inuse(vocab), n_trans, n_null_trans); /* Now create a string table from the "dictionary" */ fsg->n_word = hash_table_inuse(vocab); fsg->n_word_alloc = fsg->n_word + 10; /* Pad it a bit. */ fsg->vocab = ckd_calloc(fsg->n_word_alloc, sizeof(*fsg->vocab)); for (itor = hash_table_iter(vocab); itor; itor = hash_table_iter_next(itor)) { char const *word = hash_entry_key(itor->ent); int32 wid = (int32) (long) hash_entry_val(itor->ent); fsg->vocab[wid] = (char *) word; } hash_table_free(vocab); /* Do transitive closure on null transitions */ nulls = fsg_model_null_trans_closure(fsg, nulls); glist_free(nulls); ckd_free(lineptr); ckd_free(wordptr); return fsg; parse_error: for (itor = hash_table_iter(vocab); itor; itor = hash_table_iter_next(itor)) ckd_free((char *) hash_entry_key(itor->ent)); glist_free(nulls); hash_table_free(vocab); ckd_free(fsgname); ckd_free(lineptr); ckd_free(wordptr); fsg_model_free(fsg); return NULL; }
glist_t fsg_model_null_trans_closure(fsg_model_t * fsg, glist_t nulls) { gnode_t *gn1; int updated; fsg_link_t *tl1, *tl2; int32 k, n; E_INFO("Computing transitive closure for null transitions\n"); if (nulls == NULL) { fsg_link_t *null; int i, j; for (i = 0; i < fsg->n_state; ++i) { for (j = 0; j < fsg->n_state; ++j) { if ((null = fsg_model_null_trans(fsg, i, j))) nulls = glist_add_ptr(nulls, null); } } } /* * Probably not the most efficient closure implementation, in general, but * probably reasonably efficient for a sparse null transition matrix. */ n = 0; do { updated = FALSE; for (gn1 = nulls; gn1; gn1 = gnode_next(gn1)) { hash_iter_t *itor; tl1 = (fsg_link_t *) gnode_ptr(gn1); assert(tl1->wid < 0); if (fsg->trans[tl1->to_state].null_trans == NULL) continue; for (itor = hash_table_iter(fsg->trans[tl1->to_state].null_trans); itor; itor = hash_table_iter_next(itor)) { tl2 = (fsg_link_t *) hash_entry_val(itor->ent); k = fsg_model_null_trans_add(fsg, tl1->from_state, tl2->to_state, tl1->logs2prob + tl2->logs2prob); if (k >= 0) { updated = TRUE; if (k > 0) { nulls = glist_add_ptr(nulls, (void *) fsg_model_null_trans (fsg, tl1->from_state, tl2->to_state)); n++; } } } } } while (updated); E_INFO("%d null transitions added\n", n); return nulls; }
/* Find the confidence score and dump the confidence output into the file */ static void confidence_utt(char *uttid, FILE * _confmatchsegfp) { seg_hyp_line_t s_hypline; char line[16384]; char dagfile[16384]; const char *fmt; const char *latdir; const char *latext; E_INFO("Processing %s\n", uttid); if (fgets(line, sizeof(line), _confmatchsegfp) == NULL) E_FATAL("Fail to read a line in the matchsegfp for uttid %s\n", uttid); /* Read the hypseg */ if (read_s3hypseg_line(line, &s_hypline, lmset->cur_lm, dict) == HYPSEG_FAILURE) E_FATAL("Fail to parse matchseg in utt ID %s\n", uttid); E_INFO("Matchseg file name %s\n", s_hypline.seq); if (strcmp(uttid, s_hypline.seq)) E_FATAL("Uttids in control file and matchseg file mismatches\n"); /* Read the lattice */ latdir = cmd_ln_str_r(config, "-inlatdir"); latext = cmd_ln_str_r(config, "-latext"); if (latdir) sprintf(dagfile, "%s/%s.%s", latdir, uttid, latext); else sprintf(dagfile, "%s.%s", uttid, latext); E_INFO("Reading DAG file: %s\n", dagfile); if (confidence_word_posterior(dagfile, &s_hypline, uttid, lmset->cur_lm, dict, fpen) == CONFIDENCE_FAILURE) { E_INFO("Fail to compute word posterior probability \n"); } #if 0 if (ca_dag_load_lattice (dagfile, &word_lattice, lmset->cur_lm, dict, fpen) == CONFIDENCE_FAILURE) E_FATAL("Unable to load dag %s for uttid %s\n", dagfile, uttid); /* Compute Alpha-beta */ if (alpha_beta(&word_lattice, lmset->cur_lm, dict) == CONFIDENCE_FAILURE) E_FATAL("Unable to compute alpha beta score for uttid %s\n", uttid); /* Compute Posterior WORD probability */ if (pwp(&s_hypline, &word_lattice) == CONFIDENCE_FAILURE) E_FATAL("Unable to compute pwp for uttid %s\n", uttid); #endif /* Compute LM type */ if (compute_lmtype(&s_hypline, lmset->cur_lm, dict) == CONFIDENCE_FAILURE) E_FATAL("Fail to compute lm type\n"); /* combined LM type */ if (compute_combined_lmtype(&s_hypline) == CONFIDENCE_FAILURE) E_FATAL("Fail to compute lm type\n"); /* Dump pwp line */ fmt = cmd_ln_str_r(config, "-confoutputfmt"); if (!strcmp(fmt, "scores")) { dump_line(stdout, &s_hypline, dict); dump_line(outconfmatchsegfp, &s_hypline, dict); } else { glist_t hyp; srch_hyp_t *s; conf_srch_hyp_t *h; hyp = NULL; for (h = (conf_srch_hyp_t *) s_hypline.wordlist; h; h = h->next) { s = &(h->sh); hyp = glist_add_ptr(hyp, (void *) s); } matchseg_write(stdout, hyp, uttid, NULL, lmset->cur_lm, dict, 0, NULL, 0); matchseg_write(outconfmatchsegfp, hyp, uttid, NULL, lmset->cur_lm, dict, 0, NULL, 0); } #if 0 /* Delete lattice, delete hypsegline */ if (ca_dag_free_lattice(&word_lattice) == CONFIDENCE_FAILURE) { E_WARN("Fail to free lattice.\n"); return CONFIDENCE_FAILURE; } #endif if (free_seg_hyp_line(&s_hypline) != HYPSEG_SUCCESS) E_FATAL("Fail to free the segment hypothesis line structure. \n"); }
glist_t fsg_model_null_trans_closure(fsg_model_t * fsg, glist_t nulls) { gnode_t *gn1; int updated; fsg_link_t *tl1, *tl2; int32 k, n; E_INFO("Computing transitive closure for null transitions\n"); /* If our caller didn't give us a list of null-transitions, make such a list. Just loop through all the FSG states, and all the null-transitions in that state (which are kept in their own hash table). */ if (nulls == NULL) { int i; for (i = 0; i < fsg->n_state; ++i) { hash_iter_t *itor; hash_table_t *null_trans = fsg->trans[i].null_trans; if (null_trans == NULL) continue; for (itor = hash_table_iter(null_trans); itor != NULL; itor = hash_table_iter_next(itor)) { nulls = glist_add_ptr(nulls, hash_entry_val(itor->ent)); } } } /* * Probably not the most efficient closure implementation, in general, but * probably reasonably efficient for a sparse null transition matrix. */ n = 0; do { updated = FALSE; for (gn1 = nulls; gn1; gn1 = gnode_next(gn1)) { hash_iter_t *itor; tl1 = (fsg_link_t *) gnode_ptr(gn1); assert(tl1->wid < 0); if (fsg->trans[tl1->to_state].null_trans == NULL) continue; for (itor = hash_table_iter(fsg->trans[tl1->to_state].null_trans); itor; itor = hash_table_iter_next(itor)) { tl2 = (fsg_link_t *) hash_entry_val(itor->ent); k = fsg_model_null_trans_add(fsg, tl1->from_state, tl2->to_state, tl1->logs2prob + tl2->logs2prob); if (k >= 0) { updated = TRUE; if (k > 0) { nulls = glist_add_ptr(nulls, (void *) fsg_model_null_trans (fsg, tl1->from_state, tl2->to_state)); n++; } } } } } while (updated); E_INFO("%d null transitions added\n", n); return nulls; }
static void huff_code_canonicalize(huff_code_t *hc, huff_node_t *root) { glist_t agenda; uint32 *nextcode; int i, ncw; hc->firstcode =(uint32*) ckd_calloc(hc->maxbits+1, sizeof(*hc->firstcode)); hc->syms = (huff_codeword_t**)ckd_calloc(hc->maxbits+1, sizeof(*hc->syms)); hc->numl =(uint32*) ckd_calloc(hc->maxbits+1, sizeof(*nextcode)); nextcode =(uint32*) ckd_calloc(hc->maxbits+1, sizeof(*nextcode)); /* Traverse the tree, annotating it with the actual bit * lengths, and histogramming them in numl. */ root->nbits = 0; ncw = 0; agenda = glist_add_ptr(0, root); while (agenda) { huff_node_t *node = (huff_node_t*)gnode_ptr(agenda); agenda = gnode_free(agenda, 0); if (node->l) { node->l->nbits = node->nbits + 1; agenda = glist_add_ptr(agenda, node->l); node->r.r->nbits = node->nbits + 1; agenda = glist_add_ptr(agenda, node->r.r); } else { hc->numl[node->nbits]++; ncw++; } } /* Create starting codes and symbol tables for each bit length. */ hc->syms[hc->maxbits] = (huff_codeword_t*)ckd_calloc(hc->numl[hc->maxbits], sizeof(**hc->syms)); for (i = hc->maxbits - 1; i > 0; --i) { hc->firstcode[i] = (hc->firstcode[i+1] + hc->numl[i+1]) / 2; hc->syms[i] = (huff_codeword_t*)ckd_calloc(hc->numl[i], sizeof(**hc->syms)); } memcpy(nextcode, hc->firstcode, (hc->maxbits + 1) * sizeof(*nextcode)); /* Traverse the tree again to produce the codebook itself. */ hc->codewords = hash_table_new(ncw, HASH_CASE_YES); agenda = glist_add_ptr(0, root); while (agenda) { huff_node_t *node = (huff_node_t*)gnode_ptr(agenda); agenda = gnode_free(agenda, 0); if (node->l) { agenda = glist_add_ptr(agenda, node->l); agenda = glist_add_ptr(agenda, node->r.r); } else { /* Initialize codebook entry, which also retains symbol pointer. */ huff_codeword_t *cw; uint32 codeword = nextcode[node->nbits] & ((1 << node->nbits) - 1); cw = hc->syms[node->nbits] + (codeword - hc->firstcode[node->nbits]); cw->nbits = node->nbits; cw->r.sval = node->r.sval; /* Will copy ints too... */ cw->codeword = codeword; if (hc->type == HUFF_CODE_INT) { hash_table_enter_bkey(hc->codewords, (char const *)&cw->r.ival, sizeof(cw->r.ival), (void *)cw); } else { hash_table_enter(hc->codewords, cw->r.sval, (void *)cw); } ++nextcode[node->nbits]; } } ckd_free(nextcode); }
int32 read_classdef_file(hash_table_t * classes, const char *file_name) { FILE *fp; int32 is_pipe; int inclass; /**< Are we currently reading a list of class words? */ int32 rv = -1; gnode_t *gn; glist_t classwords = NULL; glist_t classprobs = NULL; char *classname = NULL; if ((fp = fopen_comp(file_name, "r", &is_pipe)) == NULL) { E_ERROR("File %s not found\n", file_name); return -1; } inclass = FALSE; while (!feof(fp)) { char line[512]; char *wptr[2]; int n_words; if (fgets(line, sizeof(line), fp) == NULL) break; n_words = str2words(line, wptr, 2); if (n_words <= 0) continue; if (inclass) { /* Look for an end of class marker. */ if (n_words == 2 && 0 == strcmp(wptr[0], "END")) { classdef_t *classdef; gnode_t *word, *weight; int32 i; if (classname == NULL || 0 != strcmp(wptr[1], classname)) goto error_out; inclass = FALSE; /* Construct a class from the list of words collected. */ classdef = ckd_calloc(1, sizeof(*classdef)); classwords = glist_reverse(classwords); classprobs = glist_reverse(classprobs); classdef->n_words = glist_count(classwords); classdef->words = ckd_calloc(classdef->n_words, sizeof(*classdef->words)); classdef->weights = ckd_calloc(classdef->n_words, sizeof(*classdef->weights)); word = classwords; weight = classprobs; for (i = 0; i < classdef->n_words; ++i) { classdef->words[i] = gnode_ptr(word); classdef->weights[i] = gnode_float32(weight); word = gnode_next(word); weight = gnode_next(weight); } /* Add this class to the hash table. */ if (hash_table_enter(classes, classname, classdef) != classdef) { classdef_free(classdef); goto error_out; } /* Reset everything. */ glist_free(classwords); glist_free(classprobs); classwords = NULL; classprobs = NULL; classname = NULL; } else { float32 fprob; if (n_words == 2) fprob = atof_c(wptr[1]); else fprob = 1.0f; /* Add it to the list of words for this class. */ classwords = glist_add_ptr(classwords, ckd_salloc(wptr[0])); classprobs = glist_add_float32(classprobs, fprob); } } else { /* Start a new LM class if the LMCLASS marker is seen */ if (n_words == 2 && 0 == strcmp(wptr[0], "LMCLASS")) { if (inclass) goto error_out; inclass = TRUE; classname = ckd_salloc(wptr[1]); } /* Otherwise, just ignore whatever junk we got */ } } rv = 0; /* Success. */ error_out: /* Free all the stuff we might have allocated. */ fclose_comp(fp, is_pipe); for (gn = classwords; gn; gn = gnode_next(gn)) ckd_free(gnode_ptr(gn)); glist_free(classwords); glist_free(classprobs); ckd_free(classname); return rv; }
void fsg_history_entry_add (fsg_history_t *h, word_fsglink_t *link, int32 frame, int32 score, int32 pred, int32 lc, fsg_pnode_ctxt_t rc) { fsg_hist_entry_t *entry, *new_entry; int32 s; gnode_t *gn, *prev_gn; /* Skip the optimization for the initial dummy entries; always enter them */ if (frame < 0) { new_entry = (fsg_hist_entry_t *) ckd_calloc(1, sizeof(fsg_hist_entry_t)); new_entry->fsglink = link; new_entry->frame = frame; new_entry->score = score; new_entry->pred = pred; new_entry->lc = lc; new_entry->rc = rc; blkarray_list_append(h->entries, (void *)new_entry); return; } s = word_fsglink_to_state(link); /* Locate where this entry should be inserted in frame_entries[s][lc] */ prev_gn = NULL; for (gn = h->frame_entries[s][lc]; gn; gn = gnode_next(gn)) { entry = (fsg_hist_entry_t *) gnode_ptr(gn); if (entry->score < score) break; /* Found where to insert new entry */ /* Existing entry score not worse than new score */ if (fsg_pnode_ctxt_sub(&rc, &(entry->rc)) == 0) return; /* rc set reduced to 0; new entry can be ignored */ prev_gn = gn; } /* Create new entry after prev_gn (if prev_gn is NULL, at head) */ new_entry = (fsg_hist_entry_t *) ckd_calloc(1, sizeof(fsg_hist_entry_t)); new_entry->fsglink = link; new_entry->frame = frame; new_entry->score = score; new_entry->pred = pred; new_entry->lc = lc; new_entry->rc = rc; /* Note: rc set must be non-empty at this point */ if (! prev_gn) { h->frame_entries[s][lc] = glist_add_ptr (h->frame_entries[s][lc], (void *) new_entry); prev_gn = h->frame_entries[s][lc]; } else prev_gn = glist_insert_ptr (prev_gn, (void *) new_entry); /* * Update the rc set of all the remaining entries in the list. At this * point, gn is the entry, if any, immediately following new entry. */ while (gn) { entry = (fsg_hist_entry_t *) gnode_ptr(gn); if (fsg_pnode_ctxt_sub(&(entry->rc), &rc) == 0) { /* rc set of entry reduced to 0; can prune this entry */ ckd_free ((void *)entry); gn = gnode_free (gn, prev_gn); } else { prev_gn = gn; gn = gnode_next(gn); } } }