static fsg_model_t *
jsgf_build_fsg_internal(jsgf_t * grammar, jsgf_rule_t * rule,
logmath_t * lmath, float32 lw, int do_closure)
{
fsg_model_t *fsg;
glist_t nulls;
gnode_t *gn;
int rule_entry, rule_exit;
/* Clear previous links */
for (gn = grammar->links; gn; gn = gnode_next(gn)) {
ckd_free(gnode_ptr(gn));
}
glist_free(grammar->links);
grammar->links = NULL;
grammar->nstate = 0;
/* Create the top-level entry state, and expand the
top-level rule. */
rule_entry = grammar->nstate++;
rule_exit = expand_rule(grammar, rule, rule_entry, NO_NODE);
/* If no exit-state was created, create one. */
if (rule_exit == NO_NODE) {
rule_exit = grammar->nstate++;
jsgf_add_link(grammar, NULL, rule_entry, rule_exit);
}
fsg = fsg_model_init(rule->name, lmath, lw, grammar->nstate);
fsg->start_state = rule_entry;
fsg->final_state = rule_exit;
grammar->links = glist_reverse(grammar->links);
for (gn = grammar->links; gn; gn = gnode_next(gn)) {
jsgf_link_t *link = gnode_ptr(gn);
if (link->atom) {
if (jsgf_atom_is_rule(link->atom)) {
fsg_model_null_trans_add(fsg, link->from, link->to,
logmath_log(lmath,
link->atom->weight));
}
else {
int wid = fsg_model_word_add(fsg, link->atom->name);
fsg_model_trans_add(fsg, link->from, link->to,
logmath_log(lmath, link->atom->weight),
wid);
}
}
else {
fsg_model_null_trans_add(fsg, link->from, link->to, 0);
}
}
if (do_closure) {
nulls = fsg_model_null_trans_closure(fsg, NULL);
glist_free(nulls);
}
return fsg;
}
ReturnType Recognizer::addGrammar(Integers& id, const Grammar& grammar) {
if (decoder == NULL) return BAD_STATE;
std::ostringstream grammar_name;
grammar_name << grammar_index;
grammar_names.push_back(grammar_name.str());
current_grammar = fsg_model_init(grammar_names.back().c_str(), logmath, 1.0, grammar.numStates);
if (current_grammar == NULL)
return RUNTIME_ERROR;
current_grammar->start_state = grammar.start;
current_grammar->final_state = grammar.end;
for (int i=0;i<grammar.transitions.size();i++) {
if (grammar.transitions.at(i).word.size() == 0)
fsg_model_null_trans_add(current_grammar, grammar.transitions.at(i).from, grammar.transitions.at(i).to, grammar.transitions.at(i).logp);
else
fsg_model_trans_add(current_grammar, grammar.transitions.at(i).from, grammar.transitions.at(i).to, grammar.transitions.at(i).logp, fsg_model_word_add(current_grammar, grammar.transitions.at(i).word.c_str()));
}
fsg_model_add_silence(current_grammar, "<sil>", -1, 1.0);
if(ps_set_fsg(decoder, grammar_names.back().c_str(), current_grammar)) {
return RUNTIME_ERROR;
}
if (id.size() == 0) id.push_back(grammar_index);
else id.at(0) = grammar_index;
grammar_index++;
// We switch to the newly added grammar right away
if (ps_set_search(decoder, grammar_names.back().c_str())) {
return RUNTIME_ERROR;
}
return SUCCESS;
}
static fsg_model_t *
jsgf_build_fsg_internal(jsgf_t *grammar, jsgf_rule_t *rule,
logmath_t *lmath, float32 lw, int do_closure)
{
fsg_model_t *fsg;
glist_t nulls;
gnode_t *gn;
/* Clear previous links */
for (gn = grammar->links; gn; gn = gnode_next(gn)) {
ckd_free(gnode_ptr(gn));
}
glist_free(grammar->links);
grammar->links = NULL;
rule->entry = rule->exit = 0;
grammar->nstate = 0;
expand_rule(grammar, rule);
fsg = fsg_model_init(rule->name, lmath, lw, grammar->nstate);
fsg->start_state = rule->entry;
fsg->final_state = rule->exit;
grammar->links = glist_reverse(grammar->links);
for (gn = grammar->links; gn; gn = gnode_next(gn)) {
jsgf_link_t *link = gnode_ptr(gn);
if (link->atom) {
if (jsgf_atom_is_rule(link->atom)) {
fsg_model_null_trans_add(fsg, link->from, link->to,
logmath_log(lmath, link->atom->weight));
}
else {
int wid = fsg_model_word_add(fsg, link->atom->name);
fsg_model_trans_add(fsg, link->from, link->to,
logmath_log(lmath, link->atom->weight), wid);
}
}
else {
fsg_model_null_trans_add(fsg, link->from, link->to, 0);
}
}
if (do_closure) {
nulls = fsg_model_null_trans_closure(fsg, NULL);
glist_free(nulls);
}
return fsg;
}
fsg_model_t *
fsg_model_read(FILE * fp, logmath_t * lmath, float32 lw)
{
fsg_model_t *fsg;
hash_table_t *vocab;
hash_iter_t *itor;
int32 lastwid;
char **wordptr;
char *lineptr;
char *fsgname;
int32 lineno;
int32 n, i, j;
int n_state, n_trans, n_null_trans;
glist_t nulls;
float32 p;
lineno = 0;
vocab = hash_table_new(32, FALSE);
wordptr = NULL;
lineptr = NULL;
nulls = NULL;
fsgname = NULL;
fsg = NULL;
/* Scan upto FSG_BEGIN header */
for (;;) {
n = nextline_str2words(fp, &lineno, &lineptr, &wordptr);
if (n < 0) {
E_ERROR("%s declaration missing\n", FSG_MODEL_BEGIN_DECL);
goto parse_error;
}
if ((strcmp(wordptr[0], FSG_MODEL_BEGIN_DECL) == 0)) {
if (n > 2) {
E_ERROR("Line[%d]: malformed FSG_BEGIN declaration\n",
lineno);
goto parse_error;
}
break;
}
}
/* Save FSG name, or it will get clobbered below :(.
* If name is missing, try the default.
*/
if (n == 2) {
fsgname = ckd_salloc(wordptr[1]);
}
else {
E_WARN("FSG name is missing\n");
fsgname = ckd_salloc("unknown");
}
/* Read #states */
n = nextline_str2words(fp, &lineno, &lineptr, &wordptr);
if ((n != 2)
|| ((strcmp(wordptr[0], FSG_MODEL_N_DECL) != 0)
&& (strcmp(wordptr[0], FSG_MODEL_NUM_STATES_DECL) != 0))
|| (sscanf(wordptr[1], "%d", &n_state) != 1)
|| (n_state <= 0)) {
E_ERROR
("Line[%d]: #states declaration line missing or malformed\n",
lineno);
goto parse_error;
}
/* Now create the FSG. */
fsg = fsg_model_init(fsgname, lmath, lw, n_state);
ckd_free(fsgname);
fsgname = NULL;
/* Read start state */
n = nextline_str2words(fp, &lineno, &lineptr, &wordptr);
if ((n != 2)
|| ((strcmp(wordptr[0], FSG_MODEL_S_DECL) != 0)
&& (strcmp(wordptr[0], FSG_MODEL_START_STATE_DECL) != 0))
|| (sscanf(wordptr[1], "%d", &(fsg->start_state)) != 1)
|| (fsg->start_state < 0)
|| (fsg->start_state >= fsg->n_state)) {
E_ERROR
("Line[%d]: start state declaration line missing or malformed\n",
lineno);
goto parse_error;
}
/* Read final state */
n = nextline_str2words(fp, &lineno, &lineptr, &wordptr);
if ((n != 2)
|| ((strcmp(wordptr[0], FSG_MODEL_F_DECL) != 0)
&& (strcmp(wordptr[0], FSG_MODEL_FINAL_STATE_DECL) != 0))
|| (sscanf(wordptr[1], "%d", &(fsg->final_state)) != 1)
|| (fsg->final_state < 0)
|| (fsg->final_state >= fsg->n_state)) {
E_ERROR
("Line[%d]: final state declaration line missing or malformed\n",
lineno);
goto parse_error;
}
/* Read transitions */
lastwid = 0;
n_trans = n_null_trans = 0;
for (;;) {
int32 wid, tprob;
n = nextline_str2words(fp, &lineno, &lineptr, &wordptr);
if (n <= 0) {
E_ERROR("Line[%d]: transition or FSG_END statement expected\n",
lineno);
goto parse_error;
}
if ((strcmp(wordptr[0], FSG_MODEL_END_DECL) == 0)) {
break;
}
if ((strcmp(wordptr[0], FSG_MODEL_T_DECL) == 0)
|| (strcmp(wordptr[0], FSG_MODEL_TRANSITION_DECL) == 0)) {
if (((n != 4) && (n != 5))
|| (sscanf(wordptr[1], "%d", &i) != 1)
|| (sscanf(wordptr[2], "%d", &j) != 1)
|| (i < 0) || (i >= fsg->n_state)
|| (j < 0) || (j >= fsg->n_state)) {
E_ERROR
("Line[%d]: transition spec malformed; Expecting: from-state to-state trans-prob [word]\n",
lineno);
goto parse_error;
}
p = atof_c(wordptr[3]);
if ((p <= 0.0) || (p > 1.0)) {
E_ERROR
("Line[%d]: transition spec malformed; Expecting float as transition probability\n",
lineno);
goto parse_error;
}
}
else {
E_ERROR("Line[%d]: transition or FSG_END statement expected\n",
lineno);
goto parse_error;
}
tprob = (int32) (logmath_log(lmath, p) * fsg->lw);
/* Add word to "dictionary". */
if (n > 4) {
if (hash_table_lookup_int32(vocab, wordptr[4], &wid) < 0) {
(void) hash_table_enter_int32(vocab,
ckd_salloc(wordptr[4]),
lastwid);
wid = lastwid;
++lastwid;
}
fsg_model_trans_add(fsg, i, j, tprob, wid);
++n_trans;
}
else {
if (fsg_model_null_trans_add(fsg, i, j, tprob) == 1) {
++n_null_trans;
nulls =
glist_add_ptr(nulls, fsg_model_null_trans(fsg, i, j));
}
}
}
E_INFO("FSG: %d states, %d unique words, %d transitions (%d null)\n",
fsg->n_state, hash_table_inuse(vocab), n_trans, n_null_trans);
/* Now create a string table from the "dictionary" */
fsg->n_word = hash_table_inuse(vocab);
fsg->n_word_alloc = fsg->n_word + 10; /* Pad it a bit. */
fsg->vocab = ckd_calloc(fsg->n_word_alloc, sizeof(*fsg->vocab));
for (itor = hash_table_iter(vocab); itor;
itor = hash_table_iter_next(itor)) {
char const *word = hash_entry_key(itor->ent);
int32 wid = (int32) (long) hash_entry_val(itor->ent);
fsg->vocab[wid] = (char *) word;
}
hash_table_free(vocab);
/* Do transitive closure on null transitions */
nulls = fsg_model_null_trans_closure(fsg, nulls);
glist_free(nulls);
ckd_free(lineptr);
ckd_free(wordptr);
return fsg;
parse_error:
for (itor = hash_table_iter(vocab); itor;
itor = hash_table_iter_next(itor))
ckd_free((char *) hash_entry_key(itor->ent));
glist_free(nulls);
hash_table_free(vocab);
ckd_free(fsgname);
ckd_free(lineptr);
ckd_free(wordptr);
fsg_model_free(fsg);
return NULL;
}
glist_t
fsg_model_null_trans_closure(fsg_model_t * fsg, glist_t nulls)
{
gnode_t *gn1;
int updated;
fsg_link_t *tl1, *tl2;
int32 k, n;
E_INFO("Computing transitive closure for null transitions\n");
if (nulls == NULL) {
fsg_link_t *null;
int i, j;
for (i = 0; i < fsg->n_state; ++i) {
for (j = 0; j < fsg->n_state; ++j) {
if ((null = fsg_model_null_trans(fsg, i, j)))
nulls = glist_add_ptr(nulls, null);
}
}
}
/*
* Probably not the most efficient closure implementation, in general, but
* probably reasonably efficient for a sparse null transition matrix.
*/
n = 0;
do {
updated = FALSE;
for (gn1 = nulls; gn1; gn1 = gnode_next(gn1)) {
hash_iter_t *itor;
tl1 = (fsg_link_t *) gnode_ptr(gn1);
assert(tl1->wid < 0);
if (fsg->trans[tl1->to_state].null_trans == NULL)
continue;
for (itor = hash_table_iter(fsg->trans[tl1->to_state].null_trans);
itor; itor = hash_table_iter_next(itor)) {
tl2 = (fsg_link_t *) hash_entry_val(itor->ent);
k = fsg_model_null_trans_add(fsg,
tl1->from_state,
tl2->to_state,
tl1->logs2prob +
tl2->logs2prob);
if (k >= 0) {
updated = TRUE;
if (k > 0) {
nulls = glist_add_ptr(nulls, (void *)
fsg_model_null_trans
(fsg, tl1->from_state,
tl2->to_state));
n++;
}
}
}
}
} while (updated);
E_INFO("%d null transitions added\n", n);
return nulls;
}
glist_t
fsg_model_null_trans_closure(fsg_model_t * fsg, glist_t nulls)
{
gnode_t *gn1;
int updated;
fsg_link_t *tl1, *tl2;
int32 k, n;
E_INFO("Computing transitive closure for null transitions\n");
/* If our caller didn't give us a list of null-transitions,
make such a list. Just loop through all the FSG states,
and all the null-transitions in that state (which are kept in
their own hash table). */
if (nulls == NULL) {
int i;
for (i = 0; i < fsg->n_state; ++i) {
hash_iter_t *itor;
hash_table_t *null_trans = fsg->trans[i].null_trans;
if (null_trans == NULL)
continue;
for (itor = hash_table_iter(null_trans);
itor != NULL;
itor = hash_table_iter_next(itor)) {
nulls = glist_add_ptr(nulls, hash_entry_val(itor->ent));
}
}
}
/*
* Probably not the most efficient closure implementation, in general, but
* probably reasonably efficient for a sparse null transition matrix.
*/
n = 0;
do {
updated = FALSE;
for (gn1 = nulls; gn1; gn1 = gnode_next(gn1)) {
hash_iter_t *itor;
tl1 = (fsg_link_t *) gnode_ptr(gn1);
assert(tl1->wid < 0);
if (fsg->trans[tl1->to_state].null_trans == NULL)
continue;
for (itor = hash_table_iter(fsg->trans[tl1->to_state].null_trans);
itor; itor = hash_table_iter_next(itor)) {
tl2 = (fsg_link_t *) hash_entry_val(itor->ent);
k = fsg_model_null_trans_add(fsg,
tl1->from_state,
tl2->to_state,
tl1->logs2prob +
tl2->logs2prob);
if (k >= 0) {
updated = TRUE;
if (k > 0) {
nulls = glist_add_ptr(nulls, (void *)
fsg_model_null_trans
(fsg, tl1->from_state,
tl2->to_state));
n++;
}
}
}
}
} while (updated);
E_INFO("%d null transitions added\n", n);
return nulls;
}
