int32 fsg_model_tag_trans_add(fsg_model_t * fsg, int32 from, int32 to, int32 logp, int32 wid) { fsg_link_t *link, *link2; /* Check for transition probability */ if (logp > 0) { E_FATAL("Null transition prob must be <= 1.0 (state %d -> %d)\n", from, to); } /* Self-loop null transitions (with prob <= 1.0) are redundant */ if (from == to) return -1; if (fsg->trans[from].null_trans == NULL) fsg->trans[from].null_trans = hash_table_new(5, HASH_CASE_YES); /* Check for a duplicate link; if found, keep the higher prob */ link = fsg_model_null_trans(fsg, from, to); if (link) { if (link->logs2prob < logp) { link->logs2prob = logp; return 0; } else return -1; } /* Create null transition object */ link = listelem_malloc(fsg->link_alloc); link->from_state = from; link->to_state = to; link->logs2prob = logp; link->wid = -1; link2 = (fsg_link_t *) hash_table_enter_bkey(fsg->trans[from].null_trans, (char const *) &link->to_state, sizeof(link->to_state), link); assert(link == link2); return 1; }
fsg_model_t * fsg_model_read(FILE * fp, logmath_t * lmath, float32 lw) { fsg_model_t *fsg; hash_table_t *vocab; hash_iter_t *itor; int32 lastwid; char **wordptr; char *lineptr; char *fsgname; int32 lineno; int32 n, i, j; int n_state, n_trans, n_null_trans; glist_t nulls; float32 p; lineno = 0; vocab = hash_table_new(32, FALSE); wordptr = NULL; lineptr = NULL; nulls = NULL; fsgname = NULL; fsg = NULL; /* Scan upto FSG_BEGIN header */ for (;;) { n = nextline_str2words(fp, &lineno, &lineptr, &wordptr); if (n < 0) { E_ERROR("%s declaration missing\n", FSG_MODEL_BEGIN_DECL); goto parse_error; } if ((strcmp(wordptr[0], FSG_MODEL_BEGIN_DECL) == 0)) { if (n > 2) { E_ERROR("Line[%d]: malformed FSG_BEGIN declaration\n", lineno); goto parse_error; } break; } } /* Save FSG name, or it will get clobbered below :(. * If name is missing, try the default. */ if (n == 2) { fsgname = ckd_salloc(wordptr[1]); } else { E_WARN("FSG name is missing\n"); fsgname = ckd_salloc("unknown"); } /* Read #states */ n = nextline_str2words(fp, &lineno, &lineptr, &wordptr); if ((n != 2) || ((strcmp(wordptr[0], FSG_MODEL_N_DECL) != 0) && (strcmp(wordptr[0], FSG_MODEL_NUM_STATES_DECL) != 0)) || (sscanf(wordptr[1], "%d", &n_state) != 1) || (n_state <= 0)) { E_ERROR ("Line[%d]: #states declaration line missing or malformed\n", lineno); goto parse_error; } /* Now create the FSG. */ fsg = fsg_model_init(fsgname, lmath, lw, n_state); ckd_free(fsgname); fsgname = NULL; /* Read start state */ n = nextline_str2words(fp, &lineno, &lineptr, &wordptr); if ((n != 2) || ((strcmp(wordptr[0], FSG_MODEL_S_DECL) != 0) && (strcmp(wordptr[0], FSG_MODEL_START_STATE_DECL) != 0)) || (sscanf(wordptr[1], "%d", &(fsg->start_state)) != 1) || (fsg->start_state < 0) || (fsg->start_state >= fsg->n_state)) { E_ERROR ("Line[%d]: start state declaration line missing or malformed\n", lineno); goto parse_error; } /* Read final state */ n = nextline_str2words(fp, &lineno, &lineptr, &wordptr); if ((n != 2) || ((strcmp(wordptr[0], FSG_MODEL_F_DECL) != 0) && (strcmp(wordptr[0], FSG_MODEL_FINAL_STATE_DECL) != 0)) || (sscanf(wordptr[1], "%d", &(fsg->final_state)) != 1) || (fsg->final_state < 0) || (fsg->final_state >= fsg->n_state)) { E_ERROR ("Line[%d]: final state declaration line missing or malformed\n", lineno); goto parse_error; } /* Read transitions */ lastwid = 0; n_trans = n_null_trans = 0; for (;;) { int32 wid, tprob; n = nextline_str2words(fp, &lineno, &lineptr, &wordptr); if (n <= 0) { E_ERROR("Line[%d]: transition or FSG_END statement expected\n", lineno); goto parse_error; } if ((strcmp(wordptr[0], FSG_MODEL_END_DECL) == 0)) { break; } if ((strcmp(wordptr[0], FSG_MODEL_T_DECL) == 0) || (strcmp(wordptr[0], FSG_MODEL_TRANSITION_DECL) == 0)) { if (((n != 4) && (n != 5)) || (sscanf(wordptr[1], "%d", &i) != 1) || (sscanf(wordptr[2], "%d", &j) != 1) || (i < 0) || (i >= fsg->n_state) || (j < 0) || (j >= fsg->n_state)) { E_ERROR ("Line[%d]: transition spec malformed; Expecting: from-state to-state trans-prob [word]\n", lineno); goto parse_error; } p = atof_c(wordptr[3]); if ((p <= 0.0) || (p > 1.0)) { E_ERROR ("Line[%d]: transition spec malformed; Expecting float as transition probability\n", lineno); goto parse_error; } } else { E_ERROR("Line[%d]: transition or FSG_END statement expected\n", lineno); goto parse_error; } tprob = (int32) (logmath_log(lmath, p) * fsg->lw); /* Add word to "dictionary". */ if (n > 4) { if (hash_table_lookup_int32(vocab, wordptr[4], &wid) < 0) { (void) hash_table_enter_int32(vocab, ckd_salloc(wordptr[4]), lastwid); wid = lastwid; ++lastwid; } fsg_model_trans_add(fsg, i, j, tprob, wid); ++n_trans; } else { if (fsg_model_null_trans_add(fsg, i, j, tprob) == 1) { ++n_null_trans; nulls = glist_add_ptr(nulls, fsg_model_null_trans(fsg, i, j)); } } } E_INFO("FSG: %d states, %d unique words, %d transitions (%d null)\n", fsg->n_state, hash_table_inuse(vocab), n_trans, n_null_trans); /* Now create a string table from the "dictionary" */ fsg->n_word = hash_table_inuse(vocab); fsg->n_word_alloc = fsg->n_word + 10; /* Pad it a bit. */ fsg->vocab = ckd_calloc(fsg->n_word_alloc, sizeof(*fsg->vocab)); for (itor = hash_table_iter(vocab); itor; itor = hash_table_iter_next(itor)) { char const *word = hash_entry_key(itor->ent); int32 wid = (int32) (long) hash_entry_val(itor->ent); fsg->vocab[wid] = (char *) word; } hash_table_free(vocab); /* Do transitive closure on null transitions */ nulls = fsg_model_null_trans_closure(fsg, nulls); glist_free(nulls); ckd_free(lineptr); ckd_free(wordptr); return fsg; parse_error: for (itor = hash_table_iter(vocab); itor; itor = hash_table_iter_next(itor)) ckd_free((char *) hash_entry_key(itor->ent)); glist_free(nulls); hash_table_free(vocab); ckd_free(fsgname); ckd_free(lineptr); ckd_free(wordptr); fsg_model_free(fsg); return NULL; }
glist_t fsg_model_null_trans_closure(fsg_model_t * fsg, glist_t nulls) { gnode_t *gn1; int updated; fsg_link_t *tl1, *tl2; int32 k, n; E_INFO("Computing transitive closure for null transitions\n"); if (nulls == NULL) { fsg_link_t *null; int i, j; for (i = 0; i < fsg->n_state; ++i) { for (j = 0; j < fsg->n_state; ++j) { if ((null = fsg_model_null_trans(fsg, i, j))) nulls = glist_add_ptr(nulls, null); } } } /* * Probably not the most efficient closure implementation, in general, but * probably reasonably efficient for a sparse null transition matrix. */ n = 0; do { updated = FALSE; for (gn1 = nulls; gn1; gn1 = gnode_next(gn1)) { hash_iter_t *itor; tl1 = (fsg_link_t *) gnode_ptr(gn1); assert(tl1->wid < 0); if (fsg->trans[tl1->to_state].null_trans == NULL) continue; for (itor = hash_table_iter(fsg->trans[tl1->to_state].null_trans); itor; itor = hash_table_iter_next(itor)) { tl2 = (fsg_link_t *) hash_entry_val(itor->ent); k = fsg_model_null_trans_add(fsg, tl1->from_state, tl2->to_state, tl1->logs2prob + tl2->logs2prob); if (k >= 0) { updated = TRUE; if (k > 0) { nulls = glist_add_ptr(nulls, (void *) fsg_model_null_trans (fsg, tl1->from_state, tl2->to_state)); n++; } } } } } while (updated); E_INFO("%d null transitions added\n", n); return nulls; }
glist_t fsg_model_null_trans_closure(fsg_model_t * fsg, glist_t nulls) { gnode_t *gn1; int updated; fsg_link_t *tl1, *tl2; int32 k, n; E_INFO("Computing transitive closure for null transitions\n"); /* If our caller didn't give us a list of null-transitions, make such a list. Just loop through all the FSG states, and all the null-transitions in that state (which are kept in their own hash table). */ if (nulls == NULL) { int i; for (i = 0; i < fsg->n_state; ++i) { hash_iter_t *itor; hash_table_t *null_trans = fsg->trans[i].null_trans; if (null_trans == NULL) continue; for (itor = hash_table_iter(null_trans); itor != NULL; itor = hash_table_iter_next(itor)) { nulls = glist_add_ptr(nulls, hash_entry_val(itor->ent)); } } } /* * Probably not the most efficient closure implementation, in general, but * probably reasonably efficient for a sparse null transition matrix. */ n = 0; do { updated = FALSE; for (gn1 = nulls; gn1; gn1 = gnode_next(gn1)) { hash_iter_t *itor; tl1 = (fsg_link_t *) gnode_ptr(gn1); assert(tl1->wid < 0); if (fsg->trans[tl1->to_state].null_trans == NULL) continue; for (itor = hash_table_iter(fsg->trans[tl1->to_state].null_trans); itor; itor = hash_table_iter_next(itor)) { tl2 = (fsg_link_t *) hash_entry_val(itor->ent); k = fsg_model_null_trans_add(fsg, tl1->from_state, tl2->to_state, tl1->logs2prob + tl2->logs2prob); if (k >= 0) { updated = TRUE; if (k > 0) { nulls = glist_add_ptr(nulls, (void *) fsg_model_null_trans (fsg, tl1->from_state, tl2->to_state)); n++; } } } } } while (updated); E_INFO("%d null transitions added\n", n); return nulls; }