const char* ps_get_search(ps_decoder_t *ps) { hash_iter_t *search_it; const char* name = NULL; for (search_it = hash_table_iter(ps->searches); search_it; search_it = hash_table_iter_next(search_it)) { if (hash_entry_val(search_it->ent) == ps->search) { name = hash_entry_key(search_it->ent); break; } } return name; }
static void sseq_compress(mdef_t * m) { hash_table_t *h; s3senid_t **sseq; int32 n_sseq; int32 p, j, k; glist_t g; gnode_t *gn; hash_entry_t *he; k = m->n_emit_state * sizeof(s3senid_t); h = hash_table_new(m->n_phone, HASH_CASE_YES); n_sseq = 0; /* Identify unique senone-sequence IDs. BUG: tmat-id not being considered!! */ for (p = 0; p < m->n_phone; p++) { /* Add senone sequence to hash table */ if ((j = (long) hash_table_enter_bkey(h, (char *) (m->sseq[p]), k, (void *)(long)n_sseq)) == n_sseq) n_sseq++; m->phone[p].ssid = j; } /* Generate compacted sseq table */ sseq = (s3senid_t **) ckd_calloc_2d(n_sseq, m->n_emit_state, sizeof(s3senid_t)); /* freed in mdef_free() */ g = hash_table_tolist(h, &j); assert(j == n_sseq); for (gn = g; gn; gn = gnode_next(gn)) { he = (hash_entry_t *) gnode_ptr(gn); j = (int32)(long)hash_entry_val(he); memcpy(sseq[j], hash_entry_key(he), k); } glist_free(g); /* Free the old, temporary senone sequence table, replace with compacted one */ ckd_free_2d((void **) m->sseq); m->sseq = sseq; m->n_sseq = n_sseq; hash_table_free(h); }
static void ps_free_searches(ps_decoder_t *ps) { if (ps->searches) { /* Release keys manually as we used ckd_salloc to add them, release every search too. */ hash_iter_t *search_it; for (search_it = hash_table_iter(ps->searches); search_it; search_it = hash_table_iter_next(search_it)) { ckd_free((char *) hash_entry_key(search_it->ent)); ps_search_free(hash_entry_val(search_it->ent)); } hash_table_empty(ps->searches); hash_table_free(ps->searches); } ps->searches = NULL; ps->search = NULL; }
fsg_model_t * fsg_model_read(FILE * fp, logmath_t * lmath, float32 lw) { fsg_model_t *fsg; hash_table_t *vocab; hash_iter_t *itor; int32 lastwid; char **wordptr; char *lineptr; char *fsgname; int32 lineno; int32 n, i, j; int n_state, n_trans, n_null_trans; glist_t nulls; float32 p; lineno = 0; vocab = hash_table_new(32, FALSE); wordptr = NULL; lineptr = NULL; nulls = NULL; fsgname = NULL; fsg = NULL; /* Scan upto FSG_BEGIN header */ for (;;) { n = nextline_str2words(fp, &lineno, &lineptr, &wordptr); if (n < 0) { E_ERROR("%s declaration missing\n", FSG_MODEL_BEGIN_DECL); goto parse_error; } if ((strcmp(wordptr[0], FSG_MODEL_BEGIN_DECL) == 0)) { if (n > 2) { E_ERROR("Line[%d]: malformed FSG_BEGIN declaration\n", lineno); goto parse_error; } break; } } /* Save FSG name, or it will get clobbered below :(. * If name is missing, try the default. */ if (n == 2) { fsgname = ckd_salloc(wordptr[1]); } else { E_WARN("FSG name is missing\n"); fsgname = ckd_salloc("unknown"); } /* Read #states */ n = nextline_str2words(fp, &lineno, &lineptr, &wordptr); if ((n != 2) || ((strcmp(wordptr[0], FSG_MODEL_N_DECL) != 0) && (strcmp(wordptr[0], FSG_MODEL_NUM_STATES_DECL) != 0)) || (sscanf(wordptr[1], "%d", &n_state) != 1) || (n_state <= 0)) { E_ERROR ("Line[%d]: #states declaration line missing or malformed\n", lineno); goto parse_error; } /* Now create the FSG. */ fsg = fsg_model_init(fsgname, lmath, lw, n_state); ckd_free(fsgname); fsgname = NULL; /* Read start state */ n = nextline_str2words(fp, &lineno, &lineptr, &wordptr); if ((n != 2) || ((strcmp(wordptr[0], FSG_MODEL_S_DECL) != 0) && (strcmp(wordptr[0], FSG_MODEL_START_STATE_DECL) != 0)) || (sscanf(wordptr[1], "%d", &(fsg->start_state)) != 1) || (fsg->start_state < 0) || (fsg->start_state >= fsg->n_state)) { E_ERROR ("Line[%d]: start state declaration line missing or malformed\n", lineno); goto parse_error; } /* Read final state */ n = nextline_str2words(fp, &lineno, &lineptr, &wordptr); if ((n != 2) || ((strcmp(wordptr[0], FSG_MODEL_F_DECL) != 0) && (strcmp(wordptr[0], FSG_MODEL_FINAL_STATE_DECL) != 0)) || (sscanf(wordptr[1], "%d", &(fsg->final_state)) != 1) || (fsg->final_state < 0) || (fsg->final_state >= fsg->n_state)) { E_ERROR ("Line[%d]: final state declaration line missing or malformed\n", lineno); goto parse_error; } /* Read transitions */ lastwid = 0; n_trans = n_null_trans = 0; for (;;) { int32 wid, tprob; n = nextline_str2words(fp, &lineno, &lineptr, &wordptr); if (n <= 0) { E_ERROR("Line[%d]: transition or FSG_END statement expected\n", lineno); goto parse_error; } if ((strcmp(wordptr[0], FSG_MODEL_END_DECL) == 0)) { break; } if ((strcmp(wordptr[0], FSG_MODEL_T_DECL) == 0) || (strcmp(wordptr[0], FSG_MODEL_TRANSITION_DECL) == 0)) { if (((n != 4) && (n != 5)) || (sscanf(wordptr[1], "%d", &i) != 1) || (sscanf(wordptr[2], "%d", &j) != 1) || (i < 0) || (i >= fsg->n_state) || (j < 0) || (j >= fsg->n_state)) { E_ERROR ("Line[%d]: transition spec malformed; Expecting: from-state to-state trans-prob [word]\n", lineno); goto parse_error; } p = atof_c(wordptr[3]); if ((p <= 0.0) || (p > 1.0)) { E_ERROR ("Line[%d]: transition spec malformed; Expecting float as transition probability\n", lineno); goto parse_error; } } else { E_ERROR("Line[%d]: transition or FSG_END statement expected\n", lineno); goto parse_error; } tprob = (int32) (logmath_log(lmath, p) * fsg->lw); /* Add word to "dictionary". */ if (n > 4) { if (hash_table_lookup_int32(vocab, wordptr[4], &wid) < 0) { (void) hash_table_enter_int32(vocab, ckd_salloc(wordptr[4]), lastwid); wid = lastwid; ++lastwid; } fsg_model_trans_add(fsg, i, j, tprob, wid); ++n_trans; } else { if (fsg_model_null_trans_add(fsg, i, j, tprob) == 1) { ++n_null_trans; nulls = glist_add_ptr(nulls, fsg_model_null_trans(fsg, i, j)); } } } E_INFO("FSG: %d states, %d unique words, %d transitions (%d null)\n", fsg->n_state, hash_table_inuse(vocab), n_trans, n_null_trans); /* Now create a string table from the "dictionary" */ fsg->n_word = hash_table_inuse(vocab); fsg->n_word_alloc = fsg->n_word + 10; /* Pad it a bit. */ fsg->vocab = ckd_calloc(fsg->n_word_alloc, sizeof(*fsg->vocab)); for (itor = hash_table_iter(vocab); itor; itor = hash_table_iter_next(itor)) { char const *word = hash_entry_key(itor->ent); int32 wid = (int32) (long) hash_entry_val(itor->ent); fsg->vocab[wid] = (char *) word; } hash_table_free(vocab); /* Do transitive closure on null transitions */ nulls = fsg_model_null_trans_closure(fsg, nulls); glist_free(nulls); ckd_free(lineptr); ckd_free(wordptr); return fsg; parse_error: for (itor = hash_table_iter(vocab); itor; itor = hash_table_iter_next(itor)) ckd_free((char *) hash_entry_key(itor->ent)); glist_free(nulls); hash_table_free(vocab); ckd_free(fsgname); ckd_free(lineptr); ckd_free(wordptr); fsg_model_free(fsg); return NULL; }
static int run_control_file(sphinx_wave2feat_t *wtf, char const *ctlfile) { hash_table_t *files; hash_iter_t *itor; lineiter_t *li; FILE *ctlfh; int nskip, runlen, npart, rv = 0; if ((ctlfh = fopen(ctlfile, "r")) == NULL) { E_ERROR_SYSTEM("Failed to open control file %s", ctlfile); return -1; } nskip = cmd_ln_int32_r(wtf->config, "-nskip"); runlen = cmd_ln_int32_r(wtf->config, "-runlen"); if ((npart = cmd_ln_int32_r(wtf->config, "-npart"))) { /* Count lines in the file. */ int partlen, part, nlines = 0; part = cmd_ln_int32_r(wtf->config, "-part"); for (li = lineiter_start(ctlfh); li; li = lineiter_next(li)) ++nlines; fseek(ctlfh, 0, SEEK_SET); partlen = nlines / npart; nskip = partlen * (part - 1); if (part == npart) runlen = -1; else runlen = partlen; } if (runlen != -1){ E_INFO("Processing %d utterances at position %d\n", runlen, nskip); files = hash_table_new(runlen, HASH_CASE_YES); } else { E_INFO("Processing all remaining utterances at position %d\n", nskip); files = hash_table_new(1000, HASH_CASE_YES); } for (li = lineiter_start(ctlfh); li; li = lineiter_next(li)) { char *c, *infile, *outfile; if (nskip-- > 0) continue; if (runlen == 0) { lineiter_free(li); break; } --runlen; string_trim(li->buf, STRING_BOTH); /* Extract the file ID from the control line. */ if ((c = strchr(li->buf, ' ')) != NULL) *c = '\0'; if (strlen(li->buf) == 0) { E_WARN("Empty line %d in control file, skipping\n", li->lineno); continue; } build_filenames(wtf->config, li->buf, &infile, &outfile); if (hash_table_lookup(files, infile, NULL) == 0) continue; rv = sphinx_wave2feat_convert_file(wtf, infile, outfile); hash_table_enter(files, infile, outfile); if (rv != 0) { lineiter_free(li); break; } } for (itor = hash_table_iter(files); itor; itor = hash_table_iter_next(itor)) { ckd_free((void *)hash_entry_key(itor->ent)); ckd_free(hash_entry_val(itor->ent)); } hash_table_free(files); if (fclose(ctlfh) == EOF) E_ERROR_SYSTEM("Failed to close control file"); return rv; }
/* RAH 4.16.01 This code has several leaks that must be fixed */ dict2pid_t *dict2pid_build (mdef_t *mdef, dict_t *dict) { dict2pid_t *dict2pid; s3ssid_t *internal, **ldiph, **rdiph, *single; int32 pronlen; hash_table_t *hs, *hp; glist_t g; gnode_t *gn; s3senid_t *sen; hash_entry_t *he; int32 *cslen; int32 i, j, b, l, r, w, n, p; E_INFO("Building PID tables for dictionary\n"); dict2pid = (dict2pid_t *) ckd_calloc (1, sizeof(dict2pid_t)); dict2pid->internal = (s3ssid_t **) ckd_calloc (dict_size(dict), sizeof(s3ssid_t *)); dict2pid->ldiph_lc = (s3ssid_t ***) ckd_calloc_3d (mdef->n_ciphone, mdef->n_ciphone, mdef->n_ciphone, sizeof(s3ssid_t)); dict2pid->single_lc = (s3ssid_t **) ckd_calloc_2d (mdef->n_ciphone, mdef->n_ciphone, sizeof(s3ssid_t)); dict2pid->n_comstate = 0; dict2pid->n_comsseq = 0; hs = hash_new (mdef->n_ciphone * mdef->n_ciphone * mdef->n_emit_state, HASH_CASE_YES); hp = hash_new (mdef->n_ciphone * mdef->n_ciphone, HASH_CASE_YES); for (w = 0, n = 0; w < dict_size(dict); w++) { pronlen = dict_pronlen(dict, w); if (pronlen < 0) E_FATAL("Pronunciation-length(%s)= %d\n", dict_wordstr(dict, w), pronlen); n += pronlen; } internal = (s3ssid_t *) ckd_calloc (n, sizeof(s3ssid_t)); /* Temporary */ ldiph = (s3ssid_t **) ckd_calloc_2d (mdef->n_ciphone, mdef->n_ciphone, sizeof(s3ssid_t)); rdiph = (s3ssid_t **) ckd_calloc_2d (mdef->n_ciphone, mdef->n_ciphone, sizeof(s3ssid_t)); single = (s3ssid_t *) ckd_calloc (mdef->n_ciphone, sizeof(s3ssid_t)); for (b = 0; b < mdef->n_ciphone; b++) { for (l = 0; l < mdef->n_ciphone; l++) { for (r = 0; r < mdef->n_ciphone; r++) dict2pid->ldiph_lc[b][r][l] = BAD_S3SSID; dict2pid->single_lc[b][l] = BAD_S3SSID; ldiph[b][l] = BAD_S3SSID; rdiph[b][l] = BAD_S3SSID; } single[b] = BAD_S3SSID; } for (w = 0; w < dict_size(dict); w++) { dict2pid->internal[w] = internal; pronlen = dict_pronlen(dict,w); if (pronlen >= 2) { b = dict_pron(dict, w, 0); r = dict_pron(dict, w, 1); if (NOT_S3SSID(ldiph[b][r])) { g = ldiph_comsseq(mdef, b, r); ldiph[b][r] = ssidlist2comsseq (g, mdef, dict2pid, hs, hp); glist_free (g); for (l = 0; l < mdef_n_ciphone(mdef); l++) { p = mdef_phone_id_nearest (mdef, (s3cipid_t)b, (s3cipid_t)l, (s3cipid_t)r, WORD_POSN_BEGIN); dict2pid->ldiph_lc[b][r][l] = mdef_pid2ssid(mdef, p); } } internal[0] = ldiph[b][r]; for (i = 1; i < pronlen-1; i++) { l = b; b = r; r = dict_pron(dict, w, i+1); p = mdef_phone_id_nearest(mdef, (s3cipid_t)b, (s3cipid_t)l, (s3cipid_t)r, WORD_POSN_INTERNAL); internal[i] = mdef_pid2ssid(mdef, p); } l = b; b = r; if (NOT_S3SSID(rdiph[b][l])) { g = rdiph_comsseq(mdef, b, l); rdiph[b][l] = ssidlist2comsseq (g, mdef, dict2pid, hs, hp); glist_free (g); } internal[pronlen-1] = rdiph[b][l]; } else if (pronlen == 1) { b = dict_pron(dict, w, 0); if (NOT_S3SSID(single[b])) { g = single_comsseq(mdef, b); single[b] = ssidlist2comsseq (g, mdef, dict2pid, hs, hp); glist_free (g); for (l = 0; l < mdef_n_ciphone(mdef); l++) { g = single_lc_comsseq(mdef, b, l); dict2pid->single_lc[b][l] = ssidlist2comsseq (g, mdef, dict2pid, hs, hp); glist_free (g); } } internal[0] = single[b]; } internal += pronlen; } ckd_free_2d ((void **) ldiph); ckd_free_2d ((void **) rdiph); ckd_free ((void *) single); /* Allocate space for composite state table */ cslen = (int32 *) ckd_calloc (dict2pid->n_comstate, sizeof(int32)); g = hash_tolist(hs, &n); assert (n == dict2pid->n_comstate); n = 0; for (gn = g; gn; gn = gnode_next(gn)) { he = (hash_entry_t *) gnode_ptr (gn); sen = (s3senid_t *) hash_entry_key(he); for (i = 0; IS_S3SENID(sen[i]); i++); cslen[hash_entry_val(he)] = i+1; /* +1 for terminating sentinel */ n += (i+1); } dict2pid->comstate = (s3senid_t **) ckd_calloc (dict2pid->n_comstate, sizeof(s3senid_t *)); sen = (s3senid_t *) ckd_calloc (n, sizeof(s3senid_t)); for (i = 0; i < dict2pid->n_comstate; i++) { dict2pid->comstate[i] = sen; sen += cslen[i]; } /* Build composite state table from hash table hs */ for (gn = g; gn; gn = gnode_next(gn)) { he = (hash_entry_t *) gnode_ptr (gn); sen = (s3senid_t *) hash_entry_key(he); i = hash_entry_val(he); for (j = 0; j < cslen[i]; j++) dict2pid->comstate[i][j] = sen[j]; assert (sen[j-1] == BAD_S3SENID); ckd_free ((void *)sen); } ckd_free (cslen); glist_free (g); hash_free (hs); /* Allocate space for composite sseq table */ dict2pid->comsseq = (s3senid_t **) ckd_calloc (dict2pid->n_comsseq, sizeof(s3senid_t *)); g = hash_tolist (hp, &n); assert (n == dict2pid->n_comsseq); /* Build composite sseq table */ for (gn = g; gn; gn = gnode_next(gn)) { he = (hash_entry_t *) gnode_ptr (gn); i = hash_entry_val(he); dict2pid->comsseq[i] = (s3senid_t *) hash_entry_key(he); } glist_free (g); hash_free (hp); /* Weight for each composite state */ dict2pid->comwt = (int32 *) ckd_calloc (dict2pid->n_comstate, sizeof(int32)); for (i = 0; i < dict2pid->n_comstate; i++) { sen = dict2pid->comstate[i]; for (j = 0; IS_S3SENID(sen[j]); j++); #if 0 /* if comstate i has N states, its weight= (1/N^2) (Major Hack!!) */ dict2pid->comwt[i] = - (logs3 ((float64)j) << 1); #else /* if comstate i has N states, its weight= 1/N */ dict2pid->comwt[i] = - logs3 ((float64)j); #endif } E_INFO("%d composite states; %d composite sseq\n", dict2pid->n_comstate, dict2pid->n_comsseq); return dict2pid; }