static int32 parse_base_line(char *line, int lineno, acmod_id_t *acmod_id, uint32 *tmat, uint32 *state, uint32 *n_state, acmod_set_t *acmod_set) { char *tok; unsigned int n; const char **attrib; tok = strtok(line, " \t"); if (acmod_set_name2id(acmod_set, tok) != NO_ACMOD) { E_ERROR("%s multiply defined at line %d\n", tok, lineno); return S3_ERROR; } for (n = 0; (n < 3) && (strcmp("-", strtok(NULL, " \t")) == 0); n++); if (n < 3) { E_ERROR("Expected l, r and posn to be \"-\" for ci phone %s at line %d, currently '%s'\n", tok, lineno, line); return S3_ERROR; } if (parse_rem(&attrib, tmat, state, n_state, lineno) != S3_SUCCESS) { return S3_ERROR; } *acmod_id = acmod_set_add_ci(acmod_set, tok, attrib); return S3_SUCCESS; }
state_t *next_utt_states(uint32 *n_state, lexicon_t *lex, model_inventory_t *inv, model_def_t *mdef, char *trans, int32 sil_del, char* silence_str ) { char **word; uint32 n_word; uint32 n_phone; char *btw_mark; acmod_set_t *acmod_set; acmod_id_t *phone; acmod_id_t optSil; state_t *state_seq; word = mk_wordlist(trans, &n_word); phone = mk_phone_list(&btw_mark, &n_phone, word, n_word, lex); if (phone == NULL) { E_WARN("Unable to produce CI phones for utt\n"); ckd_free(word); return NULL; } acmod_set = inv->acmod_set; #ifdef NEXT_UTT_STATES_VERBOSE print_phone_list(phone, n_phone, btw_mark, acmod_set); #endif cvt2triphone(acmod_set, phone, btw_mark, n_phone); #ifdef NEXT_UTT_STATES_VERBOSE print_phone_list(phone, n_phone, btw_mark, acmod_set); #endif optSil= acmod_set_name2id(acmod_set, silence_str); /* * Debug? * E_INFO("Silence id %d\n",optSil); */ state_seq = state_seq_make(n_state, phone, n_phone, inv, mdef,sil_del,(acmod_id_t)optSil); #ifdef NEXT_UTT_STATES_VERBOSE state_seq_print(state_seq, *n_state, mdef); #endif ckd_free(phone); ckd_free(btw_mark); ckd_free(word); return state_seq; }
static int32 parse_base_line(acmod_id_t *acmod_id, uint32 *tmat, uint32 *state, uint32 *n_state, acmod_set_t *acmod_set, uint32 *n_read, FILE *fp) { char buf[BIG_STR_LEN]; char *tok; unsigned int n; const char **attrib; if (read_line(buf, BIG_STR_LEN, n_read, fp) == NULL) return S3_ERROR; tok = strtok(buf, " \t"); if (acmod_set_name2id(acmod_set, tok) != NO_ACMOD) { E_ERROR("%s multiply defined at line %d\n", tok, *n_read); return S3_ERROR; } for (n = 0; (n < 3) && (strcmp("-", strtok(NULL, " \t")) == 0); n++); if (n < 3) { fflush(stdout); fprintf(stderr, "%s(%d): expected l, r and posn to be \"-\" for ci phone %s at line %d\n", __FILE__, __LINE__, tok, *n_read); fflush(stderr); return S3_ERROR; } if (parse_rem(&attrib, tmat, state, n_state, *n_read) != S3_SUCCESS) { return S3_ERROR; } *acmod_id = acmod_set_add_ci(acmod_set, tok, attrib); return S3_SUCCESS; }
static int add_phones(uint32 n_phone, lex_entry_t *e, acmod_set_t *acmod_set) { uint32 i; char *nxt_phone; e->phone = ckd_calloc(n_phone, sizeof(char *)); e->ci_acmod_id = ckd_calloc(n_phone, sizeof(uint32)); e->phone_cnt = n_phone; for (i = 0; (nxt_phone = strtok(NULL, " \t")); i++) { e->phone[i] = nxt_phone; e->ci_acmod_id[i] = acmod_set_name2id(acmod_set, nxt_phone); if (e->ci_acmod_id[i] == NO_ACMOD) { E_ERROR("Unknown phone %s\n", nxt_phone); ckd_free(e->phone); e->phone = NULL; ckd_free(e->ci_acmod_id); e->ci_acmod_id = NULL; e->phone_cnt = 0; return S3_ERROR; } } assert(i == n_phone); return S3_SUCCESS; }
uint32 setup_d2o_map(model_def_t *d_mdef, model_def_t *o_mdef) { model_def_entry_t *o_defn, *d_defn; uint32 d_ts; uint32 o_ts; uint32 *mapped; uint32 i, j, k, d; const char *nm; int did_warn = FALSE; if (d_mdef->n_tied_state < o_mdef-> n_tied_state) { E_FATAL("more tied states in output than in dump mdef (%u vs %u)\n", o_mdef->n_tied_state, d_mdef->n_tied_state); } if (d_mdef->n_tied_ci_state != o_mdef->n_tied_ci_state) { E_FATAL("# tied ci state in output, %u not equal to # in dmp, %u\n", o_mdef->n_tied_ci_state, d_mdef->n_tied_ci_state); } n_o2d = (uint32 *)ckd_calloc(o_mdef->n_tied_state, sizeof(uint32)); i_o2d = (uint32 *)ckd_calloc(o_mdef->n_tied_state, sizeof(uint32)); o2d = (uint32 **)ckd_calloc(o_mdef->n_tied_state, sizeof(uint32 *)); mapped = (uint32 *)ckd_calloc(d_mdef->n_tied_state, sizeof(uint32)); for (i = 0; i < o_mdef->n_defn; i++) { nm = acmod_set_id2name(o_mdef->acmod_set, i); d = acmod_set_name2id(d_mdef->acmod_set, nm); if (d == NO_ID) { if (!did_warn) { E_WARN("Some models in the output mdef not in the dump mdef\n"); did_warn = TRUE; } continue; } o_defn = &o_mdef->defn[i]; d_defn = &d_mdef->defn[d]; for (j = 0; j < o_defn->n_state; j++) { o_ts = o_defn->state[j]; d_ts = d_defn->state[j]; if ((o_ts != TYING_NO_ID) && (o_ts != TYING_NO_ID)) { if (mapped[d_ts] == FALSE) { ++n_o2d[o_ts]; mapped[d_ts] = TRUE; } } else { if (!((o_ts == TYING_NO_ID) && (o_ts == TYING_NO_ID))) { E_INFO("%s state is NULL but %s isn't.\n", (o_ts == TYING_NO_ID ? "output" : "dump"), (o_ts == TYING_NO_ID ? "dump" : "output")); } } } } for (i = 0; i < o_mdef->n_tied_state; i++) { o2d[i] = (uint32 *)ckd_calloc(n_o2d[i], sizeof(uint32)); } for (i = 0; i < o_mdef->n_defn; i++) { /* Figure out the index in the dump mdef for the model in the output mdef */ nm = acmod_set_id2name(o_mdef->acmod_set, i); d = acmod_set_name2id(d_mdef->acmod_set, nm); if (d == NO_ID) continue; o_defn = &o_mdef->defn[i]; d_defn = &d_mdef->defn[d]; for (j = 0; j < o_defn->n_state; j++) { o_ts = o_defn->state[j]; d_ts = d_defn->state[j]; if ((o_ts != TYING_NO_ID) && (o_ts != TYING_NO_ID)) { for (k = 0; k < i_o2d[o_ts]; k++) { if (o2d[o_ts][k] == d_ts) break; } if (k == i_o2d[o_ts]) { o2d[o_ts][i_o2d[o_ts]++] = d_ts; } } else { if (!((o_ts == TYING_NO_ID) && (o_ts == TYING_NO_ID))) { E_INFO("%s state is NULL but %s isn't.\n", (o_ts == TYING_NO_ID ? "output" : "dump"), (o_ts == TYING_NO_ID ? "dump" : "output")); } } } } for (i = 0; i < o_mdef->n_tied_state; i++) { if (i_o2d[i] != n_o2d[i]) { E_FATAL("%u != %u for %u\n", i_o2d[i], n_o2d[i], i); } } for (i = 0; i < o_mdef->n_tied_state; i++) { i_o2d[i] = 0; } return S3_SUCCESS; }
static int32 parse_tri_line(acmod_id_t *acmod_id, uint32 *tmat, uint32 *state, uint32 *n_state, acmod_set_t *acmod_set, uint32 *n_read, FILE *fp) { char buf[BIG_STR_LEN]; char *tok; unsigned int i; uint32 id; char *posn_map = WORD_POSN_CHAR_MAP; const char **attrib; acmod_id_t base, left, right; word_posn_t posn; if (read_line(buf, BIG_STR_LEN, n_read, fp) == NULL) return S3_ERROR; tok = strtok(buf, " \t"); if ((id = acmod_set_name2id(acmod_set, tok)) == NO_ACMOD) { E_ERROR("%s is an undefined base phone at line %d\n", tok, *n_read); return S3_ERROR; } base = id; tok = strtok(NULL, " \t"); if ((id = acmod_set_name2id(acmod_set, tok)) == NO_ACMOD) { E_ERROR("%s is an undefined base phone at line %d\n", tok, *n_read); return S3_ERROR; } left = id; tok = strtok(NULL, " \t"); if ((id = acmod_set_name2id(acmod_set, tok)) == NO_ACMOD) { E_ERROR("%s is an undefined base phone at line %d\n", tok, *n_read); return S3_ERROR; } right = id; tok = strtok(NULL, " \t"); for (i = 0; i < strlen(posn_map); i++) if (tok[0] == posn_map[i]) break; if (i < strlen(posn_map)) { posn = (word_posn_t)i; } else { E_ERROR("Unknown word posn %s found at line %d\n", tok, *n_read); return S3_ERROR; } if (parse_rem(&attrib, tmat, state, n_state, *n_read) != S3_SUCCESS) { return S3_ERROR; } *acmod_id = acmod_set_add_tri(acmod_set, base, left, right, posn, attrib); if (*acmod_id == NO_ACMOD) { return S3_ERROR; } return S3_SUCCESS; }
static int init_mixw() { model_def_t *src_mdef; float32 ***src_mixw; vector_t ***src_mean; vector_t ***src_var = NULL; vector_t ****src_fullvar = NULL; float32 ***src_tmat; model_def_t *dest_mdef; float32 ***dest_mixw; vector_t ***dest_mean; vector_t ***dest_var = NULL; vector_t ****dest_fullvar = NULL; float32 ***dest_tmat; uint32 n_mixw_src; uint32 n_mixw_dest; uint32 n_feat; uint32 tmp_n_feat; uint32 n_gau; uint32 tmp_n_gau; uint32 n_cb_src; uint32 n_cb_dest; uint32 n_state_pm; uint32 n_tmat_src; uint32 n_tmat_dest; uint32 *veclen; uint32 *tmp_veclen; uint32 m; uint32 dest_m; uint32 dest_m_base; uint32 src_m; acmod_id_t src_m_base; const char *dest_m_name; const char *dest_m_base_name; uint32 i; uint32 n_ts; uint32 n_cb; const char *ts2cbfn; E_INFO("Reading src %s\n", cmd_ln_str("-src_moddeffn")); /* read in the source model definition file */ if (model_def_read(&src_mdef, cmd_ln_str("-src_moddeffn")) != S3_SUCCESS) { return S3_ERROR; } ts2cbfn = cmd_ln_str("-src_ts2cbfn"); if (strcmp(SEMI_LABEL, ts2cbfn) == 0) { E_INFO("Generating semi-continous ts2cb mapping\n"); src_mdef->cb = semi_ts2cb(src_mdef->n_tied_state); n_ts = src_mdef->n_tied_state; n_cb = 1; } else if (strcmp(CONT_LABEL, ts2cbfn) == 0) { E_INFO("Generating continous ts2cb mapping\n"); src_mdef->cb = cont_ts2cb(src_mdef->n_tied_state); n_ts = src_mdef->n_tied_state; n_cb = src_mdef->n_tied_state; } else if (strcmp(PTM_LABEL, ts2cbfn) == 0) { E_INFO("Generating phonetically tied ts2cb mapping\n"); src_mdef->cb = ptm_ts2cb(src_mdef); n_ts = src_mdef->n_tied_state; n_cb = src_mdef->acmod_set->n_ci; } else { E_INFO("Reading src %s\n", cmd_ln_str("-src_ts2cbfn")); if (s3ts2cb_read(ts2cbfn, &src_mdef->cb, &n_ts, &n_cb) != S3_SUCCESS) { return S3_ERROR; } } E_INFO("Reading src %s\n", cmd_ln_str("-src_mixwfn")); /* read in the source mixing weight parameter file */ if (s3mixw_read(cmd_ln_str("-src_mixwfn"), &src_mixw, &n_mixw_src, &n_feat, &n_gau) != S3_SUCCESS) { return S3_ERROR; } E_INFO("Reading src %s\n", cmd_ln_str("-src_tmatfn")); if (s3tmat_read(cmd_ln_str("-src_tmatfn"), &src_tmat, &n_tmat_src, &n_state_pm) != S3_SUCCESS) { return S3_ERROR; } E_INFO("Reading src %s\n", cmd_ln_str("-src_meanfn")); if (s3gau_read(cmd_ln_str("-src_meanfn"), &src_mean, &n_cb_src, &tmp_n_feat, &tmp_n_gau, &veclen) != S3_SUCCESS) { return S3_ERROR; } if (tmp_n_feat != n_feat) { E_FATAL("src mean n_feat (== %u) != prior value (== %u)\n", tmp_n_feat, n_feat); } if (tmp_n_gau != n_gau) { E_FATAL("src mean n_gau (== %u) != prior value (== %u)\n", tmp_n_gau, n_gau); } if (n_cb_src != n_cb) { E_FATAL("src mean n_cb (== %u) is inconsistent with ts2cb mapping %u. Most probably phoneset has duplicated phones\n", n_cb_src, n_cb); } E_INFO("Reading src %s\n", cmd_ln_str("-src_varfn")); if (cmd_ln_int32("-fullvar")) { if (s3gau_read_full(cmd_ln_str("-src_varfn"), &src_fullvar, &n_cb_src, &tmp_n_feat, &tmp_n_gau, &tmp_veclen) != S3_SUCCESS) { return S3_ERROR; } } else { if (s3gau_read(cmd_ln_str("-src_varfn"), &src_var, &n_cb_src, &tmp_n_feat, &tmp_n_gau, &tmp_veclen) != S3_SUCCESS) { return S3_ERROR; } } if (tmp_n_feat != n_feat) { E_FATAL("src var n_feat (== %u) != prior value (== %u)\n", tmp_n_feat, n_feat); } if (tmp_n_gau != n_gau) { E_FATAL("src var n_gau (== %u) != prior value (== %u)\n", tmp_n_gau, n_gau); } if (n_cb_src != n_cb) { E_FATAL("src var n_cb (== %u) inconsistent w/ ts2cb mapping %u\n", n_cb_src, n_cb); } if (n_mixw_src < src_mdef->n_tied_state) { E_FATAL("Too few source mixing weights, %u, for the # of tied states, %u\n", n_mixw_src, src_mdef->n_tied_state); } for (i = 0; i < n_feat; i++) { if (veclen[i] != tmp_veclen[i]) { E_FATAL("src var veclen[%u] (== %u) != prior value (== %u)\n", i, tmp_veclen[i], veclen[i]); } } ckd_free(tmp_veclen); E_INFO("Reading dest %s\n", cmd_ln_str("-dest_moddeffn")); /* read in the destination model definition file */ if (model_def_read(&dest_mdef, cmd_ln_str("-dest_moddeffn")) < S3_SUCCESS) { return S3_ERROR; } ts2cbfn = cmd_ln_str("-dest_ts2cbfn"); if (strcmp(SEMI_LABEL, ts2cbfn) == 0) { E_INFO("Generating semi-continous ts2cb mapping\n"); dest_mdef->cb = semi_ts2cb(dest_mdef->n_tied_state); n_ts = dest_mdef->n_tied_state; n_cb = 1; } else if (strcmp(CONT_LABEL, ts2cbfn) == 0) { E_INFO("Generating continous ts2cb mapping\n"); dest_mdef->cb = cont_ts2cb(dest_mdef->n_tied_state); n_ts = dest_mdef->n_tied_state; n_cb = dest_mdef->n_tied_state; } else if (strcmp(PTM_LABEL, ts2cbfn) == 0) { E_INFO("Generating phonetically tied ts2cb mapping\n"); dest_mdef->cb = ptm_ts2cb(dest_mdef); n_ts = dest_mdef->n_tied_state; n_cb = dest_mdef->acmod_set->n_ci; } else { E_INFO("Reading dest %s\n", cmd_ln_str("-dest_ts2cbfn")); if (s3ts2cb_read(ts2cbfn, &dest_mdef->cb, &n_ts, &n_cb) != S3_SUCCESS) { return S3_ERROR; } } E_INFO("Calculating initial model parameters\n"); n_tmat_dest = dest_mdef->n_tied_tmat; tmat_dest_list = init_was_added(n_tmat_dest); E_INFO("Alloc %ux%ux%u dest tmat\n", n_tmat_dest, n_state_pm-1, n_state_pm); dest_tmat = (float32 ***)ckd_calloc_3d(n_tmat_dest, n_state_pm-1, n_state_pm, sizeof(float32)); n_mixw_dest = dest_mdef->n_tied_state; mixw_dest_list = init_was_added(n_mixw_dest); E_INFO("Alloc %ux%ux%u dest mixw\n", n_mixw_dest, n_feat, n_gau); dest_mixw = (float32 ***)ckd_calloc_3d(n_mixw_dest, n_feat, n_gau, sizeof(float32)); for (i = 0, n_cb_dest = 0; i < n_mixw_dest; i++) { if (dest_mdef->cb[i] > n_cb_dest) { n_cb_dest = dest_mdef->cb[i]; } } ++n_cb_dest; cb_dest_list = init_was_added(n_cb_dest); E_INFO("Alloc %ux%ux%u dest mean and var\n", n_cb_dest, n_feat, n_gau); dest_mean = gauden_alloc_param(n_cb_dest, n_feat, n_gau, veclen); if (src_var) dest_var = gauden_alloc_param(n_cb_dest, n_feat, n_gau, veclen); else if (src_fullvar) dest_fullvar = gauden_alloc_param_full(n_cb_dest, n_feat, n_gau, veclen); for (dest_m = 0; dest_m < dest_mdef->n_defn; dest_m++) { dest_m_name = acmod_set_id2name(dest_mdef->acmod_set, dest_m); src_m = acmod_set_name2id(src_mdef->acmod_set, dest_m_name); if (src_m == NO_ACMOD) { /* No corresponding phone model in the source set */ /* See if there is a source base phone corresponding to this destination model base phone */ dest_m_base = acmod_set_base_phone(dest_mdef->acmod_set, dest_m); dest_m_base_name = acmod_set_id2name(dest_mdef->acmod_set, dest_m_base); src_m_base = acmod_set_name2id(src_mdef->acmod_set, dest_m_base_name); if (src_m_base == NO_ACMOD) { /* No corresponding model or base model found. Use uniform distribution */ E_INFO("No source base phone %s found. Initializing %s using uniform distribution\n", dest_m_base_name, dest_m_name); if (src_tmat) { E_INFO("Uniform initialization of tmat not supported\n"); } init_uniform(dest_mixw, &dest_mdef->defn[dest_m], n_feat, n_gau); } else { /* No corresponding model, but a base model was found. Use base distribution. */ init_model(dest_mixw, dest_mean, dest_var, dest_fullvar, dest_tmat, &dest_mdef->defn[dest_m], dest_mdef->cb, dest_mdef->acmod_set, src_mixw, src_mean, src_var, src_fullvar, src_tmat, &src_mdef->defn[src_m_base], src_mdef->cb, src_mdef->acmod_set, n_feat, n_gau, n_state_pm, veclen); } } else { /* Found a corresponding model in the source set, so use source distributions to init the destination */ init_model(dest_mixw, dest_mean, dest_var, dest_fullvar, dest_tmat, &dest_mdef->defn[dest_m], dest_mdef->cb, dest_mdef->acmod_set, src_mixw, src_mean, src_var, src_fullvar, src_tmat, &src_mdef->defn[src_m], src_mdef->cb, src_mdef->acmod_set, n_feat, n_gau, n_state_pm, veclen); } } for (m = 0; m < n_mixw_dest; m++) { if (mixw_dest_list[m] == NULL) { E_WARN("Destination state %u has not been initialized!\n", m); } } for (m = 0; m < n_cb_dest; m++) { if (cb_dest_list[m] == NULL) { E_WARN("Destination cb %u has not been initialized!\n", m); } else if (cb_dest_list[m]->next != NULL) { E_WARN("dest cb %u has > 1 corresponding source cb\n", m); } } E_INFO("Writing dest %s\n", cmd_ln_str("-dest_tmatfn")); if (s3tmat_write(cmd_ln_str("-dest_tmatfn"), dest_tmat, n_tmat_dest, n_state_pm) != S3_SUCCESS) { return S3_ERROR; } E_INFO("Writing dest %s\n", cmd_ln_str("-dest_mixwfn")); if (s3mixw_write(cmd_ln_str("-dest_mixwfn"), dest_mixw, dest_mdef->n_tied_state, n_feat, n_gau) < S3_SUCCESS) { return S3_ERROR; } E_INFO("Writing dest %s\n", cmd_ln_str("-dest_meanfn")); if (s3gau_write(cmd_ln_str("-dest_meanfn"), (const vector_t ***)dest_mean, n_cb_dest, n_feat, n_gau, veclen) != S3_SUCCESS) { return S3_ERROR; } E_INFO("Writing dest %s\n", cmd_ln_str("-dest_varfn")); if (cmd_ln_int32("-fullvar")) { if (s3gau_write_full(cmd_ln_str("-dest_varfn"), (const vector_t ****)dest_fullvar, n_cb_dest, n_feat, n_gau, veclen) != S3_SUCCESS) { return S3_ERROR; } } else { if (s3gau_write(cmd_ln_str("-dest_varfn"), (const vector_t ***)dest_var, n_cb_dest, n_feat, n_gau, veclen) != S3_SUCCESS) { return S3_ERROR; } } ckd_free(veclen); return S3_SUCCESS; }
int s3phseg_read(const char *fn, acmod_set_t *acmod_set, s3phseg_t **out_phseg) { FILE *fp; char txt[512]; s3phseg_t *plist = NULL; int n; if ((fp = fopen(fn, "r")) == NULL) { E_ERROR("Failed to open phseg file %s\n", fn); return S3_ERROR; } /* Should be a header of column names */ while ((n = fscanf(fp, "%511s", txt))) { if (n == EOF) { E_ERROR("Failed to read column headers from phseg file\n"); goto error_out; } if (!strcmp(txt, "Phone")) break; } /* Get each line */ while (!feof(fp)) { unsigned int sf, ef; int score; acmod_id_t ci, phone; s3phseg_t *phseg; char *c, *cc; n = fscanf(fp, "%u %u %d", &sf, &ef, &score); if (n < 3) /* We probably hit EOF or "Total score" */ break; fgets(txt, sizeof(txt), fp); /* Remove newline. */ if (txt[strlen(txt)-1] == '\n') txt[strlen(txt)-1] = '\0'; /* Find the base phone. */ cc = txt + strspn(txt, " \t"); if ((c = strchr(cc, ' '))) *c = '\0'; if ((ci = phone = acmod_set_name2id(acmod_set, txt)) == NO_ACMOD) { E_ERROR("Unknown CI phone (%s) in phseg file\n", txt); goto error_out; } /* Restore the space and find the triphone if necessary. */ if (c) *c = ' '; if (acmod_set->n_multi != 0 && !acmod_set_has_attrib(acmod_set, phone, "filler")) { if ((phone = acmod_set_name2id(acmod_set, txt)) == NO_ACMOD) { /* This might be too verbose. */ E_WARN("Unknown triphone (%s) in phseg file\n", txt); /* Back off to CI phone. */ phone = ci; } } phseg = ckd_calloc(1, sizeof(*phseg)); phseg->next = plist; phseg->phone = phone; phseg->sf = sf; phseg->ef = ef; phseg->score = score; plist = phseg; } fclose(fp); if (out_phseg) { s3phseg_t *next, *last = NULL; /* Now reverse the list. */ while (plist) { next = plist->next; *out_phseg = plist; (*out_phseg)->next = last; last = *out_phseg; plist = next; } } else s3phseg_free(plist); return S3_SUCCESS; error_out: fclose(fp); return S3_ERROR; }