/**
* @brief Convert whole words in word dictionary to word-internal triphone.
*
* Normally triphone conversion will be performed directly when reading
* dictionary file. This function is for post conversion only.
*
* @param winfo [i/o] word dictionary information
* @param hmminfo [in] HTK %HMM definition
*
* @return TRUE on success, FALSE on failure.
*/
boolean
voca_mono2tri(WORD_INFO *winfo, HTK_HMM_INFO *hmminfo)
{
WORD_ID w;
int ph;
char *p;
HMM_Logical *tmplg;
boolean ok_flag = TRUE;
for (w=0; w<winfo->num; w++) {
cycle_triphone(NULL);
cycle_triphone(winfo->wseq[w][0]->name);
for (ph = 0; ph < winfo->wlen[w] ; ph++) {
if (ph == winfo->wlen[w] - 1) {
p = cycle_triphone_flush();
} else {
p = cycle_triphone(winfo->wseq[w][ph + 1]->name);
}
if ((tmplg = htk_hmmdata_lookup_logical(hmminfo, p)) == NULL) {
jlog("Error: voca_load_htkdict: word \"%s[%s]\"(id=%d): HMM \"%s\" not found\n", winfo->wname[w], winfo->woutput[w], w, p);
ok_flag = FALSE;
continue;
}
winfo->wseq[w][ph] = tmplg;
}
}
return (ok_flag);
}
/**
* @brief Search for left context %HMM in logical %HMM
*
* The name of a new left context %HMM, given base phone %HMM and a
* left context phone string, will be generated, and search it in
* the list of logical %HMM.
* If found, return the pointer to the logical %HMM.
*
* @param base [in] base phone %HMM
* @param lc_name [in] left context phone name (allow context-dependent name)
* @param hmminfo [in] HTK %HMM definition data
*
* @return the pointer to the logical %HMM, or NULL if not found.
*/
HMM_Logical *
get_left_context_HMM(HMM_Logical *base, char *lc_name, HTK_HMM_INFO *hmminfo)
{
strcpy(gbuf, base->name);
add_left_context(gbuf, lc_name);
return(htk_hmmdata_lookup_logical(hmminfo, gbuf));
}
/**
* <JA>
* 指定された論理名のHMMを検索し，その情報を出力する.
*
* @param name [in] 論理HMMの名前
* @param hmminfo [in] HMM定義
* </JA>
* <EN>
* Lookup an HMM of given name, and display specs of it.
*
* @param name [in] HMM logical name
* @param hmminfo [in] HMM definition
* </EN>
*/
static void
print_hmminfo(char *name, HTK_HMM_INFO *hmminfo)
{
HMM_Logical *l;
l = htk_hmmdata_lookup_logical(hmminfo, name);
if (l == NULL) {
printf("no HMM named \"%s\"\n", name);
} else {
put_logical_hmm(stdout, l);
}
}
/**
* Sub function to Add a dictionary entry line to the word dictionary.
*
* @param buf [i/o] buffer to hold the input string, will be modified in this function
* @param vnum_p [in] current number of words in @a winfo
* @param linenum [in] current line number of the input
* @param winfo [out] pointer to word dictionary to append the data.
* @param hmminfo [in] HTK %HMM definition data. if NULL, phonemes are ignored.
* @param do_conv [in] TRUE if performing triphone conversion
* @param ok_flag [out] will be set to FALSE if an error occured for this input.
*
* @return FALSE if buf == "DICEND", else TRUE will be returned.
*/
boolean
voca_load_htkdict_line(char *buf, WORD_ID *vnum_p, int linenum, WORD_INFO *winfo, HTK_HMM_INFO *hmminfo, boolean do_conv, boolean *ok_flag)
{
char *ptmp, *lp = NULL, *p;
static char cbuf[MAX_HMMNAME_LEN];
HMM_Logical **tmpwseq;
int len;
HMM_Logical *tmplg;
boolean pok;
int vnum;
vnum = *vnum_p;
if (strmatch(buf, "DICEND")) return FALSE;
/* allocate temporal work area for the first call */
if (winfo->work == NULL) {
winfo->work_num = PHONEMELEN_STEP;
winfo->work = (void *)mybmalloc2(sizeof(HMM_Logical *) * winfo->work_num, &(winfo->mroot));
}
tmpwseq = (HMM_Logical **)winfo->work;
/* backup whole line for debug output */
strcpy(bufbak, buf);
/* GrammarEntry */
if ((ptmp = mystrtok_quote(buf, " \t\n")) == NULL) {
jlog("Error: voca_load_htkdict: line %d: corrupted data:\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
winfo->wname[vnum] = strcpy((char *)mybmalloc2(strlen(ptmp)+1, &(winfo->mroot)), ptmp);
/* just move pointer to next token */
if ((ptmp = mystrtok_movetonext(NULL, " \t\n")) == NULL) {
jlog("Error: voca_load_htkdict: line %d: corrupted data:\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
#ifdef CLASS_NGRAM
winfo->cprob[vnum] = 0.0; /* prob = 1.0, logprob = 0.0 */
#endif
if (ptmp[0] == '@') { /* class N-gram prob */
#ifdef CLASS_NGRAM
/* word probability within the class (for class N-gram) */
/* format: classname @classprob wordname [output] phoneseq */
/* classname equals to wname, and wordname will be omitted */
/* format: @%f (log scale) */
/* if "@" not found or "@0", it means class == word */
if ((ptmp = mystrtok(NULL, " \t\n")) == NULL) {
jlog("Error: voca_load_htkdict: line %d: corrupted data:\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
if (ptmp[1] == '\0') { /* space between '@' and figures */
jlog("Error: voca_load_htkdict: line %d: value after '@' missing, maybe wrong space?\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
winfo->cprob[vnum] = atof(&(ptmp[1]));
if (winfo->cprob[vnum] != 0.0) winfo->cwnum++;
/* read next word entry (just skip them) */
if ((ptmp = mystrtok(NULL, " \t\n")) == NULL) {
jlog("Error: voca_load_htkdict: line %d: corrupted data:\n> %s\n", linenum,bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
/* move to the next word entry */
if ((ptmp = mystrtok_movetonext(NULL, " \t\n")) == NULL) {
jlog("Error: voca_load_htkdict: line %d: corrupted data:\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
#else /* ~CLASS_NGRAM */
jlog("Error: voca_load_htkdict: line %d: cannot handle in-class word probability\n> %s\n", linenum, ptmp, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
#endif /* CLASS_NGRAM */
}
/* OutputString */
switch(ptmp[0]) {
case '[': /* not transparent word */
winfo->is_transparent[vnum] = FALSE;
ptmp = mystrtok_quotation(NULL, " \t\n", '[', ']', 0);
break;
case '{': /* transparent word */
winfo->is_transparent[vnum] = TRUE;
ptmp = mystrtok_quotation(NULL, " \t\n", '{', '}', 0);
break;
default:
#if 1
/* ALLOW no entry for output */
/* same as wname is used */
winfo->is_transparent[vnum] = FALSE;
ptmp = winfo->wname[vnum];
#else
/* error */
jlog("Error: voca_load_htkdict: line %d: missing output string??\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
#endif
}
if (ptmp == NULL) {
jlog("Error: voca_load_htkdict: line %d: corrupted data:\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
winfo->woutput[vnum] = strcpy((char *)mybmalloc2(strlen(ptmp)+1, &(winfo->mroot)), ptmp);
/* phoneme sequence */
if (hmminfo == NULL) {
/* don't read */
winfo->wseq[vnum] = NULL;
winfo->wlen[vnum] = 0;
} else {
/* store converted phone sequence to temporal bufffer */
len = 0;
if (do_conv) {
/* convert phoneme to triphone expression (word-internal) */
cycle_triphone(NULL);
if ((lp = mystrtok(NULL, " \t\n")) == NULL) {
jlog("Error: voca_load_htkdict: line %d: word %s has no phoneme:\n> %s\n", linenum, winfo->wname[vnum], bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
if (strlen(lp) >= MAX_HMMNAME_LEN) {
jlog("Error: voca_load_htkdict: line %d: too long phone name: %s\n", linenum, lp);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
cycle_triphone(lp);
}
pok = TRUE;
for (;;) {
if (do_conv) {
/* if (lp != NULL) jlog(" %d%s",len,lp);*/
if (lp != NULL) lp = mystrtok(NULL, " \t\n");
if (lp != NULL) {
if (strlen(lp) >= MAX_HMMNAME_LEN) {
jlog("Error: voca_load_htkdict: line %d: too long phone name: %s\n", linenum, lp);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
p = cycle_triphone(lp);
}
else p = cycle_triphone_flush();
} else {
p = mystrtok(NULL, " \t\n");
}
if (p == NULL) break;
/* both defined/pseudo phone is allowed */
tmplg = htk_hmmdata_lookup_logical(hmminfo, p);
if (tmplg == NULL) {
/* not found */
if (do_conv) {
/* both defined or pseudo phone are not found */
if (len == 0 && lp == NULL) {
jlog("Error: voca_load_htkdict: line %d: triphone \"*-%s+*\" or monophone \"%s\" not found\n", linenum, p, p);
snprintf(cbuf,MAX_HMMNAME_LEN,"*-%s+* or monophone %s", p, p);
} else if (len == 0) {
jlog("Error: voca_load_htkdict: line %d: triphone \"*-%s\" or biphone \"%s\" not found\n", linenum, p, p);
snprintf(cbuf,MAX_HMMNAME_LEN,"*-%s or biphone %s", p, p);
} else if (lp == NULL) {
jlog("Error: voca_load_htkdict: line %d: triphone \"%s+*\" or biphone \"%s\" not found\n", linenum, p, p);
snprintf(cbuf,MAX_HMMNAME_LEN,"%s+* or biphone %s", p, p);
} else {
jlog("Error: voca_load_htkdict: line %d: triphone \"%s\" not found\n", linenum, p);
snprintf(cbuf,MAX_HMMNAME_LEN,"%s", p);
}
} else {
jlog("Error: voca_load_htkdict: line %d: phone \"%s\" not found\n", linenum, p);
snprintf(cbuf, MAX_HMMNAME_LEN, "%s", p);
}
add_to_error(winfo, cbuf);
pok = FALSE;
} else {
/* found */
if (len >= winfo->work_num) {
/* expand wseq area by PHONEMELEN_STEP */
winfo->work_num += PHONEMELEN_STEP;
winfo->work = (void *)mybmalloc2(sizeof(HMM_Logical *) * winfo->work_num, &(winfo->mroot));
memcpy(winfo->work, tmpwseq, sizeof(HMM_Logical *) * (winfo->work_num - PHONEMELEN_STEP));
tmpwseq = (HMM_Logical **)winfo->work;
}
/* store to temporal buffer */
tmpwseq[len] = tmplg;
}
len++;
}
if (!pok) { /* error in phoneme */
jlog("Error: voca_load_htkdict: the line content was: %s\n", bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
if (len == 0) {
jlog("Error: voca_load_htkdict: line %d: no phone specified:\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
/* store to winfo */
winfo->wseq[vnum] = (HMM_Logical **)mybmalloc2(sizeof(HMM_Logical *) * len, &(winfo->mroot));
memcpy(winfo->wseq[vnum], tmpwseq, sizeof(HMM_Logical *) * len);
winfo->wlen[vnum] = len;
}
vnum++;
*vnum_p = vnum;
return(TRUE);
}
/**
* Sub function to Add a dictionary entry line to the word dictionary.
*
* @param buf [i/o] buffer to hold the input string, will be modified in this function
* @param vnum_p [in] current number of words in @a winfo
* @param linenum [in] current line number of the input
* @param winfo [out] pointer to word dictionary to append the data.
* @param hmminfo [in] HTK %HMM definition data. if NULL, phonemes are ignored.
* @param do_conv [in] TRUE if performing triphone conversion
* @param ok_flag [out] will be set to FALSE if an error occured for this input.
* @param headphone [in] word head silence model name
* @param tailphone [in] word tail silence model name
* @param contextphone [in] silence context name to be used at head and tail
*
* @return FALSE if buf == "DICEND", else TRUE will be returned.
*/
boolean
voca_load_wordlist_line(char *buf, WORD_ID *vnum_p, int linenum, WORD_INFO *winfo, HTK_HMM_INFO *hmminfo, boolean do_conv, boolean *ok_flag, char *headphone, char *tailphone, char *contextphone)
{
char *ptmp, *lp = NULL, *p;
static char cbuf[MAX_HMMNAME_LEN];
static HMM_Logical **tmpwseq = NULL;
static int tmpmaxlen;
int len;
HMM_Logical *tmplg;
boolean pok, first;
int vnum;
vnum = *vnum_p;
if (strmatch(buf, "DICEND")) return FALSE;
/* allocate temporal work area for the first call */
if (tmpwseq == NULL) {
tmpmaxlen = PHONEMELEN_STEP;
tmpwseq = (HMM_Logical **)mymalloc(sizeof(HMM_Logical *) * tmpmaxlen);
}
/* backup whole line for debug output */
strcpy(bufbak, buf);
/* Output string */
if ((ptmp = mystrtok_quote(buf, " \t\n")) == NULL) {
jlog("Error: voca_load_wordlist: line %d: corrupted data:\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
winfo->wname[vnum] = strcpy((char *)mybmalloc2(strlen(ptmp)+1, &(winfo->mroot)), ptmp);
/* reset transparent flag */
winfo->is_transparent[vnum] = FALSE;
/* just move pointer to next token */
if ((ptmp = mystrtok_movetonext(NULL, " \t\n")) == NULL) {
jlog("Error: voca_load_wordlist: line %d: corrupted data:\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
#ifdef CLASS_NGRAM
winfo->cprob[vnum] = 0.0; /* prob = 1.0, logprob = 0.0 */
#endif
if (ptmp[0] == '@') { /* class N-gram prob */
#ifdef CLASS_NGRAM
/* word probability within the class (for class N-gram) */
/* format: classname @classprob wordname [output] phoneseq */
/* classname equals to wname, and wordname will be omitted */
/* format: @%f (log scale) */
/* if "@" not found or "@0", it means class == word */
if ((ptmp = mystrtok(NULL, " \t\n")) == NULL) {
jlog("Error: voca_load_wordlist: line %d: corrupted data:\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
if (ptmp[1] == '\0') { /* space between '@' and figures */
jlog("Error: voca_load_wordlist: line %d: value after '@' missing, maybe wrong space?\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
winfo->cprob[vnum] = atof(&(ptmp[1]));
if (winfo->cprob[vnum] != 0.0) winfo->cwnum++;
/* read next word entry (just skip them) */
if ((ptmp = mystrtok(NULL, " \t\n")) == NULL) {
jlog("Error: voca_load_wordlist: line %d: corrupted data:\n> %s\n", linenum,bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
/* move to the next word entry */
if ((ptmp = mystrtok_movetonext(NULL, " \t\n")) == NULL) {
jlog("Error: voca_load_wordlist: line %d: corrupted data:\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
#else /* ~CLASS_NGRAM */
jlog("Error: voca_load_wordlist: line %d: cannot handle in-class word probability\n> %s\n", linenum, ptmp, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
#endif /* CLASS_NGRAM */
}
/* OutputString */
switch(ptmp[0]) {
case '[': /* ignore transparency */
ptmp = mystrtok_quotation(NULL, " \t\n", '[', ']', 0);
break;
case '{': /* ignore transparency */
ptmp = mystrtok_quotation(NULL, " \t\n", '{', '}', 0);
break;
default:
/* ALLOW no entry for output */
/* same as wname is used */
ptmp = winfo->wname[vnum];
}
if (ptmp == NULL) {
jlog("Error: voca_load_htkdict: line %d: corrupted data:\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
winfo->woutput[vnum] = strcpy((char *)mybmalloc2(strlen(ptmp)+1, &(winfo->mroot)), ptmp);
/* phoneme sequence */
if (hmminfo == NULL) {
/* don't read */
winfo->wseq[vnum] = NULL;
winfo->wlen[vnum] = 0;
} else {
len = 0;
first = TRUE;
pok = TRUE;
for (;;) {
if (do_conv) {
if (first) {
/* init phone cycler */
cycle_triphone(NULL);
/* insert head phone at beginning of word */
if (contextphone) {
if (strlen(contextphone) >= MAX_HMMNAME_LEN) {
jlog("Error: voca_load_htkdict: line %d: too long phone name: %s\n", linenum, contextphone);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
cycle_triphone(contextphone);
} else {
cycle_triphone("NULL_C");
}
if ((lp = mystrtok(NULL, " \t\n")) == NULL) {
jlog("Error: voca_load_wordlist: line %d: word %s has no phoneme:\n> %s\n", linenum, winfo->wname[vnum], bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
if (strlen(lp) >= MAX_HMMNAME_LEN) {
jlog("Error: voca_load_htkdict: line %d: too long phone name: %s\n", linenum, lp);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
p = cycle_triphone(lp);
first = FALSE;
} else { /* do_conv, not first */
if (lp != NULL) { /* some token processed at last loop */
lp = mystrtok(NULL, " \t\n");
if (lp != NULL) {
/* token exist */
if (strlen(lp) >= MAX_HMMNAME_LEN) {
jlog("Error: voca_load_htkdict: line %d: too long phone name: %s\n", linenum, lp);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
p = cycle_triphone(lp);
} else {
/* no more token, insert tail phone at end of word */
if (contextphone) {
if (strlen(contextphone) >= MAX_HMMNAME_LEN) {
jlog("Error: voca_load_htkdict: line %d: too long phone name: %s\n", linenum, contextphone);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
p = cycle_triphone(contextphone);
} else {
p = cycle_triphone("NULL_C");
}
}
} else { /* no more token at last input */
/* flush tone cycler */
p = cycle_triphone_flush();
}
}
} else { /* not do_conv */
if (first) {
p = lp = headphone;
first = FALSE;
} else {
if (lp != NULL) { /* some token processed at last loop */
p = lp = mystrtok(NULL, " \t\n");
/* if no more token, use tailphone */
if (lp == NULL) p = tailphone;
} else {
/* no more token at last input, exit loop */
p = NULL;
}
}
}
if (p == NULL) break;
/* for headphone and tailphone, their context should not be handled */
/* and when they appear as context they should be replaced by contextphone */
if (do_conv) {
center_name(p, cbuf);
if (contextphone) {
if (strmatch(cbuf, contextphone)) {
if (len == 0) {
p = headphone;
} else if (lp == NULL) {
p = tailphone;
}
}
} else {
if (strmatch(cbuf, "NULL_C")) {
if (len == 0) {
p = headphone;
} else if (lp == NULL) {
p = tailphone;
}
} else {
if (strnmatch(p, "NULL_C", 6)) {
if (strnmatch(&(p[strlen(p)-6]), "NULL_C", 6)) {
p = cbuf;
} else {
p = rightcenter_name(p, cbuf);
}
} else if (strnmatch(&(p[strlen(p)-6]), "NULL_C", 6)) {
p = leftcenter_name(p, cbuf);
}
}
}
}
//printf("[[%s]]\n", p);
/* both defined/pseudo phone is allowed */
tmplg = htk_hmmdata_lookup_logical(hmminfo, p);
if (tmplg == NULL) {
/* not found */
if (do_conv) {
/* logical phone was not found */
jlog("Error: voca_load_wordlist: line %d: logical phone \"%s\" not found\n", linenum, p);
snprintf(cbuf,MAX_HMMNAME_LEN,"%s", p);
} else {
jlog("Error: voca_load_wordlist: line %d: phone \"%s\" not found\n", linenum, p);
snprintf(cbuf, MAX_HMMNAME_LEN, "%s", p);
}
add_to_error(winfo, cbuf);
pok = FALSE;
} else {
/* found */
if (len >= tmpmaxlen) {
/* expand wseq area by PHONEMELEN_STEP */
tmpmaxlen += PHONEMELEN_STEP;
tmpwseq = (HMM_Logical **)myrealloc(tmpwseq, sizeof(HMM_Logical *) * tmpmaxlen);
}
/* store to temporal buffer */
tmpwseq[len] = tmplg;
}
len++;
}
if (!pok) { /* error in phoneme */
jlog("Error: voca_load_wordlist: the line content was: %s\n", bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
if (len == 0) {
jlog("Error: voca_load_wordlist: line %d: no phone specified:\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
/* store to winfo */
winfo->wseq[vnum] = (HMM_Logical **)mybmalloc2(sizeof(HMM_Logical *) * len, &(winfo->mroot));
memcpy(winfo->wseq[vnum], tmpwseq, sizeof(HMM_Logical *) * len);
winfo->wlen[vnum] = len;
winfo->wton[vnum] = 0;
}
vnum++;
*vnum_p = vnum;
return(TRUE);
}
