/**
* Read next token and ste it to rdhmmdef_token.
*
* @param fp [in] file pointer
*
* @return the pointer to the read token, or NULL on end of file or error.
*/
char *
read_token(FILE *fp)
{
int len;
int bp = 0;
int maxlen = MAXBUFLEN;
static char delims[] = HMMDEF_DELM;
if ((rdhmmdef_token = mystrtok_quote(NULL, HMMDEF_DELM)) != NULL) {
/* has token */
if (mystrtok_movetonext(NULL, HMMDEF_DELM) != NULL || last_line_full == FALSE) {
/* return the current token, if this is not a last token, or
last is newline terminated */
return rdhmmdef_token;
} else {
/* concatinate the last token with next line */
len = strlen(rdhmmdef_token);
memmove(buf, rdhmmdef_token, len);
bp = len;
maxlen -= len;
}
}
/* read new 1 line a*/
while(
#ifdef HAVE_ZLIB
gzgets((gzFile)fp, &(buf[bp]), maxlen) != Z_NULL
#else
fgets(&(buf[bp]), maxlen, fp) != NULL
#endif
) {
/* chop delimiters at end of line (incl. newline) */
/* if no delimiter at end of line, last_line_full is TRUE */
last_line_full = TRUE;
len = strlen(buf)-1;
while (len >= 0 && strchr(delims, buf[len])) {
last_line_full = FALSE;
buf[len--] = '\0';
}
if (buf[0] != '\0') {
/* start getting next token */
rdhmmdef_token = mystrtok_quote(buf, HMMDEF_DELM);
/* increment line */
line++;
return rdhmmdef_token;
}
}
/* when reading error, return NULL */
rdhmmdef_token = NULL;
return rdhmmdef_token;
}
/**
* Sub function to Add a dictionary entry line to the word dictionary.
*
* @param buf [i/o] buffer to hold the input string, will be modified in this function
* @param vnum_p [in] current number of words in @a winfo
* @param linenum [in] current line number of the input
* @param winfo [out] pointer to word dictionary to append the data.
* @param hmminfo [in] HTK %HMM definition data. if NULL, phonemes are ignored.
* @param do_conv [in] TRUE if performing triphone conversion
* @param ok_flag [out] will be set to FALSE if an error occured for this input.
*
* @return FALSE if buf == "DICEND", else TRUE will be returned.
*/
boolean
voca_load_htkdict_line(char *buf, WORD_ID *vnum_p, int linenum, WORD_INFO *winfo, HTK_HMM_INFO *hmminfo, boolean do_conv, boolean *ok_flag)
{
char *ptmp, *lp = NULL, *p;
static char cbuf[MAX_HMMNAME_LEN];
HMM_Logical **tmpwseq;
int len;
HMM_Logical *tmplg;
boolean pok;
int vnum;
vnum = *vnum_p;
if (strmatch(buf, "DICEND")) return FALSE;
/* allocate temporal work area for the first call */
if (winfo->work == NULL) {
winfo->work_num = PHONEMELEN_STEP;
winfo->work = (void *)mybmalloc2(sizeof(HMM_Logical *) * winfo->work_num, &(winfo->mroot));
}
tmpwseq = (HMM_Logical **)winfo->work;
/* backup whole line for debug output */
strcpy(bufbak, buf);
/* GrammarEntry */
if ((ptmp = mystrtok_quote(buf, " \t\n")) == NULL) {
jlog("Error: voca_load_htkdict: line %d: corrupted data:\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
winfo->wname[vnum] = strcpy((char *)mybmalloc2(strlen(ptmp)+1, &(winfo->mroot)), ptmp);
/* just move pointer to next token */
if ((ptmp = mystrtok_movetonext(NULL, " \t\n")) == NULL) {
jlog("Error: voca_load_htkdict: line %d: corrupted data:\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
#ifdef CLASS_NGRAM
winfo->cprob[vnum] = 0.0; /* prob = 1.0, logprob = 0.0 */
#endif
if (ptmp[0] == '@') { /* class N-gram prob */
#ifdef CLASS_NGRAM
/* word probability within the class (for class N-gram) */
/* format: classname @classprob wordname [output] phoneseq */
/* classname equals to wname, and wordname will be omitted */
/* format: @%f (log scale) */
/* if "@" not found or "@0", it means class == word */
if ((ptmp = mystrtok(NULL, " \t\n")) == NULL) {
jlog("Error: voca_load_htkdict: line %d: corrupted data:\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
if (ptmp[1] == '\0') { /* space between '@' and figures */
jlog("Error: voca_load_htkdict: line %d: value after '@' missing, maybe wrong space?\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
winfo->cprob[vnum] = atof(&(ptmp[1]));
if (winfo->cprob[vnum] != 0.0) winfo->cwnum++;
/* read next word entry (just skip them) */
if ((ptmp = mystrtok(NULL, " \t\n")) == NULL) {
jlog("Error: voca_load_htkdict: line %d: corrupted data:\n> %s\n", linenum,bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
/* move to the next word entry */
if ((ptmp = mystrtok_movetonext(NULL, " \t\n")) == NULL) {
jlog("Error: voca_load_htkdict: line %d: corrupted data:\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
#else /* ~CLASS_NGRAM */
jlog("Error: voca_load_htkdict: line %d: cannot handle in-class word probability\n> %s\n", linenum, ptmp, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
#endif /* CLASS_NGRAM */
}
/* OutputString */
switch(ptmp[0]) {
case '[': /* not transparent word */
winfo->is_transparent[vnum] = FALSE;
ptmp = mystrtok_quotation(NULL, " \t\n", '[', ']', 0);
break;
case '{': /* transparent word */
winfo->is_transparent[vnum] = TRUE;
ptmp = mystrtok_quotation(NULL, " \t\n", '{', '}', 0);
break;
default:
#if 1
/* ALLOW no entry for output */
/* same as wname is used */
winfo->is_transparent[vnum] = FALSE;
ptmp = winfo->wname[vnum];
#else
/* error */
jlog("Error: voca_load_htkdict: line %d: missing output string??\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
#endif
}
if (ptmp == NULL) {
jlog("Error: voca_load_htkdict: line %d: corrupted data:\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
winfo->woutput[vnum] = strcpy((char *)mybmalloc2(strlen(ptmp)+1, &(winfo->mroot)), ptmp);
/* phoneme sequence */
if (hmminfo == NULL) {
/* don't read */
winfo->wseq[vnum] = NULL;
winfo->wlen[vnum] = 0;
} else {
/* store converted phone sequence to temporal bufffer */
len = 0;
if (do_conv) {
/* convert phoneme to triphone expression (word-internal) */
cycle_triphone(NULL);
if ((lp = mystrtok(NULL, " \t\n")) == NULL) {
jlog("Error: voca_load_htkdict: line %d: word %s has no phoneme:\n> %s\n", linenum, winfo->wname[vnum], bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
if (strlen(lp) >= MAX_HMMNAME_LEN) {
jlog("Error: voca_load_htkdict: line %d: too long phone name: %s\n", linenum, lp);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
cycle_triphone(lp);
}
pok = TRUE;
for (;;) {
if (do_conv) {
/* if (lp != NULL) jlog(" %d%s",len,lp);*/
if (lp != NULL) lp = mystrtok(NULL, " \t\n");
if (lp != NULL) {
if (strlen(lp) >= MAX_HMMNAME_LEN) {
jlog("Error: voca_load_htkdict: line %d: too long phone name: %s\n", linenum, lp);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
p = cycle_triphone(lp);
}
else p = cycle_triphone_flush();
} else {
p = mystrtok(NULL, " \t\n");
}
if (p == NULL) break;
/* both defined/pseudo phone is allowed */
tmplg = htk_hmmdata_lookup_logical(hmminfo, p);
if (tmplg == NULL) {
/* not found */
if (do_conv) {
/* both defined or pseudo phone are not found */
if (len == 0 && lp == NULL) {
jlog("Error: voca_load_htkdict: line %d: triphone \"*-%s+*\" or monophone \"%s\" not found\n", linenum, p, p);
snprintf(cbuf,MAX_HMMNAME_LEN,"*-%s+* or monophone %s", p, p);
} else if (len == 0) {
jlog("Error: voca_load_htkdict: line %d: triphone \"*-%s\" or biphone \"%s\" not found\n", linenum, p, p);
snprintf(cbuf,MAX_HMMNAME_LEN,"*-%s or biphone %s", p, p);
} else if (lp == NULL) {
jlog("Error: voca_load_htkdict: line %d: triphone \"%s+*\" or biphone \"%s\" not found\n", linenum, p, p);
snprintf(cbuf,MAX_HMMNAME_LEN,"%s+* or biphone %s", p, p);
} else {
jlog("Error: voca_load_htkdict: line %d: triphone \"%s\" not found\n", linenum, p);
snprintf(cbuf,MAX_HMMNAME_LEN,"%s", p);
}
} else {
jlog("Error: voca_load_htkdict: line %d: phone \"%s\" not found\n", linenum, p);
snprintf(cbuf, MAX_HMMNAME_LEN, "%s", p);
}
add_to_error(winfo, cbuf);
pok = FALSE;
} else {
/* found */
if (len >= winfo->work_num) {
/* expand wseq area by PHONEMELEN_STEP */
winfo->work_num += PHONEMELEN_STEP;
winfo->work = (void *)mybmalloc2(sizeof(HMM_Logical *) * winfo->work_num, &(winfo->mroot));
memcpy(winfo->work, tmpwseq, sizeof(HMM_Logical *) * (winfo->work_num - PHONEMELEN_STEP));
tmpwseq = (HMM_Logical **)winfo->work;
}
/* store to temporal buffer */
tmpwseq[len] = tmplg;
}
len++;
}
if (!pok) { /* error in phoneme */
jlog("Error: voca_load_htkdict: the line content was: %s\n", bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
if (len == 0) {
jlog("Error: voca_load_htkdict: line %d: no phone specified:\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
/* store to winfo */
winfo->wseq[vnum] = (HMM_Logical **)mybmalloc2(sizeof(HMM_Logical *) * len, &(winfo->mroot));
memcpy(winfo->wseq[vnum], tmpwseq, sizeof(HMM_Logical *) * len);
winfo->wlen[vnum] = len;
}
vnum++;
*vnum_p = vnum;
return(TRUE);
}
/**
* Sub function to Add a dictionary entry line to the word dictionary.
*
* @param buf [i/o] buffer to hold the input string, will be modified in this function
* @param vnum_p [in] current number of words in @a winfo
* @param linenum [in] current line number of the input
* @param winfo [out] pointer to word dictionary to append the data.
* @param hmminfo [in] HTK %HMM definition data. if NULL, phonemes are ignored.
* @param do_conv [in] TRUE if performing triphone conversion
* @param ok_flag [out] will be set to FALSE if an error occured for this input.
* @param headphone [in] word head silence model name
* @param tailphone [in] word tail silence model name
* @param contextphone [in] silence context name to be used at head and tail
*
* @return FALSE if buf == "DICEND", else TRUE will be returned.
*/
boolean
voca_load_wordlist_line(char *buf, WORD_ID *vnum_p, int linenum, WORD_INFO *winfo, HTK_HMM_INFO *hmminfo, boolean do_conv, boolean *ok_flag, char *headphone, char *tailphone, char *contextphone)
{
char *ptmp, *lp = NULL, *p;
static char cbuf[MAX_HMMNAME_LEN];
static HMM_Logical **tmpwseq = NULL;
static int tmpmaxlen;
int len;
HMM_Logical *tmplg;
boolean pok, first;
int vnum;
vnum = *vnum_p;
if (strmatch(buf, "DICEND")) return FALSE;
/* allocate temporal work area for the first call */
if (tmpwseq == NULL) {
tmpmaxlen = PHONEMELEN_STEP;
tmpwseq = (HMM_Logical **)mymalloc(sizeof(HMM_Logical *) * tmpmaxlen);
}
/* backup whole line for debug output */
strcpy(bufbak, buf);
/* Output string */
if ((ptmp = mystrtok_quote(buf, " \t\n")) == NULL) {
jlog("Error: voca_load_wordlist: line %d: corrupted data:\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
winfo->wname[vnum] = strcpy((char *)mybmalloc2(strlen(ptmp)+1, &(winfo->mroot)), ptmp);
/* reset transparent flag */
winfo->is_transparent[vnum] = FALSE;
/* just move pointer to next token */
if ((ptmp = mystrtok_movetonext(NULL, " \t\n")) == NULL) {
jlog("Error: voca_load_wordlist: line %d: corrupted data:\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
#ifdef CLASS_NGRAM
winfo->cprob[vnum] = 0.0; /* prob = 1.0, logprob = 0.0 */
#endif
if (ptmp[0] == '@') { /* class N-gram prob */
#ifdef CLASS_NGRAM
/* word probability within the class (for class N-gram) */
/* format: classname @classprob wordname [output] phoneseq */
/* classname equals to wname, and wordname will be omitted */
/* format: @%f (log scale) */
/* if "@" not found or "@0", it means class == word */
if ((ptmp = mystrtok(NULL, " \t\n")) == NULL) {
jlog("Error: voca_load_wordlist: line %d: corrupted data:\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
if (ptmp[1] == '\0') { /* space between '@' and figures */
jlog("Error: voca_load_wordlist: line %d: value after '@' missing, maybe wrong space?\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
winfo->cprob[vnum] = atof(&(ptmp[1]));
if (winfo->cprob[vnum] != 0.0) winfo->cwnum++;
/* read next word entry (just skip them) */
if ((ptmp = mystrtok(NULL, " \t\n")) == NULL) {
jlog("Error: voca_load_wordlist: line %d: corrupted data:\n> %s\n", linenum,bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
/* move to the next word entry */
if ((ptmp = mystrtok_movetonext(NULL, " \t\n")) == NULL) {
jlog("Error: voca_load_wordlist: line %d: corrupted data:\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
#else /* ~CLASS_NGRAM */
jlog("Error: voca_load_wordlist: line %d: cannot handle in-class word probability\n> %s\n", linenum, ptmp, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
#endif /* CLASS_NGRAM */
}
/* OutputString */
switch(ptmp[0]) {
case '[': /* ignore transparency */
ptmp = mystrtok_quotation(NULL, " \t\n", '[', ']', 0);
break;
case '{': /* ignore transparency */
ptmp = mystrtok_quotation(NULL, " \t\n", '{', '}', 0);
break;
default:
/* ALLOW no entry for output */
/* same as wname is used */
ptmp = winfo->wname[vnum];
}
if (ptmp == NULL) {
jlog("Error: voca_load_htkdict: line %d: corrupted data:\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
winfo->woutput[vnum] = strcpy((char *)mybmalloc2(strlen(ptmp)+1, &(winfo->mroot)), ptmp);
/* phoneme sequence */
if (hmminfo == NULL) {
/* don't read */
winfo->wseq[vnum] = NULL;
winfo->wlen[vnum] = 0;
} else {
len = 0;
first = TRUE;
pok = TRUE;
for (;;) {
if (do_conv) {
if (first) {
/* init phone cycler */
cycle_triphone(NULL);
/* insert head phone at beginning of word */
if (contextphone) {
if (strlen(contextphone) >= MAX_HMMNAME_LEN) {
jlog("Error: voca_load_htkdict: line %d: too long phone name: %s\n", linenum, contextphone);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
cycle_triphone(contextphone);
} else {
cycle_triphone("NULL_C");
}
if ((lp = mystrtok(NULL, " \t\n")) == NULL) {
jlog("Error: voca_load_wordlist: line %d: word %s has no phoneme:\n> %s\n", linenum, winfo->wname[vnum], bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
if (strlen(lp) >= MAX_HMMNAME_LEN) {
jlog("Error: voca_load_htkdict: line %d: too long phone name: %s\n", linenum, lp);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
p = cycle_triphone(lp);
first = FALSE;
} else { /* do_conv, not first */
if (lp != NULL) { /* some token processed at last loop */
lp = mystrtok(NULL, " \t\n");
if (lp != NULL) {
/* token exist */
if (strlen(lp) >= MAX_HMMNAME_LEN) {
jlog("Error: voca_load_htkdict: line %d: too long phone name: %s\n", linenum, lp);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
p = cycle_triphone(lp);
} else {
/* no more token, insert tail phone at end of word */
if (contextphone) {
if (strlen(contextphone) >= MAX_HMMNAME_LEN) {
jlog("Error: voca_load_htkdict: line %d: too long phone name: %s\n", linenum, contextphone);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
p = cycle_triphone(contextphone);
} else {
p = cycle_triphone("NULL_C");
}
}
} else { /* no more token at last input */
/* flush tone cycler */
p = cycle_triphone_flush();
}
}
} else { /* not do_conv */
if (first) {
p = lp = headphone;
first = FALSE;
} else {
if (lp != NULL) { /* some token processed at last loop */
p = lp = mystrtok(NULL, " \t\n");
/* if no more token, use tailphone */
if (lp == NULL) p = tailphone;
} else {
/* no more token at last input, exit loop */
p = NULL;
}
}
}
if (p == NULL) break;
/* for headphone and tailphone, their context should not be handled */
/* and when they appear as context they should be replaced by contextphone */
if (do_conv) {
center_name(p, cbuf);
if (contextphone) {
if (strmatch(cbuf, contextphone)) {
if (len == 0) {
p = headphone;
} else if (lp == NULL) {
p = tailphone;
}
}
} else {
if (strmatch(cbuf, "NULL_C")) {
if (len == 0) {
p = headphone;
} else if (lp == NULL) {
p = tailphone;
}
} else {
if (strnmatch(p, "NULL_C", 6)) {
if (strnmatch(&(p[strlen(p)-6]), "NULL_C", 6)) {
p = cbuf;
} else {
p = rightcenter_name(p, cbuf);
}
} else if (strnmatch(&(p[strlen(p)-6]), "NULL_C", 6)) {
p = leftcenter_name(p, cbuf);
}
}
}
}
//printf("[[%s]]\n", p);
/* both defined/pseudo phone is allowed */
tmplg = htk_hmmdata_lookup_logical(hmminfo, p);
if (tmplg == NULL) {
/* not found */
if (do_conv) {
/* logical phone was not found */
jlog("Error: voca_load_wordlist: line %d: logical phone \"%s\" not found\n", linenum, p);
snprintf(cbuf,MAX_HMMNAME_LEN,"%s", p);
} else {
jlog("Error: voca_load_wordlist: line %d: phone \"%s\" not found\n", linenum, p);
snprintf(cbuf, MAX_HMMNAME_LEN, "%s", p);
}
add_to_error(winfo, cbuf);
pok = FALSE;
} else {
/* found */
if (len >= tmpmaxlen) {
/* expand wseq area by PHONEMELEN_STEP */
tmpmaxlen += PHONEMELEN_STEP;
tmpwseq = (HMM_Logical **)myrealloc(tmpwseq, sizeof(HMM_Logical *) * tmpmaxlen);
}
/* store to temporal buffer */
tmpwseq[len] = tmplg;
}
len++;
}
if (!pok) { /* error in phoneme */
jlog("Error: voca_load_wordlist: the line content was: %s\n", bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
if (len == 0) {
jlog("Error: voca_load_wordlist: line %d: no phone specified:\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
/* store to winfo */
winfo->wseq[vnum] = (HMM_Logical **)mybmalloc2(sizeof(HMM_Logical *) * len, &(winfo->mroot));
memcpy(winfo->wseq[vnum], tmpwseq, sizeof(HMM_Logical *) * len);
winfo->wlen[vnum] = len;
winfo->wton[vnum] = 0;
}
vnum++;
*vnum_p = vnum;
return(TRUE);
}