/** 
 * Read next token and ste it to rdhmmdef_token.
 * 
 * @param fp [in] file pointer
 * 
 * @return the pointer to the read token, or NULL on end of file or error.
 */
char *
read_token(FILE *fp)
{
  int len;
  int bp = 0;
  int maxlen = MAXBUFLEN;
  static char delims[] = HMMDEF_DELM;

  if ((rdhmmdef_token = mystrtok_quote(NULL, HMMDEF_DELM)) != NULL) {
    /* has token */
    if (mystrtok_movetonext(NULL, HMMDEF_DELM) != NULL || last_line_full == FALSE) {
      /* return the current token, if this is not a last token, or
	 last is newline terminated */
      return rdhmmdef_token;
    } else {
      /* concatinate the last token with next line */
      len = strlen(rdhmmdef_token);
      memmove(buf, rdhmmdef_token, len);
      bp = len;
      maxlen -= len;
    }
  }

  /* read new 1 line a*/
  while(
#ifdef HAVE_ZLIB
	gzgets((gzFile)fp, &(buf[bp]), maxlen) != Z_NULL
#else
	fgets(&(buf[bp]), maxlen, fp) != NULL
#endif
	) {
    /* chop delimiters at end of line (incl. newline) */
    /* if no delimiter at end of line, last_line_full is TRUE */
    last_line_full = TRUE;
    len = strlen(buf)-1;
    while (len >= 0 && strchr(delims, buf[len])) {
      last_line_full = FALSE;
      buf[len--] = '\0';
    }
    if (buf[0] != '\0') {
      /* start getting next token */
      rdhmmdef_token = mystrtok_quote(buf, HMMDEF_DELM);
      /* increment line */
      line++;
      return rdhmmdef_token;
    }
  }
  /* when reading error, return NULL */
  rdhmmdef_token = NULL;
  return rdhmmdef_token;
}
/**
 * Sub function to Add a dictionary entry line to the word dictionary.
 *
 * @param buf [i/o] buffer to hold the input string, will be modified in this function
 * @param vnum_p [in] current number of words in @a winfo
 * @param linenum [in] current line number of the input
 * @param winfo [out] pointer to word dictionary to append the data.
 * @param hmminfo [in] HTK %HMM definition data.  if NULL, phonemes are ignored.
 * @param do_conv [in] TRUE if performing triphone conversion
 * @param ok_flag [out] will be set to FALSE if an error occured for this input.
 *
 * @return FALSE if buf == "DICEND", else TRUE will be returned.
 */
boolean
voca_load_htkdict_line(char *buf, WORD_ID *vnum_p, int linenum, WORD_INFO *winfo, HTK_HMM_INFO *hmminfo, boolean do_conv, boolean *ok_flag)
{
    char *ptmp, *lp = NULL, *p;
    static char cbuf[MAX_HMMNAME_LEN];
    HMM_Logical **tmpwseq;
    int len;
    HMM_Logical *tmplg;
    boolean pok;
    int vnum;

    vnum = *vnum_p;

    if (strmatch(buf, "DICEND")) return FALSE;

    /* allocate temporal work area for the first call */
    if (winfo->work == NULL) {
        winfo->work_num = PHONEMELEN_STEP;
        winfo->work = (void *)mybmalloc2(sizeof(HMM_Logical *) * winfo->work_num, &(winfo->mroot));
    }
    tmpwseq = (HMM_Logical **)winfo->work;

    /* backup whole line for debug output */
    strcpy(bufbak, buf);

    /* GrammarEntry */
    if ((ptmp = mystrtok_quote(buf, " \t\n")) == NULL) {
        jlog("Error: voca_load_htkdict: line %d: corrupted data:\n> %s\n", linenum, bufbak);
        winfo->errnum++;
        *ok_flag = FALSE;
        return TRUE;
    }
    winfo->wname[vnum] = strcpy((char *)mybmalloc2(strlen(ptmp)+1, &(winfo->mroot)), ptmp);

    /* just move pointer to next token */
    if ((ptmp = mystrtok_movetonext(NULL, " \t\n")) == NULL) {
        jlog("Error: voca_load_htkdict: line %d: corrupted data:\n> %s\n", linenum, bufbak);
        winfo->errnum++;
        *ok_flag = FALSE;
        return TRUE;
    }
#ifdef CLASS_NGRAM
    winfo->cprob[vnum] = 0.0;	/* prob = 1.0, logprob = 0.0 */
#endif

    if (ptmp[0] == '@') {		/* class N-gram prob */
#ifdef CLASS_NGRAM
        /* word probability within the class (for class N-gram) */
        /* format: classname @classprob wordname [output] phoneseq */
        /* classname equals to wname, and wordname will be omitted */
        /* format: @%f (log scale) */
        /* if "@" not found or "@0", it means class == word */
        if ((ptmp = mystrtok(NULL, " \t\n")) == NULL) {
            jlog("Error: voca_load_htkdict: line %d: corrupted data:\n> %s\n", linenum, bufbak);
            winfo->errnum++;
            *ok_flag = FALSE;
            return TRUE;
        }
        if (ptmp[1] == '\0') {	/* space between '@' and figures */
            jlog("Error: voca_load_htkdict: line %d: value after '@' missing, maybe wrong space?\n> %s\n", linenum, bufbak);
            winfo->errnum++;
            *ok_flag = FALSE;
            return TRUE;
        }
        winfo->cprob[vnum] = atof(&(ptmp[1]));
        if (winfo->cprob[vnum] != 0.0) winfo->cwnum++;
        /* read next word entry (just skip them) */
        if ((ptmp = mystrtok(NULL, " \t\n")) == NULL) {
            jlog("Error: voca_load_htkdict: line %d: corrupted data:\n> %s\n", linenum,bufbak);
            winfo->errnum++;
            *ok_flag = FALSE;
            return TRUE;
        }
        /* move to the next word entry */
        if ((ptmp = mystrtok_movetonext(NULL, " \t\n")) == NULL) {
            jlog("Error: voca_load_htkdict: line %d: corrupted data:\n> %s\n", linenum, bufbak);
            winfo->errnum++;
            *ok_flag = FALSE;
            return TRUE;
        }
#else  /* ~CLASS_NGRAM */
        jlog("Error: voca_load_htkdict: line %d: cannot handle in-class word probability\n> %s\n", linenum, ptmp, bufbak);
        winfo->errnum++;
        *ok_flag = FALSE;
        return TRUE;
#endif /* CLASS_NGRAM */
    }

    /* OutputString */
    switch(ptmp[0]) {
    case '[':			/* not transparent word */
        winfo->is_transparent[vnum] = FALSE;
        ptmp = mystrtok_quotation(NULL, " \t\n", '[', ']', 0);
        break;
    case '{':			/* transparent word */
        winfo->is_transparent[vnum] = TRUE;
        ptmp = mystrtok_quotation(NULL, " \t\n", '{', '}', 0);
        break;
    default:
#if 1
        /* ALLOW no entry for output */
        /* same as wname is used */
        winfo->is_transparent[vnum] = FALSE;
        ptmp = winfo->wname[vnum];
#else
        /* error */
        jlog("Error: voca_load_htkdict: line %d: missing output string??\n> %s\n", linenum, bufbak);
        winfo->errnum++;
        *ok_flag = FALSE;
        return TRUE;
#endif
    }
    if (ptmp == NULL) {
        jlog("Error: voca_load_htkdict: line %d: corrupted data:\n> %s\n", linenum, bufbak);
        winfo->errnum++;
        *ok_flag = FALSE;
        return TRUE;
    }
    winfo->woutput[vnum] = strcpy((char *)mybmalloc2(strlen(ptmp)+1, &(winfo->mroot)), ptmp);

    /* phoneme sequence */
    if (hmminfo == NULL) {
        /* don't read */
        winfo->wseq[vnum] = NULL;
        winfo->wlen[vnum] = 0;
    } else {

        /* store converted phone sequence to temporal bufffer */
        len = 0;

        if (do_conv) {
            /* convert phoneme to triphone expression (word-internal) */
            cycle_triphone(NULL);
            if ((lp = mystrtok(NULL, " \t\n")) == NULL) {
                jlog("Error: voca_load_htkdict: line %d: word %s has no phoneme:\n> %s\n", linenum, winfo->wname[vnum], bufbak);
                winfo->errnum++;
                *ok_flag = FALSE;
                return TRUE;
            }
            if (strlen(lp) >= MAX_HMMNAME_LEN) {
                jlog("Error: voca_load_htkdict: line %d: too long phone name: %s\n", linenum, lp);
                winfo->errnum++;
                *ok_flag = FALSE;
                return TRUE;
            }
            cycle_triphone(lp);
        }

        pok = TRUE;
        for (;;) {
            if (do_conv) {
                /*	if (lp != NULL) jlog(" %d%s",len,lp);*/
                if (lp != NULL) lp = mystrtok(NULL, " \t\n");
                if (lp != NULL) {
                    if (strlen(lp) >= MAX_HMMNAME_LEN) {
                        jlog("Error: voca_load_htkdict: line %d: too long phone name: %s\n", linenum, lp);
                        winfo->errnum++;
                        *ok_flag = FALSE;
                        return TRUE;
                    }
                    p = cycle_triphone(lp);
                }
                else p = cycle_triphone_flush();
            } else {
                p = mystrtok(NULL, " \t\n");
            }
            if (p == NULL) break;

            /* both defined/pseudo phone is allowed */
            tmplg = htk_hmmdata_lookup_logical(hmminfo, p);
            if (tmplg == NULL) {
                /* not found */
                if (do_conv) {
                    /* both defined or pseudo phone are not found */
                    if (len == 0 && lp == NULL) {
                        jlog("Error: voca_load_htkdict: line %d: triphone \"*-%s+*\" or monophone \"%s\" not found\n", linenum, p, p);
                        snprintf(cbuf,MAX_HMMNAME_LEN,"*-%s+* or monophone %s", p, p);
                    } else if (len == 0) {
                        jlog("Error: voca_load_htkdict: line %d: triphone \"*-%s\" or biphone \"%s\" not found\n", linenum, p, p);
                        snprintf(cbuf,MAX_HMMNAME_LEN,"*-%s or biphone %s", p, p);
                    } else if (lp == NULL) {
                        jlog("Error: voca_load_htkdict: line %d: triphone \"%s+*\" or biphone \"%s\" not found\n", linenum, p, p);
                        snprintf(cbuf,MAX_HMMNAME_LEN,"%s+* or biphone %s", p, p);
                    } else {
                        jlog("Error: voca_load_htkdict: line %d: triphone \"%s\" not found\n", linenum, p);
                        snprintf(cbuf,MAX_HMMNAME_LEN,"%s", p);
                    }
                } else {
                    jlog("Error: voca_load_htkdict: line %d: phone \"%s\" not found\n", linenum, p);
                    snprintf(cbuf, MAX_HMMNAME_LEN, "%s", p);
                }
                add_to_error(winfo, cbuf);
                pok = FALSE;
            } else {
                /* found */
                if (len >= winfo->work_num) {
                    /* expand wseq area by PHONEMELEN_STEP */
                    winfo->work_num += PHONEMELEN_STEP;
                    winfo->work = (void *)mybmalloc2(sizeof(HMM_Logical *) * winfo->work_num, &(winfo->mroot));
                    memcpy(winfo->work, tmpwseq, sizeof(HMM_Logical *) * (winfo->work_num - PHONEMELEN_STEP));
                    tmpwseq = (HMM_Logical **)winfo->work;
                }
                /* store to temporal buffer */
                tmpwseq[len] = tmplg;
            }
            len++;
        }
        if (!pok) {			/* error in phoneme */
            jlog("Error: voca_load_htkdict: the line content was: %s\n", bufbak);
            winfo->errnum++;
            *ok_flag = FALSE;
            return TRUE;
        }
        if (len == 0) {
            jlog("Error: voca_load_htkdict: line %d: no phone specified:\n> %s\n", linenum, bufbak);
            winfo->errnum++;
            *ok_flag = FALSE;
            return TRUE;
        }
        /* store to winfo */
        winfo->wseq[vnum] = (HMM_Logical **)mybmalloc2(sizeof(HMM_Logical *) * len, &(winfo->mroot));
        memcpy(winfo->wseq[vnum], tmpwseq, sizeof(HMM_Logical *) * len);
        winfo->wlen[vnum] = len;
    }

    vnum++;

    *vnum_p = vnum;

    return(TRUE);
}
/**
 * Sub function to Add a dictionary entry line to the word dictionary.
 *
 * @param buf [i/o] buffer to hold the input string, will be modified in this function
 * @param vnum_p [in] current number of words in @a winfo
 * @param linenum [in] current line number of the input
 * @param winfo [out] pointer to word dictionary to append the data.
 * @param hmminfo [in] HTK %HMM definition data.  if NULL, phonemes are ignored.
 * @param do_conv [in] TRUE if performing triphone conversion
 * @param ok_flag [out] will be set to FALSE if an error occured for this input.
 * @param headphone [in] word head silence model name
 * @param tailphone [in] word tail silence model name
 * @param contextphone [in] silence context name to be used at head and tail
 *
 * @return FALSE if buf == "DICEND", else TRUE will be returned.
 */
boolean
voca_load_wordlist_line(char *buf, WORD_ID *vnum_p, int linenum, WORD_INFO *winfo, HTK_HMM_INFO *hmminfo, boolean do_conv, boolean *ok_flag, char *headphone, char *tailphone, char *contextphone)
{
    char *ptmp, *lp = NULL, *p;
    static char cbuf[MAX_HMMNAME_LEN];
    static HMM_Logical **tmpwseq = NULL;
    static int tmpmaxlen;
    int len;
    HMM_Logical *tmplg;
    boolean pok, first;
    int vnum;

    vnum = *vnum_p;

    if (strmatch(buf, "DICEND")) return FALSE;

    /* allocate temporal work area for the first call */
    if (tmpwseq == NULL) {
        tmpmaxlen = PHONEMELEN_STEP;
        tmpwseq = (HMM_Logical **)mymalloc(sizeof(HMM_Logical *) * tmpmaxlen);
    }

    /* backup whole line for debug output */
    strcpy(bufbak, buf);

    /* Output string */
    if ((ptmp = mystrtok_quote(buf, " \t\n")) == NULL) {
        jlog("Error: voca_load_wordlist: line %d: corrupted data:\n> %s\n", linenum, bufbak);
        winfo->errnum++;
        *ok_flag = FALSE;
        return TRUE;
    }
    winfo->wname[vnum] = strcpy((char *)mybmalloc2(strlen(ptmp)+1, &(winfo->mroot)), ptmp);

    /* reset transparent flag */
    winfo->is_transparent[vnum] = FALSE;

    /* just move pointer to next token */
    if ((ptmp = mystrtok_movetonext(NULL, " \t\n")) == NULL) {
        jlog("Error: voca_load_wordlist: line %d: corrupted data:\n> %s\n", linenum, bufbak);
        winfo->errnum++;
        *ok_flag = FALSE;
        return TRUE;
    }
#ifdef CLASS_NGRAM
    winfo->cprob[vnum] = 0.0;	/* prob = 1.0, logprob = 0.0 */
#endif

    if (ptmp[0] == '@') {		/* class N-gram prob */
#ifdef CLASS_NGRAM
        /* word probability within the class (for class N-gram) */
        /* format: classname @classprob wordname [output] phoneseq */
        /* classname equals to wname, and wordname will be omitted */
        /* format: @%f (log scale) */
        /* if "@" not found or "@0", it means class == word */
        if ((ptmp = mystrtok(NULL, " \t\n")) == NULL) {
            jlog("Error: voca_load_wordlist: line %d: corrupted data:\n> %s\n", linenum, bufbak);
            winfo->errnum++;
            *ok_flag = FALSE;
            return TRUE;
        }
        if (ptmp[1] == '\0') {	/* space between '@' and figures */
            jlog("Error: voca_load_wordlist: line %d: value after '@' missing, maybe wrong space?\n> %s\n", linenum, bufbak);
            winfo->errnum++;
            *ok_flag = FALSE;
            return TRUE;
        }
        winfo->cprob[vnum] = atof(&(ptmp[1]));
        if (winfo->cprob[vnum] != 0.0) winfo->cwnum++;
        /* read next word entry (just skip them) */
        if ((ptmp = mystrtok(NULL, " \t\n")) == NULL) {
            jlog("Error: voca_load_wordlist: line %d: corrupted data:\n> %s\n", linenum,bufbak);
            winfo->errnum++;
            *ok_flag = FALSE;
            return TRUE;
        }
        /* move to the next word entry */
        if ((ptmp = mystrtok_movetonext(NULL, " \t\n")) == NULL) {
            jlog("Error: voca_load_wordlist: line %d: corrupted data:\n> %s\n", linenum, bufbak);
            winfo->errnum++;
            *ok_flag = FALSE;
            return TRUE;
        }
#else  /* ~CLASS_NGRAM */
        jlog("Error: voca_load_wordlist: line %d: cannot handle in-class word probability\n> %s\n", linenum, ptmp, bufbak);
        winfo->errnum++;
        *ok_flag = FALSE;
        return TRUE;
#endif /* CLASS_NGRAM */
    }

    /* OutputString */
    switch(ptmp[0]) {
    case '[':			/* ignore transparency */
        ptmp = mystrtok_quotation(NULL, " \t\n", '[', ']', 0);
        break;
    case '{':			/* ignore transparency */
        ptmp = mystrtok_quotation(NULL, " \t\n", '{', '}', 0);
        break;
    default:
        /* ALLOW no entry for output */
        /* same as wname is used */
        ptmp = winfo->wname[vnum];
    }
    if (ptmp == NULL) {
        jlog("Error: voca_load_htkdict: line %d: corrupted data:\n> %s\n", linenum, bufbak);
        winfo->errnum++;
        *ok_flag = FALSE;
        return TRUE;
    }
    winfo->woutput[vnum] = strcpy((char *)mybmalloc2(strlen(ptmp)+1, &(winfo->mroot)), ptmp);

    /* phoneme sequence */
    if (hmminfo == NULL) {
        /* don't read */
        winfo->wseq[vnum] = NULL;
        winfo->wlen[vnum] = 0;
    } else {

        len = 0;
        first = TRUE;
        pok = TRUE;

        for (;;) {
            if (do_conv) {
                if (first) {
                    /* init phone cycler */
                    cycle_triphone(NULL);
                    /* insert head phone at beginning of word */
                    if (contextphone) {
                        if (strlen(contextphone) >= MAX_HMMNAME_LEN) {
                            jlog("Error: voca_load_htkdict: line %d: too long phone name: %s\n", linenum, contextphone);
                            winfo->errnum++;
                            *ok_flag = FALSE;
                            return TRUE;
                        }
                        cycle_triphone(contextphone);
                    } else {
                        cycle_triphone("NULL_C");
                    }
                    if ((lp = mystrtok(NULL, " \t\n")) == NULL) {
                        jlog("Error: voca_load_wordlist: line %d: word %s has no phoneme:\n> %s\n", linenum, winfo->wname[vnum], bufbak);
                        winfo->errnum++;
                        *ok_flag = FALSE;
                        return TRUE;
                    }
                    if (strlen(lp) >= MAX_HMMNAME_LEN) {
                        jlog("Error: voca_load_htkdict: line %d: too long phone name: %s\n", linenum, lp);
                        winfo->errnum++;
                        *ok_flag = FALSE;
                        return TRUE;
                    }
                    p = cycle_triphone(lp);
                    first = FALSE;
                } else {		/* do_conv, not first */
                    if (lp != NULL) {	/* some token processed at last loop */
                        lp = mystrtok(NULL, " \t\n");
                        if (lp != NULL) {
                            /* token exist */
                            if (strlen(lp) >= MAX_HMMNAME_LEN) {
                                jlog("Error: voca_load_htkdict: line %d: too long phone name: %s\n", linenum, lp);
                                winfo->errnum++;
                                *ok_flag = FALSE;
                                return TRUE;
                            }
                            p = cycle_triphone(lp);
                        } else {
                            /* no more token, insert tail phone at end of word */
                            if (contextphone) {
                                if (strlen(contextphone) >= MAX_HMMNAME_LEN) {
                                    jlog("Error: voca_load_htkdict: line %d: too long phone name: %s\n", linenum, contextphone);
                                    winfo->errnum++;
                                    *ok_flag = FALSE;
                                    return TRUE;
                                }
                                p = cycle_triphone(contextphone);
                            } else {
                                p = cycle_triphone("NULL_C");
                            }
                        }
                    } else {		/* no more token at last input  */
                        /* flush tone cycler */
                        p = cycle_triphone_flush();
                    }
                }
            } else {			/* not do_conv */
                if (first) {
                    p = lp = headphone;
                    first = FALSE;
                } else {
                    if (lp != NULL) {	/* some token processed at last loop */
                        p = lp = mystrtok(NULL, " \t\n");
                        /* if no more token, use tailphone */
                        if (lp == NULL) p = tailphone;
                    } else {
                        /* no more token at last input, exit loop */
                        p = NULL;
                    }
                }
            }
            if (p == NULL) break;
            /* for headphone and tailphone, their context should not be handled */
            /* and when they appear as context they should be replaced by contextphone */
            if (do_conv) {
                center_name(p, cbuf);
                if (contextphone) {
                    if (strmatch(cbuf, contextphone)) {
                        if (len == 0) {
                            p = headphone;
                        } else if (lp == NULL) {
                            p = tailphone;
                        }
                    }
                } else {
                    if (strmatch(cbuf, "NULL_C")) {
                        if (len == 0) {
                            p = headphone;
                        } else if (lp == NULL) {
                            p = tailphone;
                        }
                    } else {
                        if (strnmatch(p, "NULL_C", 6)) {
                            if (strnmatch(&(p[strlen(p)-6]), "NULL_C", 6)) {
                                p = cbuf;
                            } else {
                                p = rightcenter_name(p, cbuf);
                            }
                        } else if (strnmatch(&(p[strlen(p)-6]), "NULL_C", 6)) {
                            p = leftcenter_name(p, cbuf);
                        }
                    }
                }
            }
            //printf("[[%s]]\n", p);

            /* both defined/pseudo phone is allowed */
            tmplg = htk_hmmdata_lookup_logical(hmminfo, p);
            if (tmplg == NULL) {
                /* not found */
                if (do_conv) {
                    /* logical phone was not found */
                    jlog("Error: voca_load_wordlist: line %d: logical phone \"%s\" not found\n", linenum, p);
                    snprintf(cbuf,MAX_HMMNAME_LEN,"%s", p);
                } else {
                    jlog("Error: voca_load_wordlist: line %d: phone \"%s\" not found\n", linenum, p);
                    snprintf(cbuf, MAX_HMMNAME_LEN, "%s", p);
                }
                add_to_error(winfo, cbuf);
                pok = FALSE;
            } else {
                /* found */
                if (len >= tmpmaxlen) {
                    /* expand wseq area by PHONEMELEN_STEP */
                    tmpmaxlen += PHONEMELEN_STEP;
                    tmpwseq = (HMM_Logical **)myrealloc(tmpwseq, sizeof(HMM_Logical *) * tmpmaxlen);
                }
                /* store to temporal buffer */
                tmpwseq[len] = tmplg;
            }
            len++;
        }
        if (!pok) {			/* error in phoneme */
            jlog("Error: voca_load_wordlist: the line content was: %s\n", bufbak);
            winfo->errnum++;
            *ok_flag = FALSE;
            return TRUE;
        }
        if (len == 0) {
            jlog("Error: voca_load_wordlist: line %d: no phone specified:\n> %s\n", linenum, bufbak);
            winfo->errnum++;
            *ok_flag = FALSE;
            return TRUE;
        }
        /* store to winfo */
        winfo->wseq[vnum] = (HMM_Logical **)mybmalloc2(sizeof(HMM_Logical *) * len, &(winfo->mroot));
        memcpy(winfo->wseq[vnum], tmpwseq, sizeof(HMM_Logical *) * len);
        winfo->wlen[vnum] = len;
        winfo->wton[vnum] = 0;
    }

    vnum++;
    *vnum_p = vnum;

    return(TRUE);
}