/**
* Load CMN parameter from file. If the number of MFCC dimension in the
* file does not match the specified one, an error will occur.
*
* Format can be either HTK ascii format or binary format (made by Julius older than ver.4.2.3)
*
* @param c [i/o] CMN calculation work area
* @param filename [in] file name
*
* @return TRUE on success, FALSE on failure.
*/
boolean
CMN_load_from_file(CMNWork *c, char *filename)
{
FILE *fp;
int veclen;
char ch[5];
char *buf;
jlog("Stat: wav2mfcc-pipe: reading initial cepstral mean/variance from file \"%s\"\n", filename);
if ((fp = fopen_readfile(filename)) == NULL) {
jlog("Error: wav2mfcc-pipe: failed to open %s\n", filename);
return(FALSE);
}
/* detect file format */
if (myread(&ch, sizeof(char), 5, fp) == FALSE) {
jlog("Error: wav2mfcc-pipe: failed to read CMN/CVN file\n");
fclose_readfile(fp);
return(FALSE);
}
myfrewind(fp);
if (ch[0] == '<' &&
(ch[1] == 'C' || ch[1] == 'c') &&
(ch[2] == 'E' || ch[2] == 'e') &&
(ch[3] == 'P' || ch[3] == 'p') &&
(ch[4] == 'S' || ch[4] == 's') ) {
/* ascii HTK format (>=4.3) */
char *p;
int mode;
int d, dv, len;
jlog("Stat: wav2mfcc-pipe: reading HTK-format cepstral vectors\n");
buf = (char *)mymalloc(MAXBUFLEN);
mode = 0;
while(getl(buf, MAXBUFLEN, fp) != NULL) {
for (p = mystrtok_quote(buf, "<> \t\r\n"); p; p = mystrtok_quote(NULL, "<> \t\r\n")) {
switch(mode){
case 0:
if (strmatch(p, "MEAN")) {
mode = 1;
} else if (strmatch(p, "VARIANCE")) {
mode = 3;
}
break;
case 1:
len = atof(p);
if (len != c->veclen && len != c->mfcc_dim) {
jlog("Error: wav2mfcc-pipe: cepstral dimension mismatch\n");
jlog("Error: wav2mfcc-pipe: process = %d (%d), file = %d\n", c->veclen, c->mfcc_dim, len);
free(buf); fclose_readfile(fp);
return(FALSE);
}
for (d = 0; d < c->veclen; d++) c->cmean_init[d] = 0.0;
d = 0;
mode = 2;
break;
case 2:
if (strmatch(p, "VARIANCE")) {
mode = 3;
} else {
if (d >= len) {
jlog("Error: wav2mfcc-pipe: corrupted data\n");
free(buf); fclose_readfile(fp);
return(FALSE);
}
c->cmean_init[d++] = atof(p);
}
break;
case 3:
len = atof(p);
if (len != c->veclen) {
jlog("Error: wav2mfcc-pipe: cepstral dimension mismatch\n");
jlog("Error: wav2mfcc-pipe: process = %d, file = %d\n", c->veclen, len);
free(buf); fclose_readfile(fp);
return(FALSE);
}
dv = 0;
mode = 4;
break;
case 4:
if (dv >= len) {
jlog("Error: wav2mfcc-pipe: corrupted data\n");
free(buf); fclose_readfile(fp);
return(FALSE);
}
c->cvar_init[dv++] = atof(p);
break;
}
}
}
free(buf);
if (d != len || (mode >= 3 && dv != len)) {
jlog("Error: wav2mfcc-pipe: corrupted data\n");
fclose_readfile(fp);
return(FALSE);
}
} else {
/* binary (<4.3) */
jlog("Stat: wav2mfcc-pipe: reading binary-format cepstral vectors\n");
/* read header */
if (myread(&veclen, sizeof(int), 1, fp) == FALSE) {
jlog("Error: wav2mfcc-pipe: failed to read header\n");
fclose_readfile(fp);
return(FALSE);
}
/* check length */
if (veclen != c->veclen) {
jlog("Error: wav2mfcc-pipe: cepstral dimension mismatch\n");
jlog("Error: wav2mfcc-pipe: process = %d, file = %d\n", c->veclen, veclen);
fclose_readfile(fp);
return(FALSE);
}
/* read body */
if (myread(c->cmean_init, sizeof(float), c->veclen, fp) == FALSE) {
jlog("Error: wav2mfcc-pipe: failed to read mean for CMN\n");
fclose_readfile(fp);
return(FALSE);
}
if (c->var) {
if (myread(c->cvar_init, sizeof(float), c->veclen, fp) == FALSE) {
jlog("Error: wav2mfcc-pipe: failed to read variance for CVN\n");
fclose_readfile(fp);
return(FALSE);
}
}
}
if (fclose_readfile(fp) == -1) {
jlog("Error: wav2mfcc-pipe: failed to close\n");
return(FALSE);
}
c->cmean_init_set = TRUE;
c->loaded_from_file = TRUE;
jlog("Stat: wav2mfcc-pipe: finished reading CMN/CVN parameter\n");
return(TRUE);
}
static boolean
ngram_read_bin_compat(FILE *fp, NGRAM_INFO *ndata, int *retry_ret)
{
int i,n,len;
char *w, *p;
NNID *n3_bgn;
NNID d, ntmp;
#ifdef WORDS_INT
unsigned short *buf;
#endif
NGRAM_TUPLE_INFO *t, *tt, *ttt;
/* old binary N-gram assumes these types */
ndata->bigram_index_reversed = TRUE;
ndata->n = 3;
ndata->dir = DIR_RL;
/* read total info and set max_word_num */
ndata->d = (NGRAM_TUPLE_INFO *)mymalloc(sizeof(NGRAM_TUPLE_INFO) * ndata->n);
memset(ndata->d, 0, sizeof(NGRAM_TUPLE_INFO) * ndata->n);
for(n=0;n<ndata->n;n++) {
rdn(fp, &(ndata->d[n].totalnum), sizeof(NNID), 1);
}
ndata->max_word_num = ndata->d[0].totalnum;
if (file_version == 4) {
rdn(fp, &(ndata->d[1].context_num), sizeof(NNID), 1);
}
for(n=0;n<ndata->n;n++) {
if (n < 2) {
ndata->d[n].is24bit = FALSE;
} else {
if (ndata->d[n].totalnum >= NNID_MAX_24) {
jlog("Warning: ngram_read_bin_compat: num of %d-gram exceeds 24bit, now switch to %dbit index\n", n+1, sizeof(NNID) * 8);
ndata->d[n].is24bit = FALSE;
} else {
ndata->d[n].is24bit = TRUE;
}
}
ndata->d[n].nnid2ctid_upper = NULL;
ndata->d[n].nnid2ctid_lower = NULL;
}
/* always do back-off compaction for 3-gram and up */
/* mark 2-gram and up */
ndata->d[0].ct_compaction = FALSE;
for(n=1;n<ndata->n;n++) {
ndata->d[n].ct_compaction = TRUE;
}
/* read wname */
rdn(fp, &len, sizeof(int), 1);
w = mymalloc(len);
rdn(fp, w, 1, len);
/* assign... */
ndata->wname = (char **)mymalloc(sizeof(char *) * ndata->max_word_num);
p = w; i = 0;
while (p < w + len) {
ndata->wname[i++] = p;
while(*p != '\0') p++;
p++;
}
if (i != ndata->max_word_num) {
jlog("Error: ngram_read_bin_compat: wname error??\n");
return FALSE;
}
/* malloc 1-gram */
t = &(ndata->d[0]);
tt = &(ndata->d[1]);
ttt = &(ndata->d[2]);
t->bgn_upper = NULL;
t->bgn_lower = NULL;
t->bgn = NULL;
t->num = NULL;
t->bgnlistlen = 0;
t->nnid2wid = NULL;
t->nnid2ctid_upper = NULL;
t->nnid2ctid_lower = NULL;
t->context_num = t->totalnum;
t->prob = (LOGPROB *)mymalloc_big(sizeof(LOGPROB), t->totalnum);
ndata->bo_wt_1 = (LOGPROB *)mymalloc_big(sizeof(LOGPROB), t->context_num);
t->bo_wt = (LOGPROB *)mymalloc_big(sizeof(LOGPROB), t->context_num);
tt->bgnlistlen = t->context_num;
tt->bgn = (NNID *)mymalloc_big(sizeof(NNID), tt->bgnlistlen);
tt->num = (WORD_ID *)mymalloc_big(sizeof(WORD_ID), tt->bgnlistlen);
/* read 1-gram */
jlog("stat: ngram_read_bin_compat: reading 1-gram\n");
rdn(fp, t->prob, sizeof(LOGPROB), t->totalnum);
rdn(fp, ndata->bo_wt_1, sizeof(LOGPROB), t->context_num);
rdn(fp, t->bo_wt, sizeof(LOGPROB), t->context_num);
rdn(fp, tt->bgn, sizeof(NNID), tt->bgnlistlen);
#ifdef WORDS_INT
rdn_wordid(fp, tt->num, tt->bgnlistlen, need_conv);
#else
rdn(fp, tt->num, sizeof(WORD_ID), tt->bgnlistlen);
#endif
#ifdef WORDS_INT
{
/* check if we are wrongly reading word_id=2byte bingram
(if bingram version >= 4, this should not be happen because
header correctly tells the word_id byte size. This will
occur only if matches all the conditions below:
- you run Julius with --enable-words-int,
- you use old bingram of version <= 3, and
- you use bingram file converted without --enable-words-int
*/
WORD_ID w;
for(w=0;w<ndata->max_word_num;w++) {
if (ndata->d[1].num[w] > ndata->max_word_num) {
if (words_int_retry) {
jlog("Error: ngram_read_bin_compat: retry failed, wrong bingram format\n");
return FALSE;
}
jlog("Warning: ngram_read_bin_compat: incorrect data, may be a 2-byte v3 bingram, retry with conversion\n");
free(ndata->wname[0]);
free(ndata->wname);
free(t->prob);
free(ndata->bo_wt_1);
free(t->bo_wt);
free(tt->bgn);
free(tt->num);
myfrewind(fp);
words_int_retry = TRUE;
*retry_ret = 1;
return FALSE;
}
}
}
#endif
/* malloc the rest */
tt->nnid2wid = (WORD_ID *)mymalloc_big(sizeof(WORD_ID), tt->totalnum);
tt->prob = (LOGPROB *)mymalloc_big(sizeof(LOGPROB), tt->totalnum);
ndata->p_2 = (LOGPROB *)mymalloc_big(sizeof(LOGPROB), tt->totalnum);
if (file_version == 4) { /* context compaction and 24bit */
tt->nnid2ctid_upper = (NNID_UPPER *)mymalloc_big(sizeof(NNID_UPPER), tt->totalnum);
tt->nnid2ctid_lower = (NNID_LOWER *)mymalloc_big(sizeof(NNID_LOWER), tt->totalnum);
tt->bo_wt = (LOGPROB *)mymalloc_big(sizeof(LOGPROB), tt->context_num);
ttt->bgnlistlen = tt->context_num;
ttt->bgn_upper = (NNID_UPPER *)mymalloc_big(sizeof(NNID_UPPER), ttt->bgnlistlen);
ttt->bgn_lower = (NNID_LOWER *)mymalloc_big(sizeof(NNID_LOWER), ttt->bgnlistlen);
ttt->num = (WORD_ID *)mymalloc_big(sizeof(WORD_ID), ttt->bgnlistlen);
} else {
tt->context_num = tt->totalnum;
tt->bo_wt = (LOGPROB *)mymalloc_big(sizeof(LOGPROB), tt->context_num);
ttt->bgnlistlen = tt->context_num;
ttt->num = (WORD_ID *)mymalloc_big(sizeof(WORD_ID), ttt->bgnlistlen);
if (ttt->is24bit) {
ttt->bgn_upper = (NNID_UPPER *)mymalloc_big(sizeof(NNID_UPPER), ttt->bgnlistlen);
ttt->bgn_lower = (NNID_LOWER *)mymalloc_big(sizeof(NNID_LOWER), ttt->bgnlistlen);
n3_bgn = (NNID *)mymalloc_big(sizeof(NNID), ttt->bgnlistlen);
} else {
ttt->bgn = (NNID *)mymalloc_big(sizeof(NNID), ttt->bgnlistlen);
}
}
ttt->nnid2wid = (WORD_ID *)mymalloc_big(sizeof(WORD_ID), ttt->totalnum);
ttt->prob = (LOGPROB *)mymalloc_big(sizeof(LOGPROB), ttt->totalnum);
ttt->bo_wt = NULL;
/* read 2-gram*/
jlog("Stat: ngram_read_bin_compat: reading 2-gram\n");
#ifdef WORDS_INT
rdn_wordid(fp, tt->nnid2wid, tt->totalnum, need_conv);
#else
rdn(fp, tt->nnid2wid, sizeof(WORD_ID), tt->totalnum);
#endif
rdn(fp, ndata->p_2, sizeof(LOGPROB), tt->totalnum);
rdn(fp, tt->prob, sizeof(LOGPROB), tt->totalnum);
if (file_version == 4) {
rdn(fp, tt->nnid2ctid_upper, sizeof(NNID_UPPER), tt->totalnum);
rdn(fp, tt->nnid2ctid_lower, sizeof(NNID_LOWER), tt->totalnum);
rdn(fp, tt->bo_wt, sizeof(LOGPROB), tt->context_num);
rdn(fp, ttt->bgn_upper, sizeof(NNID_UPPER), ttt->bgnlistlen);
rdn(fp, ttt->bgn_lower, sizeof(NNID_LOWER), ttt->bgnlistlen);
#ifdef WORDS_INT
rdn_wordid(fp, ttt->num, ttt->bgnlistlen, need_conv);
#else
rdn(fp, ttt->num, sizeof(WORD_ID), ttt->bgnlistlen);
#endif
} else {
rdn(fp, tt->bo_wt, sizeof(LOGPROB), tt->context_num);
if (ttt->is24bit) {
rdn(fp, n3_bgn, sizeof(NNID), ttt->bgnlistlen);
for(d=0;d<ttt->bgnlistlen;d++) {
if (n3_bgn[d] == NNID_INVALID) {
ttt->bgn_lower[d] = 0;
ttt->bgn_upper[d] = NNID_INVALID_UPPER;
} else {
ntmp = n3_bgn[d] & 0xffff;
ttt->bgn_lower[d] = ntmp;
ntmp = n3_bgn[d] >> 16;
ttt->bgn_upper[d] = ntmp;
}
}
} else {
