/** * Load CMN parameter from file. If the number of MFCC dimension in the * file does not match the specified one, an error will occur. * * Format can be either HTK ascii format or binary format (made by Julius older than ver.4.2.3) * * @param c [i/o] CMN calculation work area * @param filename [in] file name * * @return TRUE on success, FALSE on failure. */ boolean CMN_load_from_file(CMNWork *c, char *filename) { FILE *fp; int veclen; char ch[5]; char *buf; jlog("Stat: wav2mfcc-pipe: reading initial cepstral mean/variance from file \"%s\"\n", filename); if ((fp = fopen_readfile(filename)) == NULL) { jlog("Error: wav2mfcc-pipe: failed to open %s\n", filename); return(FALSE); } /* detect file format */ if (myread(&ch, sizeof(char), 5, fp) == FALSE) { jlog("Error: wav2mfcc-pipe: failed to read CMN/CVN file\n"); fclose_readfile(fp); return(FALSE); } myfrewind(fp); if (ch[0] == '<' && (ch[1] == 'C' || ch[1] == 'c') && (ch[2] == 'E' || ch[2] == 'e') && (ch[3] == 'P' || ch[3] == 'p') && (ch[4] == 'S' || ch[4] == 's') ) { /* ascii HTK format (>=4.3) */ char *p; int mode; int d, dv, len; jlog("Stat: wav2mfcc-pipe: reading HTK-format cepstral vectors\n"); buf = (char *)mymalloc(MAXBUFLEN); mode = 0; while(getl(buf, MAXBUFLEN, fp) != NULL) { for (p = mystrtok_quote(buf, "<> \t\r\n"); p; p = mystrtok_quote(NULL, "<> \t\r\n")) { switch(mode){ case 0: if (strmatch(p, "MEAN")) { mode = 1; } else if (strmatch(p, "VARIANCE")) { mode = 3; } break; case 1: len = atof(p); if (len != c->veclen && len != c->mfcc_dim) { jlog("Error: wav2mfcc-pipe: cepstral dimension mismatch\n"); jlog("Error: wav2mfcc-pipe: process = %d (%d), file = %d\n", c->veclen, c->mfcc_dim, len); free(buf); fclose_readfile(fp); return(FALSE); } for (d = 0; d < c->veclen; d++) c->cmean_init[d] = 0.0; d = 0; mode = 2; break; case 2: if (strmatch(p, "VARIANCE")) { mode = 3; } else { if (d >= len) { jlog("Error: wav2mfcc-pipe: corrupted data\n"); free(buf); fclose_readfile(fp); return(FALSE); } c->cmean_init[d++] = atof(p); } break; case 3: len = atof(p); if (len != c->veclen) { jlog("Error: wav2mfcc-pipe: cepstral dimension mismatch\n"); jlog("Error: wav2mfcc-pipe: process = %d, file = %d\n", c->veclen, len); free(buf); fclose_readfile(fp); return(FALSE); } dv = 0; mode = 4; break; case 4: if (dv >= len) { jlog("Error: wav2mfcc-pipe: corrupted data\n"); free(buf); fclose_readfile(fp); return(FALSE); } c->cvar_init[dv++] = atof(p); break; } } } free(buf); if (d != len || (mode >= 3 && dv != len)) { jlog("Error: wav2mfcc-pipe: corrupted data\n"); fclose_readfile(fp); return(FALSE); } } else { /* binary (<4.3) */ jlog("Stat: wav2mfcc-pipe: reading binary-format cepstral vectors\n"); /* read header */ if (myread(&veclen, sizeof(int), 1, fp) == FALSE) { jlog("Error: wav2mfcc-pipe: failed to read header\n"); fclose_readfile(fp); return(FALSE); } /* check length */ if (veclen != c->veclen) { jlog("Error: wav2mfcc-pipe: cepstral dimension mismatch\n"); jlog("Error: wav2mfcc-pipe: process = %d, file = %d\n", c->veclen, veclen); fclose_readfile(fp); return(FALSE); } /* read body */ if (myread(c->cmean_init, sizeof(float), c->veclen, fp) == FALSE) { jlog("Error: wav2mfcc-pipe: failed to read mean for CMN\n"); fclose_readfile(fp); return(FALSE); } if (c->var) { if (myread(c->cvar_init, sizeof(float), c->veclen, fp) == FALSE) { jlog("Error: wav2mfcc-pipe: failed to read variance for CVN\n"); fclose_readfile(fp); return(FALSE); } } } if (fclose_readfile(fp) == -1) { jlog("Error: wav2mfcc-pipe: failed to close\n"); return(FALSE); } c->cmean_init_set = TRUE; c->loaded_from_file = TRUE; jlog("Stat: wav2mfcc-pipe: finished reading CMN/CVN parameter\n"); return(TRUE); }
static boolean ngram_read_bin_compat(FILE *fp, NGRAM_INFO *ndata, int *retry_ret) { int i,n,len; char *w, *p; NNID *n3_bgn; NNID d, ntmp; #ifdef WORDS_INT unsigned short *buf; #endif NGRAM_TUPLE_INFO *t, *tt, *ttt; /* old binary N-gram assumes these types */ ndata->bigram_index_reversed = TRUE; ndata->n = 3; ndata->dir = DIR_RL; /* read total info and set max_word_num */ ndata->d = (NGRAM_TUPLE_INFO *)mymalloc(sizeof(NGRAM_TUPLE_INFO) * ndata->n); memset(ndata->d, 0, sizeof(NGRAM_TUPLE_INFO) * ndata->n); for(n=0;n<ndata->n;n++) { rdn(fp, &(ndata->d[n].totalnum), sizeof(NNID), 1); } ndata->max_word_num = ndata->d[0].totalnum; if (file_version == 4) { rdn(fp, &(ndata->d[1].context_num), sizeof(NNID), 1); } for(n=0;n<ndata->n;n++) { if (n < 2) { ndata->d[n].is24bit = FALSE; } else { if (ndata->d[n].totalnum >= NNID_MAX_24) { jlog("Warning: ngram_read_bin_compat: num of %d-gram exceeds 24bit, now switch to %dbit index\n", n+1, sizeof(NNID) * 8); ndata->d[n].is24bit = FALSE; } else { ndata->d[n].is24bit = TRUE; } } ndata->d[n].nnid2ctid_upper = NULL; ndata->d[n].nnid2ctid_lower = NULL; } /* always do back-off compaction for 3-gram and up */ /* mark 2-gram and up */ ndata->d[0].ct_compaction = FALSE; for(n=1;n<ndata->n;n++) { ndata->d[n].ct_compaction = TRUE; } /* read wname */ rdn(fp, &len, sizeof(int), 1); w = mymalloc(len); rdn(fp, w, 1, len); /* assign... */ ndata->wname = (char **)mymalloc(sizeof(char *) * ndata->max_word_num); p = w; i = 0; while (p < w + len) { ndata->wname[i++] = p; while(*p != '\0') p++; p++; } if (i != ndata->max_word_num) { jlog("Error: ngram_read_bin_compat: wname error??\n"); return FALSE; } /* malloc 1-gram */ t = &(ndata->d[0]); tt = &(ndata->d[1]); ttt = &(ndata->d[2]); t->bgn_upper = NULL; t->bgn_lower = NULL; t->bgn = NULL; t->num = NULL; t->bgnlistlen = 0; t->nnid2wid = NULL; t->nnid2ctid_upper = NULL; t->nnid2ctid_lower = NULL; t->context_num = t->totalnum; t->prob = (LOGPROB *)mymalloc_big(sizeof(LOGPROB), t->totalnum); ndata->bo_wt_1 = (LOGPROB *)mymalloc_big(sizeof(LOGPROB), t->context_num); t->bo_wt = (LOGPROB *)mymalloc_big(sizeof(LOGPROB), t->context_num); tt->bgnlistlen = t->context_num; tt->bgn = (NNID *)mymalloc_big(sizeof(NNID), tt->bgnlistlen); tt->num = (WORD_ID *)mymalloc_big(sizeof(WORD_ID), tt->bgnlistlen); /* read 1-gram */ jlog("stat: ngram_read_bin_compat: reading 1-gram\n"); rdn(fp, t->prob, sizeof(LOGPROB), t->totalnum); rdn(fp, ndata->bo_wt_1, sizeof(LOGPROB), t->context_num); rdn(fp, t->bo_wt, sizeof(LOGPROB), t->context_num); rdn(fp, tt->bgn, sizeof(NNID), tt->bgnlistlen); #ifdef WORDS_INT rdn_wordid(fp, tt->num, tt->bgnlistlen, need_conv); #else rdn(fp, tt->num, sizeof(WORD_ID), tt->bgnlistlen); #endif #ifdef WORDS_INT { /* check if we are wrongly reading word_id=2byte bingram (if bingram version >= 4, this should not be happen because header correctly tells the word_id byte size. This will occur only if matches all the conditions below: - you run Julius with --enable-words-int, - you use old bingram of version <= 3, and - you use bingram file converted without --enable-words-int */ WORD_ID w; for(w=0;w<ndata->max_word_num;w++) { if (ndata->d[1].num[w] > ndata->max_word_num) { if (words_int_retry) { jlog("Error: ngram_read_bin_compat: retry failed, wrong bingram format\n"); return FALSE; } jlog("Warning: ngram_read_bin_compat: incorrect data, may be a 2-byte v3 bingram, retry with conversion\n"); free(ndata->wname[0]); free(ndata->wname); free(t->prob); free(ndata->bo_wt_1); free(t->bo_wt); free(tt->bgn); free(tt->num); myfrewind(fp); words_int_retry = TRUE; *retry_ret = 1; return FALSE; } } } #endif /* malloc the rest */ tt->nnid2wid = (WORD_ID *)mymalloc_big(sizeof(WORD_ID), tt->totalnum); tt->prob = (LOGPROB *)mymalloc_big(sizeof(LOGPROB), tt->totalnum); ndata->p_2 = (LOGPROB *)mymalloc_big(sizeof(LOGPROB), tt->totalnum); if (file_version == 4) { /* context compaction and 24bit */ tt->nnid2ctid_upper = (NNID_UPPER *)mymalloc_big(sizeof(NNID_UPPER), tt->totalnum); tt->nnid2ctid_lower = (NNID_LOWER *)mymalloc_big(sizeof(NNID_LOWER), tt->totalnum); tt->bo_wt = (LOGPROB *)mymalloc_big(sizeof(LOGPROB), tt->context_num); ttt->bgnlistlen = tt->context_num; ttt->bgn_upper = (NNID_UPPER *)mymalloc_big(sizeof(NNID_UPPER), ttt->bgnlistlen); ttt->bgn_lower = (NNID_LOWER *)mymalloc_big(sizeof(NNID_LOWER), ttt->bgnlistlen); ttt->num = (WORD_ID *)mymalloc_big(sizeof(WORD_ID), ttt->bgnlistlen); } else { tt->context_num = tt->totalnum; tt->bo_wt = (LOGPROB *)mymalloc_big(sizeof(LOGPROB), tt->context_num); ttt->bgnlistlen = tt->context_num; ttt->num = (WORD_ID *)mymalloc_big(sizeof(WORD_ID), ttt->bgnlistlen); if (ttt->is24bit) { ttt->bgn_upper = (NNID_UPPER *)mymalloc_big(sizeof(NNID_UPPER), ttt->bgnlistlen); ttt->bgn_lower = (NNID_LOWER *)mymalloc_big(sizeof(NNID_LOWER), ttt->bgnlistlen); n3_bgn = (NNID *)mymalloc_big(sizeof(NNID), ttt->bgnlistlen); } else { ttt->bgn = (NNID *)mymalloc_big(sizeof(NNID), ttt->bgnlistlen); } } ttt->nnid2wid = (WORD_ID *)mymalloc_big(sizeof(WORD_ID), ttt->totalnum); ttt->prob = (LOGPROB *)mymalloc_big(sizeof(LOGPROB), ttt->totalnum); ttt->bo_wt = NULL; /* read 2-gram*/ jlog("Stat: ngram_read_bin_compat: reading 2-gram\n"); #ifdef WORDS_INT rdn_wordid(fp, tt->nnid2wid, tt->totalnum, need_conv); #else rdn(fp, tt->nnid2wid, sizeof(WORD_ID), tt->totalnum); #endif rdn(fp, ndata->p_2, sizeof(LOGPROB), tt->totalnum); rdn(fp, tt->prob, sizeof(LOGPROB), tt->totalnum); if (file_version == 4) { rdn(fp, tt->nnid2ctid_upper, sizeof(NNID_UPPER), tt->totalnum); rdn(fp, tt->nnid2ctid_lower, sizeof(NNID_LOWER), tt->totalnum); rdn(fp, tt->bo_wt, sizeof(LOGPROB), tt->context_num); rdn(fp, ttt->bgn_upper, sizeof(NNID_UPPER), ttt->bgnlistlen); rdn(fp, ttt->bgn_lower, sizeof(NNID_LOWER), ttt->bgnlistlen); #ifdef WORDS_INT rdn_wordid(fp, ttt->num, ttt->bgnlistlen, need_conv); #else rdn(fp, ttt->num, sizeof(WORD_ID), ttt->bgnlistlen); #endif } else { rdn(fp, tt->bo_wt, sizeof(LOGPROB), tt->context_num); if (ttt->is24bit) { rdn(fp, n3_bgn, sizeof(NNID), ttt->bgnlistlen); for(d=0;d<ttt->bgnlistlen;d++) { if (n3_bgn[d] == NNID_INVALID) { ttt->bgn_lower[d] = 0; ttt->bgn_upper[d] = NNID_INVALID_UPPER; } else { ntmp = n3_bgn[d] & 0xffff; ttt->bgn_lower[d] = ntmp; ntmp = n3_bgn[d] >> 16; ttt->bgn_upper[d] = ntmp; } } } else {