static int32 senone_mixw_read(senone_t * s, char const *file_name, logmath_t *lmath) { char eofchk; FILE *fp; int32 byteswap, chksum_present; uint32 chksum; float32 *pdf; int32 i, f, c, p, n_err; char **argname, **argval; E_INFO("Reading senone mixture weights: %s\n", file_name); if ((fp = fopen(file_name, "rb")) == NULL) E_FATAL_SYSTEM("Failed to open mixture weights file '%s' for reading", file_name); /* Read header, including argument-value info and 32-bit byteorder magic */ if (bio_readhdr(fp, &argname, &argval, &byteswap) < 0) E_FATAL("Failed to read header from file '%s'\n", file_name); /* Parse argument-value list */ chksum_present = 0; for (i = 0; argname[i]; i++) { if (strcmp(argname[i], "version") == 0) { if (strcmp(argval[i], MIXW_PARAM_VERSION) != 0) E_WARN("Version mismatch(%s): %s, expecting %s\n", file_name, argval[i], MIXW_PARAM_VERSION); } else if (strcmp(argname[i], "chksum0") == 0) { chksum_present = 1; /* Ignore the associated value */ } } bio_hdrarg_free(argname, argval); argname = argval = NULL; chksum = 0; /* Read #senones, #features, #codewords, arraysize */ if ((bio_fread(&(s->n_sen), sizeof(int32), 1, fp, byteswap, &chksum) != 1) || (bio_fread(&(s->n_feat), sizeof(int32), 1, fp, byteswap, &chksum) != 1) || (bio_fread(&(s->n_cw), sizeof(int32), 1, fp, byteswap, &chksum) != 1) || (bio_fread(&i, sizeof(int32), 1, fp, byteswap, &chksum) != 1)) { E_FATAL("bio_fread(%s) (arraysize) failed\n", file_name); } if (i != s->n_sen * s->n_feat * s->n_cw) { E_FATAL ("%s: #float32s(%d) doesn't match dimensions: %d x %d x %d\n", file_name, i, s->n_sen, s->n_feat, s->n_cw); } /* * Compute #LSB bits to be dropped to represent mixwfloor with 8 bits. * All PDF values will be truncated (in the LSB positions) by these many bits. */ if ((s->mixwfloor <= 0.0) || (s->mixwfloor >= 1.0)) E_FATAL("mixwfloor (%e) not in range (0, 1)\n", s->mixwfloor); /* Use a fixed shift for compatibility with everything else. */ E_INFO("Truncating senone logs3(pdf) values by %d bits\n", SENSCR_SHIFT); /* * Allocate memory for senone PDF data. Organize normally or transposed depending on * s->n_gauden. */ if (s->n_gauden > 1) { E_INFO("Not transposing mixture weights in memory\n"); s->pdf = (senprob_t ***) ckd_calloc_3d(s->n_sen, s->n_feat, s->n_cw, sizeof(senprob_t)); } else { E_INFO("Transposing mixture weights in memory\n"); s->pdf = (senprob_t ***) ckd_calloc_3d(s->n_feat, s->n_cw, s->n_sen, sizeof(senprob_t)); } /* Temporary structure to read in floats */ pdf = (float32 *) ckd_calloc(s->n_cw, sizeof(float32)); /* Read senone probs data, normalize, floor, convert to logs3, truncate to 8 bits */ n_err = 0; for (i = 0; i < s->n_sen; i++) { for (f = 0; f < s->n_feat; f++) { if (bio_fread ((void *) pdf, sizeof(float32), s->n_cw, fp, byteswap, &chksum) != s->n_cw) { E_FATAL("bio_fread(%s) (arraydata) failed\n", file_name); } /* Normalize and floor */ if (vector_sum_norm(pdf, s->n_cw) <= 0.0) n_err++; vector_floor(pdf, s->n_cw, s->mixwfloor); vector_sum_norm(pdf, s->n_cw); /* Convert to logs3, truncate to 8 bits, and store in s->pdf */ for (c = 0; c < s->n_cw; c++) { p = -(logmath_log(lmath, pdf[c])); p += (1 << (SENSCR_SHIFT - 1)) - 1; /* Rounding before truncation */ if (s->n_gauden > 1) s->pdf[i][f][c] = (p < (255 << SENSCR_SHIFT)) ? (p >> SENSCR_SHIFT) : 255; else s->pdf[f][c][i] = (p < (255 << SENSCR_SHIFT)) ? (p >> SENSCR_SHIFT) : 255; } } }
static int32 read_mixw(s2_semi_mgau_t * s, char const *file_name, double SmoothMin) { char **argname, **argval; char eofchk; FILE *fp; int32 byteswap, chksum_present; uint32 chksum; float32 *pdf; int32 i, f, c, n; int32 n_sen; int32 n_feat; int32 n_comp; int32 n_err; E_INFO("Reading mixture weights file '%s'\n", file_name); if ((fp = fopen(file_name, "rb")) == NULL) E_FATAL("fopen(%s,rb) failed\n", file_name); /* Read header, including argument-value info and 32-bit byteorder magic */ if (bio_readhdr(fp, &argname, &argval, &byteswap) < 0) E_FATAL("bio_readhdr(%s) failed\n", file_name); /* Parse argument-value list */ chksum_present = 0; for (i = 0; argname[i]; i++) { if (strcmp(argname[i], "version") == 0) { if (strcmp(argval[i], MGAU_MIXW_VERSION) != 0) E_WARN("Version mismatch(%s): %s, expecting %s\n", file_name, argval[i], MGAU_MIXW_VERSION); } else if (strcmp(argname[i], "chksum0") == 0) { chksum_present = 1; /* Ignore the associated value */ } } bio_hdrarg_free(argname, argval); argname = argval = NULL; chksum = 0; /* Read #senones, #features, #codewords, arraysize */ if ((bio_fread(&n_sen, sizeof(int32), 1, fp, byteswap, &chksum) != 1) || (bio_fread(&n_feat, sizeof(int32), 1, fp, byteswap, &chksum) != 1) || (bio_fread(&n_comp, sizeof(int32), 1, fp, byteswap, &chksum) != 1) || (bio_fread(&n, sizeof(int32), 1, fp, byteswap, &chksum) != 1)) { E_FATAL("bio_fread(%s) (arraysize) failed\n", file_name); } if (n_feat != s->n_feat) E_FATAL("#Features streams(%d) != %d\n", n_feat, s->n_feat); if (n != n_sen * n_feat * n_comp) { E_FATAL ("%s: #float32s(%d) doesn't match header dimensions: %d x %d x %d\n", file_name, i, n_sen, n_feat, n_comp); } /* n_sen = number of mixture weights per codeword, which is * fixed at the number of senones since we have only one codebook. */ s->n_sen = n_sen; /* Quantized mixture weight arrays. */ s->mixw = ckd_calloc_3d(s->n_feat, s->n_density, n_sen, sizeof(***s->mixw)); /* Temporary structure to read in floats before conversion to (int32) logs3 */ pdf = (float32 *) ckd_calloc(n_comp, sizeof(float32)); /* Read senone probs data, normalize, floor, convert to logs3, truncate to 8 bits */ n_err = 0; for (i = 0; i < n_sen; i++) { for (f = 0; f < n_feat; f++) { if (bio_fread((void *) pdf, sizeof(float32), n_comp, fp, byteswap, &chksum) != n_comp) { E_FATAL("bio_fread(%s) (arraydata) failed\n", file_name); } /* Normalize and floor */ if (vector_sum_norm(pdf, n_comp) <= 0.0) n_err++; vector_floor(pdf, n_comp, SmoothMin); vector_sum_norm(pdf, n_comp); /* Convert to LOG, quantize, and transpose */ for (c = 0; c < n_comp; c++) { int32 qscr; qscr = -logmath_log(s->lmath_8b, pdf[c]); if ((qscr > MAX_NEG_MIXW) || (qscr < 0)) qscr = MAX_NEG_MIXW; s->mixw[f][c][i] = qscr; } } } if (n_err > 0) E_WARN("Weight normalization failed for %d senones\n", n_err); ckd_free(pdf); if (chksum_present) bio_verify_chksum(fp, byteswap, chksum); if (fread(&eofchk, 1, 1, fp) == 1) E_FATAL("More data than expected in %s\n", file_name); fclose(fp); E_INFO("Read %d x %d x %d mixture weights\n", n_sen, n_feat, n_comp); return n_sen; }
tmat_t *tmat_init (char *file_name, float64 tpfloor) { char tmp; int32 n_src, n_dst; FILE *fp; int32 byteswap, chksum_present; uint32 chksum; float32 **tp; int32 i, j, k, tp_per_tmat; char **argname, **argval; tmat_t *t; E_INFO("Reading HMM transition probability matrices: %s\n", file_name); t = (tmat_t *) ckd_calloc (1, sizeof(tmat_t)); if ((fp = fopen(file_name, "rb")) == NULL) E_FATAL_SYSTEM("fopen(%s,rb) failed\n", file_name); /* Read header, including argument-value info and 32-bit byteorder magic */ if (bio_readhdr (fp, &argname, &argval, &byteswap) < 0) E_FATAL("bio_readhdr(%s) failed\n", file_name); /* Parse argument-value list */ chksum_present = 0; for (i = 0; argname[i]; i++) { if (strcmp (argname[i], "version") == 0) { if (strcmp(argval[i], TMAT_PARAM_VERSION) != 0) E_WARN("Version mismatch(%s): %s, expecting %s\n", file_name, argval[i], TMAT_PARAM_VERSION); } else if (strcmp (argname[i], "chksum0") == 0) { chksum_present = 1; /* Ignore the associated value */ } } bio_hdrarg_free (argname, argval); argname = argval = NULL; chksum = 0; /* Read #tmat, #from-states, #to-states, arraysize */ if ((bio_fread (&(t->n_tmat), sizeof(int32), 1, fp, byteswap, &chksum) != 1) || (bio_fread (&n_src, sizeof(int32), 1, fp, byteswap, &chksum) != 1) || (bio_fread (&n_dst, sizeof(int32), 1, fp, byteswap, &chksum) != 1) || (bio_fread (&i, sizeof(int32), 1, fp, byteswap, &chksum) != 1)) { E_FATAL("bio_fread(%s) (arraysize) failed\n", file_name); } if (t->n_tmat >= MAX_S3TMATID) E_FATAL("%s: #tmat (%d) exceeds limit (%d)\n", file_name, t->n_tmat, MAX_S3TMATID); if (n_dst != n_src+1) E_FATAL("%s: #from-states(%d) != #to-states(%d)-1\n", file_name, n_src, n_dst); t->n_state = n_src; if (i != t->n_tmat * n_src * n_dst) { E_FATAL("%s: #float32s(%d) doesn't match dimensions: %d x %d x %d\n", file_name, i, t->n_tmat, n_src, n_dst); } /* Allocate memory for tmat data */ t->tp = (int32 ***) ckd_calloc_3d (t->n_tmat, n_src, n_dst, sizeof(int32)); /* Temporary structure to read in the float data */ tp = (float32 **) ckd_calloc_2d (n_src, n_dst, sizeof(float32)); /* Read transition matrices, normalize and floor them, and convert to logs3 domain */ tp_per_tmat = n_src * n_dst; for (i = 0; i < t->n_tmat; i++) { if (bio_fread (tp[0], sizeof(float32), tp_per_tmat, fp, byteswap, &chksum) != tp_per_tmat) { E_FATAL("fread(%s) (arraydata) failed\n", file_name); } /* Normalize and floor */ for (j = 0; j < n_src; j++) { if (vector_sum_norm (tp[j], n_dst) == 0.0) E_ERROR("Normalization failed for tmat %d from state %d\n", i, j); vector_nz_floor (tp[j], n_dst, tpfloor); vector_sum_norm (tp[j], n_dst); /* Convert to logs3. Take care of special case when tp = 0.0! */ for (k = 0; k < n_dst; k++) t->tp[i][j][k] = (tp[j][k] == 0.0) ? S3_LOGPROB_ZERO : logs3(tp[j][k]); } } ckd_free_2d ((void **) tp); if (chksum_present) bio_verify_chksum (fp, byteswap, chksum); if (fread (&tmp, 1, 1, fp) == 1) E_ERROR("Non-empty file beyond end of data\n"); fclose(fp); E_INFO("Read %d transition matrices of size %dx%d\n", t->n_tmat, t->n_state, t->n_state+1); if (tmat_chk_uppertri (t) < 0) E_FATAL("Tmat not upper triangular\n"); return t; }
static int32 senone_mixw_read(logmath_t * logmath, senone_t *s, const char *file_name, float64 mixwfloor) { FILE *fp; char **argname, **argval; int32 byteswap, chksum_present; uint32 chksum; float32 *pdf; int32 i, j, f, m, c, p, n_sen, n_err, n_cw, nval; char eofchk; mixw_t *fw; E_INFO("Reading senone mixture weights: %s\n", file_name); if ((fp = fopen(file_name, "rb")) == NULL) E_FATAL_SYSTEM("fopen(%s,rb) failed\n", file_name); /* Read header, including argument-value info and 32-bit byteorder magic */ if (bio_readhdr (fp, &argname, &argval, &byteswap) < 0) E_FATAL("bio_readhdr(%s) failed\n", file_name); /* Parse argument-value list */ chksum_present = 0; for (i = 0; argname[i]; i++) { if (strcmp (argname[i], "version") == 0) { if (strcmp(argval[i], MIXW_PARAM_VERSION) != 0) E_WARN("Version mismatch(%s): %s, expecting %s\n", file_name, argval[i], MIXW_PARAM_VERSION); } else if (strcmp (argname[i], "chksum0") == 0) { chksum_present = 1; /* Ignore the associated value */ } } bio_hdrarg_free (argname, argval); argname = argval = NULL; chksum = 0; /* Read #senones, #features, #codewords, arraysize */ n_sen = s->n_sen; if ((bio_fread (&(s->n_sen), sizeof(int32), 1, fp, byteswap, &chksum) != 1) || (bio_fread (&(s->n_feat), sizeof(int32), 1, fp, byteswap, &chksum) != 1) || (bio_fread (&(n_cw), sizeof(int32), 1, fp, byteswap, &chksum) != 1) || (bio_fread (&nval, sizeof(int32), 1, fp, byteswap, &chksum) != 1)) { E_FATAL("bio_fread(%s) (arraysize) failed\n", file_name); } if ((n_sen != 0) && (s->n_sen != n_sen)) E_FATAL("#senones(%d) conflict with mapping file(%d)\n", s->n_sen, n_sen); if (s->n_sen >= MAX_SENID) E_FATAL("%s: #senones (%d) exceeds limit (%d)\n", file_name, s->n_sen, MAX_SENID); if (s->n_feat <= 0) E_FATAL("Bad #features: %d\n", s->n_feat); if (n_cw <= 0) E_FATAL("Bad #mixing-wts/senone: %d\n", n_cw); /* Allocate sen2mgau map if not yet done so (i.e. no explicit mapping file given */ if (! s->sen2mgau) { assert ((s->n_mgau == 0) || (s->n_mgau == 1)); s->sen2mgau = (uint32 *) ckd_calloc (s->n_sen, sizeof(int32)); if (s->n_mgau == 1) { /* Semicontinuous mode; all senones map to single, shared gaussian: 0 */ for (i = 0; i < s->n_sen; i++) s->sen2mgau[i] = 0; } else { /* Fully continuous mode; each senone maps to own parent gaussian */ s->n_mgau = s->n_sen; for (i = 0; i < s->n_sen; i++) s->sen2mgau[i] = i; } } else assert (s->n_mgau != 0); if (s->n_mgau >= MAX_MGAUID) E_FATAL("%s: #gauden (%d) exceeds limit (%d)\n", file_name, s->n_mgau, MAX_MGAUID); if (nval != s->n_sen * s->n_feat * n_cw) { E_FATAL("%s: #float32 values(%d) doesn't match dimensions: %d x %d x %d\n", file_name, nval, s->n_sen, s->n_feat, n_cw); } /* * Compute #LSB bits to be dropped to represent mixwfloor with 8 bits. * All PDF values will be truncated (in the LSB positions) by these many bits. */ if ((mixwfloor <= 0.0) || (mixwfloor >= 1.0)) E_FATAL("mixwfloor (%e) not in range (0, 1)\n", mixwfloor); /* Allocate memory for s->mgau2sen and senone PDF data */ build_mgau2sen (s, n_cw); /* Temporary structure to read in floats */ pdf = (float32 *) ckd_calloc (n_cw, sizeof(float32)); /* Read senone probs data, normalize, floor, convert to logs3, truncate to 8 bits */ n_err = 0; for (i = 0; i < s->n_sen; i++) { m = s->sen2mgau[i]; /* Parent mgau */ j = s->mgau2sen_idx[i]; /* Index of senone i within list of senones for mgau m */ fw = s->mgau2sen[m].feat_mixw; for (f = 0; f < s->n_feat; f++) { if (bio_fread((void *)pdf, sizeof(float32), n_cw, fp, byteswap, &chksum) != n_cw) { E_FATAL("bio_fread(%s) (arraydata) failed\n", file_name); } /* Normalize and floor */ if (vector_sum_norm (pdf, n_cw) == 0.0) n_err++; vector_floor (pdf, n_cw, mixwfloor); vector_sum_norm (pdf, n_cw); /* Convert to logs3, truncate to 8 bits, and store in s->pdf */ for (c = 0; c < n_cw; c++) { p = -logmath_log(logmath, pdf[c]); printf ("%f %d\n", pdf[c], p); fw[f].prob[j][c] = p; } } } if (n_err > 0) E_WARN("Weight normalization failed for %d senones\n", n_err); ckd_free (pdf); if (chksum_present) bio_verify_chksum (fp, byteswap, chksum); if (fread (&eofchk, 1, 1, fp) == 1) E_FATAL("More data than expected in %s\n", file_name); fclose(fp); E_INFO("Read mixture weights for %d senones: %d features x %d codewords\n", s->n_sen, s->n_feat, n_cw); return 0; }