static int32 gauden_param_read(float32 ***** out_param, /* Alloc space iff *out_param == 0 */ int32 * out_n_mgau, int32 * out_n_feat, int32 * out_n_density, int32 ** out_veclen, const char *file_name) { char tmp; FILE *fp; int32 i, j, k, l, n, blk; int32 n_mgau; int32 n_feat; int32 n_density; int32 *veclen; int32 byteswap, chksum_present; float32 ****out; float32 *buf; char **argname, **argval; uint32 chksum; E_INFO("Reading mixture gaussian parameter: %s\n", file_name); if ((fp = fopen(file_name, "rb")) == 0) E_FATAL_SYSTEM("Failed to open file '%s' for reading", file_name); /* Read header, including argument-value info and 32-bit byteorder magic */ if (bio_readhdr(fp, &argname, &argval, &byteswap) < 0) E_FATAL("Failed to read header from file '%s'\n", file_name); /* Parse argument-value list */ chksum_present = 0; for (i = 0; argname[i]; i++) { if (strcmp(argname[i], "version") == 0) { if (strcmp(argval[i], GAUDEN_PARAM_VERSION) != 0) E_WARN("Version mismatch(%s): %s, expecting %s\n", file_name, argval[i], GAUDEN_PARAM_VERSION); } else if (strcmp(argname[i], "chksum0") == 0) { chksum_present = 1; /* Ignore the associated value */ } } bio_hdrarg_free(argname, argval); argname = argval = 0; chksum = 0; /* #Codebooks */ if (bio_fread(&n_mgau, sizeof(int32), 1, fp, byteswap, &chksum) != 1) E_FATAL("fread(%s) (#codebooks) failed\n", file_name); *out_n_mgau = n_mgau; /* #Features/codebook */ if (bio_fread(&n_feat, sizeof(int32), 1, fp, byteswap, &chksum) != 1) E_FATAL("fread(%s) (#features) failed\n", file_name); *out_n_feat = n_feat; /* #Gaussian densities/feature in each codebook */ if (bio_fread(&n_density, sizeof(int32), 1, fp, byteswap, &chksum) != 1) E_FATAL("fread(%s) (#density/codebook) failed\n", file_name); *out_n_density = n_density; /* #Dimensions in each feature stream */ veclen = (int32*)ckd_calloc(n_feat, sizeof(uint32)); *out_veclen = veclen; if (bio_fread(veclen, sizeof(int32), n_feat, fp, byteswap, &chksum) != n_feat) E_FATAL("fread(%s) (feature-lengths) failed\n", file_name); /* blk = total vector length of all feature streams */ for (i = 0, blk = 0; i < n_feat; i++) blk += veclen[i]; /* #Floats to follow; for the ENTIRE SET of CODEBOOKS */ if (bio_fread(&n, sizeof(int32), 1, fp, byteswap, &chksum) != 1) E_FATAL("fread(%s) (total #floats) failed\n", file_name); if (n != n_mgau * n_density * blk) { E_FATAL ("%s: #mfcc_ts(%d) doesn't match dimensions: %d x %d x %d\n", file_name, n, n_mgau, n_density, blk); } /* Allocate memory for mixture gaussian densities if not already allocated */ if (!(*out_param)) { out = (float32 ****) ckd_calloc_3d(n_mgau, n_feat, n_density, sizeof(float32 *)); buf = (float32 *) ckd_calloc(n, sizeof(float32)); for (i = 0, l = 0; i < n_mgau; i++) { for (j = 0; j < n_feat; j++) { for (k = 0; k < n_density; k++) { out[i][j][k] = &buf[l]; l += veclen[j]; } } } } else { out = (float32 ****) *out_param; buf = out[0][0][0]; } /* Read mixture gaussian densities data */ if (bio_fread(buf, sizeof(float32), n, fp, byteswap, &chksum) != n) E_FATAL("fread(%s) (densitydata) failed\n", file_name); if (chksum_present) bio_verify_chksum(fp, byteswap, chksum); if (fread(&tmp, 1, 1, fp) == 1) E_FATAL("More data than expected in %s\n", file_name); fclose(fp); *out_param = out; E_INFO("%d codebook, %d feature, size: \n", n_mgau, n_feat); for (i = 0; i < n_feat; i++) E_INFO(" %dx%d\n", n_density, veclen[i]); return 0; }
static int32 senone_mixw_read(senone_t * s, char const *file_name, logmath_t *lmath) { char eofchk; FILE *fp; int32 byteswap, chksum_present; uint32 chksum; float32 *pdf; int32 i, f, c, p, n_err; char **argname, **argval; E_INFO("Reading senone mixture weights: %s\n", file_name); if ((fp = fopen(file_name, "rb")) == NULL) E_FATAL_SYSTEM("Failed to open mixture weights file '%s' for reading", file_name); /* Read header, including argument-value info and 32-bit byteorder magic */ if (bio_readhdr(fp, &argname, &argval, &byteswap) < 0) E_FATAL("Failed to read header from file '%s'\n", file_name); /* Parse argument-value list */ chksum_present = 0; for (i = 0; argname[i]; i++) { if (strcmp(argname[i], "version") == 0) { if (strcmp(argval[i], MIXW_PARAM_VERSION) != 0) E_WARN("Version mismatch(%s): %s, expecting %s\n", file_name, argval[i], MIXW_PARAM_VERSION); } else if (strcmp(argname[i], "chksum0") == 0) { chksum_present = 1; /* Ignore the associated value */ } } bio_hdrarg_free(argname, argval); argname = argval = NULL; chksum = 0; /* Read #senones, #features, #codewords, arraysize */ if ((bio_fread(&(s->n_sen), sizeof(int32), 1, fp, byteswap, &chksum) != 1) || (bio_fread(&(s->n_feat), sizeof(int32), 1, fp, byteswap, &chksum) != 1) || (bio_fread(&(s->n_cw), sizeof(int32), 1, fp, byteswap, &chksum) != 1) || (bio_fread(&i, sizeof(int32), 1, fp, byteswap, &chksum) != 1)) { E_FATAL("bio_fread(%s) (arraysize) failed\n", file_name); } if (i != s->n_sen * s->n_feat * s->n_cw) { E_FATAL ("%s: #float32s(%d) doesn't match dimensions: %d x %d x %d\n", file_name, i, s->n_sen, s->n_feat, s->n_cw); } /* * Compute #LSB bits to be dropped to represent mixwfloor with 8 bits. * All PDF values will be truncated (in the LSB positions) by these many bits. */ if ((s->mixwfloor <= 0.0) || (s->mixwfloor >= 1.0)) E_FATAL("mixwfloor (%e) not in range (0, 1)\n", s->mixwfloor); /* Use a fixed shift for compatibility with everything else. */ E_INFO("Truncating senone logs3(pdf) values by %d bits\n", SENSCR_SHIFT); /* * Allocate memory for senone PDF data. Organize normally or transposed depending on * s->n_gauden. */ if (s->n_gauden > 1) { E_INFO("Not transposing mixture weights in memory\n"); s->pdf = (senprob_t ***) ckd_calloc_3d(s->n_sen, s->n_feat, s->n_cw, sizeof(senprob_t)); } else { E_INFO("Transposing mixture weights in memory\n"); s->pdf = (senprob_t ***) ckd_calloc_3d(s->n_feat, s->n_cw, s->n_sen, sizeof(senprob_t)); } /* Temporary structure to read in floats */ pdf = (float32 *) ckd_calloc(s->n_cw, sizeof(float32)); /* Read senone probs data, normalize, floor, convert to logs3, truncate to 8 bits */ n_err = 0; for (i = 0; i < s->n_sen; i++) { for (f = 0; f < s->n_feat; f++) { if (bio_fread ((void *) pdf, sizeof(float32), s->n_cw, fp, byteswap, &chksum) != s->n_cw) { E_FATAL("bio_fread(%s) (arraydata) failed\n", file_name); } /* Normalize and floor */ if (vector_sum_norm(pdf, s->n_cw) <= 0.0) n_err++; vector_floor(pdf, s->n_cw, s->mixwfloor); vector_sum_norm(pdf, s->n_cw); /* Convert to logs3, truncate to 8 bits, and store in s->pdf */ for (c = 0; c < s->n_cw; c++) { p = -(logmath_log(lmath, pdf[c])); p += (1 << (SENSCR_SHIFT - 1)) - 1; /* Rounding before truncation */ if (s->n_gauden > 1) s->pdf[i][f][c] = (p < (255 << SENSCR_SHIFT)) ? (p >> SENSCR_SHIFT) : 255; else s->pdf[f][c][i] = (p < (255 << SENSCR_SHIFT)) ? (p >> SENSCR_SHIFT) : 255; } } }
static int32 senone_mgau_map_read(senone_t * s, char const *file_name) { FILE *fp; int32 byteswap, chksum_present, n_gauden_present; uint32 chksum; int32 i; char eofchk; char **argname, **argval; void *ptr; float32 v; E_INFO("Reading senone gauden-codebook map file: %s\n", file_name); if ((fp = fopen(file_name, "rb")) == NULL) E_FATAL_SYSTEM("Failed to open map file '%s' for reading", file_name); /* Read header, including argument-value info and 32-bit byteorder magic */ if (bio_readhdr(fp, &argname, &argval, &byteswap) < 0) E_FATAL("Failed to read header from file '%s'\n", file_name); /* Parse argument-value list */ chksum_present = 0; n_gauden_present = 0; for (i = 0; argname[i]; i++) { if (strcmp(argname[i], "version") == 0) { if (strcmp(argval[i], SPDEF_PARAM_VERSION) != 0) { E_WARN("Version mismatch(%s): %s, expecting %s\n", file_name, argval[i], SPDEF_PARAM_VERSION); } /* HACK!! Convert version# to float32 and take appropriate action */ if (sscanf(argval[i], "%f", &v) != 1) E_FATAL("%s: Bad version no. string: %s\n", file_name, argval[i]); n_gauden_present = (v > 1.1) ? 1 : 0; } else if (strcmp(argname[i], "chksum0") == 0) { chksum_present = 1; /* Ignore the associated value */ } } bio_hdrarg_free(argname, argval); argname = argval = NULL; chksum = 0; /* Read #gauden (if version matches) */ if (n_gauden_present) { E_INFO("Reading number of codebooks from %s\n", file_name); if (bio_fread (&(s->n_gauden), sizeof(int32), 1, fp, byteswap, &chksum) != 1) E_FATAL("fread(%s) (#gauden) failed\n", file_name); } /* Read 1d array data */ if (bio_fread_1d(&ptr, sizeof(uint32), &(s->n_sen), fp, byteswap, &chksum) < 0) { E_FATAL("bio_fread_1d(%s) failed\n", file_name); } s->mgau = ptr; E_INFO("Mapping %d senones to %d codebooks\n", s->n_sen, s->n_gauden); /* Infer n_gauden if not present in this version */ if (!n_gauden_present) { s->n_gauden = 1; for (i = 0; i < s->n_sen; i++) if (s->mgau[i] >= s->n_gauden) s->n_gauden = s->mgau[i] + 1; } if (chksum_present) bio_verify_chksum(fp, byteswap, chksum); if (fread(&eofchk, 1, 1, fp) == 1) E_FATAL("More data than expected in %s: %d\n", file_name, eofchk); fclose(fp); E_INFO("Read %d->%d senone-codebook mappings\n", s->n_sen, s->n_gauden); return 1; }
int s3gau_read_maybe_full(const char *fn, vector_t *****out, uint32 *out_n_mgau, uint32 *out_n_feat, uint32 *out_n_density, uint32 **out_veclen, uint32 need_full) { FILE *fp; const char *do_chk; const char *ver; uint32 n_mgau, n_feat, n_density; uint32 *veclen, maxveclen; uint32 blk, i, j, k, l, r, n; uint32 chksum = 0; uint32 sv_chksum, ignore = 0; float32 *raw; vector_t ****o; uint32 swap; fp = s3open(fn, "rb", &swap); if (fp == NULL) return S3_ERROR; /* check version id */ ver = s3get_gvn_fattr("version"); if (ver) { if (strcmp(ver, GAU_FILE_VERSION) != 0) { E_FATAL("Version mismatch for %s, file ver: %s != reader ver: %s\n", fn, ver, GAU_FILE_VERSION); } } else { E_FATAL("No version attribute for %s\n", fn); } /* if do_chk is non-NULL, there is a checksum after the data in the file */ do_chk = s3get_gvn_fattr("chksum0"); if (do_chk && !strcmp(do_chk, "no")) { do_chk = NULL; } if (bio_fread(&n_mgau, sizeof(uint32), 1, fp, swap, &chksum) != 1) { goto error; } if (bio_fread(&n_feat, sizeof(uint32), 1, fp, swap, &chksum) != 1) { goto error; } if (bio_fread(&n_density, sizeof(uint32), 1, fp, swap, &chksum) != 1) { goto error; } veclen = ckd_calloc(n_feat, sizeof(uint32)); if (bio_fread(veclen, sizeof(uint32), n_feat, fp, swap, &chksum) != n_feat) { goto error; } if (bio_fread_1d((void **)&raw, sizeof(float32), &n, fp, swap, &chksum) < 0) { ckd_free(veclen); goto error; } for (i = 0, blk = 0, maxveclen = 0; i < n_feat; i++) { blk += veclen[i] * veclen[i]; if (veclen[i] > maxveclen) maxveclen = veclen[i]; } if (n != n_mgau * n_density * blk) { if (need_full) E_ERROR("Failed to read full covariance file %s (expected %d values, got %d)\n", fn, n_mgau * n_density * blk, n); goto error; } o = (vector_t ****)ckd_calloc_4d(n_mgau, n_feat, n_density, maxveclen, sizeof(vector_t)); for (i = 0, r = 0; i < n_mgau; i++) { for (j = 0; j < n_feat; j++) { for (k = 0; k < n_density; k++) { for (l = 0; l < veclen[j]; l++) { o[i][j][k][l] = &raw[r]; r += veclen[j]; } } } } if (do_chk) { /* See if the checksum in the file matches that which was computed from the read data */ if (bio_fread(&sv_chksum, sizeof(uint32), 1, fp, swap, &ignore) != 1) { goto error; } if (sv_chksum != chksum) { E_FATAL("Checksum error; read corrupt data.\n"); } } *out = o; *out_n_mgau = n_mgau; *out_n_feat = n_feat; *out_n_density = n_density; *out_veclen = veclen; s3close(fp); E_INFO("Read %s [%ux%ux%u array of full matrices]\n", fn, n_mgau, n_feat, n_density); return S3_SUCCESS; error: if (fp) s3close(fp); return S3_ERROR; }
static int32 interp_read(interp_t * ip, const char *file_name) { FILE *fp; int32 byteswap, chksum_present; int32 i; char eofchk; float f; char **argname, **argval; uint32 chksum; E_INFO("Reading interpolation weights: %s\n", file_name); if ((fp = fopen(file_name, "rb")) == NULL) E_FATAL_SYSTEM("fopen(%s,rb) failed\n", file_name); /* Read header, including argument-value info and 32-bit byteorder magic */ if (bio_readhdr(fp, &argname, &argval, &byteswap) < 0) E_FATAL("bio_readhdr(%s) failed\n", file_name); /* Parse argument-value list */ chksum_present = 0; for (i = 0; argname[i]; i++) { if (strcmp(argname[i], "version") == 0) { if (strcmp(argval[i], INTERP_VERSION) != 0) E_WARN("Version mismatch(%s): %s, expecting %s\n", file_name, argval[i], INTERP_VERSION); } else if (strcmp(argname[i], "chksum0") == 0) { chksum_present = 1; /* Ignore the associated value */ } } bio_hdrarg_free(argname, argval); argname = argval = NULL; chksum = 0; /* Read #senones */ if (bio_fread(&(ip->n_sen), sizeof(int32), 1, fp, byteswap, &chksum) != 1) E_FATAL("fread(%s) (arraysize) failed\n", file_name); if (ip->n_sen <= 0) E_FATAL("%s: arraysize= %d in header\n", file_name, ip->n_sen); ip->wt = (struct interp_wt_s *) ckd_calloc(ip->n_sen, sizeof(struct interp_wt_s)); for (i = 0; i < ip->n_sen; i++) { if (bio_fread(&f, sizeof(float32), 1, fp, byteswap, &chksum) != 1) E_FATAL("fread(%s) (arraydata) failed\n", file_name); if ((f < 0.0) || (f > 1.0)) E_FATAL("%s: interpolation weight(%d)= %e\n", file_name, i, f); ip->wt[i].cd = (f == 0.0) ? S3_LOGPROB_ZERO : logs3(ip->logmath, f); ip->wt[i].ci = (f == 1.0) ? S3_LOGPROB_ZERO : logs3(ip->logmath, 1.0 - f); } if (chksum_present) bio_verify_chksum(fp, byteswap, chksum); if (fread(&eofchk, 1, 1, fp) == 1) E_FATAL("More data than expected in %s\n", file_name); fclose(fp); E_INFO("Read %d interpolation weights\n", ip->n_sen); return 1; }
tmat_t *tmat_init (char *file_name, float64 tpfloor) { char tmp; int32 n_src, n_dst; FILE *fp; int32 byteswap, chksum_present; uint32 chksum; float32 **tp; int32 i, j, k, tp_per_tmat; char **argname, **argval; tmat_t *t; E_INFO("Reading HMM transition probability matrices: %s\n", file_name); t = (tmat_t *) ckd_calloc (1, sizeof(tmat_t)); if ((fp = fopen(file_name, "rb")) == NULL) E_FATAL_SYSTEM("fopen(%s,rb) failed\n", file_name); /* Read header, including argument-value info and 32-bit byteorder magic */ if (bio_readhdr (fp, &argname, &argval, &byteswap) < 0) E_FATAL("bio_readhdr(%s) failed\n", file_name); /* Parse argument-value list */ chksum_present = 0; for (i = 0; argname[i]; i++) { if (strcmp (argname[i], "version") == 0) { if (strcmp(argval[i], TMAT_PARAM_VERSION) != 0) E_WARN("Version mismatch(%s): %s, expecting %s\n", file_name, argval[i], TMAT_PARAM_VERSION); } else if (strcmp (argname[i], "chksum0") == 0) { chksum_present = 1; /* Ignore the associated value */ } } bio_hdrarg_free (argname, argval); argname = argval = NULL; chksum = 0; /* Read #tmat, #from-states, #to-states, arraysize */ if ((bio_fread (&(t->n_tmat), sizeof(int32), 1, fp, byteswap, &chksum) != 1) || (bio_fread (&n_src, sizeof(int32), 1, fp, byteswap, &chksum) != 1) || (bio_fread (&n_dst, sizeof(int32), 1, fp, byteswap, &chksum) != 1) || (bio_fread (&i, sizeof(int32), 1, fp, byteswap, &chksum) != 1)) { E_FATAL("bio_fread(%s) (arraysize) failed\n", file_name); } if (t->n_tmat >= MAX_S3TMATID) E_FATAL("%s: #tmat (%d) exceeds limit (%d)\n", file_name, t->n_tmat, MAX_S3TMATID); if (n_dst != n_src+1) E_FATAL("%s: #from-states(%d) != #to-states(%d)-1\n", file_name, n_src, n_dst); t->n_state = n_src; if (i != t->n_tmat * n_src * n_dst) { E_FATAL("%s: #float32s(%d) doesn't match dimensions: %d x %d x %d\n", file_name, i, t->n_tmat, n_src, n_dst); } /* Allocate memory for tmat data */ t->tp = (int32 ***) ckd_calloc_3d (t->n_tmat, n_src, n_dst, sizeof(int32)); /* Temporary structure to read in the float data */ tp = (float32 **) ckd_calloc_2d (n_src, n_dst, sizeof(float32)); /* Read transition matrices, normalize and floor them, and convert to logs3 domain */ tp_per_tmat = n_src * n_dst; for (i = 0; i < t->n_tmat; i++) { if (bio_fread (tp[0], sizeof(float32), tp_per_tmat, fp, byteswap, &chksum) != tp_per_tmat) { E_FATAL("fread(%s) (arraydata) failed\n", file_name); } /* Normalize and floor */ for (j = 0; j < n_src; j++) { if (vector_sum_norm (tp[j], n_dst) == 0.0) E_ERROR("Normalization failed for tmat %d from state %d\n", i, j); vector_nz_floor (tp[j], n_dst, tpfloor); vector_sum_norm (tp[j], n_dst); /* Convert to logs3. Take care of special case when tp = 0.0! */ for (k = 0; k < n_dst; k++) t->tp[i][j][k] = (tp[j][k] == 0.0) ? S3_LOGPROB_ZERO : logs3(tp[j][k]); } } ckd_free_2d ((void **) tp); if (chksum_present) bio_verify_chksum (fp, byteswap, chksum); if (fread (&tmp, 1, 1, fp) == 1) E_ERROR("Non-empty file beyond end of data\n"); fclose(fp); E_INFO("Read %d transition matrices of size %dx%d\n", t->n_tmat, t->n_state, t->n_state+1); if (tmat_chk_uppertri (t) < 0) E_FATAL("Tmat not upper triangular\n"); return t; }
int s3gaucnt_read_full(const char *fn, vector_t ****out_wt_mean, vector_t *****out_wt_var, int32 *out_pass2var, float32 ****out_dnom, uint32 *out_n_cb, uint32 *out_n_feat, uint32 *out_n_density, uint32 **out_veclen) { uint32 rd_chksum = 0; uint32 sv_chksum; uint32 ignore; char *ver; char *do_chk; FILE *fp; uint32 swap; uint32 has_means; uint32 has_vars; uint32 pass2var; uint32 n_cb; uint32 n_feat; uint32 n_density; uint32 *veclen; float32 *buf; float32 ***dnom; uint32 n, i, b_i, j, k, l, d1, d2, d3; vector_t ***wt_mean = NULL; vector_t ****wt_var = NULL; fp = s3open(fn, "rb", &swap); if (fp == NULL) return S3_ERROR; /* check version id */ ver = s3get_gvn_fattr("version"); if (ver) { if (strcmp(ver, GAUCNT_FILE_VERSION) != 0) { E_FATAL("Version mismatch for %s, file ver: %s != reader ver: %s\n", fn, ver, GAUCNT_FILE_VERSION); } } else { E_FATAL("No version attribute for %s\n", fn); } /* if do_chk is non-NULL, there is a checksum after the data in the file */ do_chk = s3get_gvn_fattr("chksum0"); if (bio_fread((void *)&has_means, sizeof(uint32), 1, fp, swap, &rd_chksum) != 1) { return S3_ERROR; } if (bio_fread((void *)&has_vars, sizeof(uint32), 1, fp, swap, &rd_chksum) != 1) { return S3_ERROR; } if (bio_fread((void *)&pass2var, sizeof(uint32), 1, fp, swap, &rd_chksum) != 1) { return S3_ERROR; } if (bio_fread((void *)&n_cb, sizeof(uint32), 1, fp, swap, &rd_chksum) != 1) { return S3_ERROR; } if (bio_fread((void *)&n_density, sizeof(uint32), 1, fp, swap, &rd_chksum) != 1) { return S3_ERROR; } if (bio_fread_1d((void **)&veclen, sizeof(uint32), &n_feat, fp, swap, &rd_chksum) < 0) { return S3_ERROR; } if (has_means) { if (bio_fread_1d((void *)&buf, sizeof(float32), &n, fp, swap, &rd_chksum) < 0) { return S3_ERROR; } wt_mean = (vector_t ***)ckd_calloc_3d(n_cb, n_feat, n_density, sizeof(vector_t)); for (i = 0, b_i = 0; i < n_cb; i++) { for (j = 0; j < n_feat; j++) { for (k = 0; k < n_density; k++) { wt_mean[i][j][k] = &buf[b_i]; b_i += veclen[j]; } } } } if (has_vars) { uint32 blk, maxveclen; for (i = 0, blk = 0, maxveclen = 0; i < n_feat; i++) { blk += veclen[i]; if (veclen[i] > maxveclen) maxveclen = veclen[i]; } if (bio_fread_1d((void *)&buf, sizeof(float32), &n, fp, swap, &rd_chksum) < 0) { return S3_ERROR; } assert(n == n_cb * n_density * blk * blk); wt_var = (vector_t ****)ckd_calloc_4d(n_cb, n_feat, n_density, maxveclen, sizeof(vector_t)); for (i = 0, b_i = 0; i < n_cb; i++) { for (j = 0; j < n_feat; j++) { for (k = 0; k < n_density; k++) { for (l = 0; l < veclen[j]; l++) { wt_var[i][j][k][l] = &buf[b_i]; b_i += veclen[j]; } } } } } if (bio_fread_3d((void ****)&dnom, sizeof(float32), &d1, &d2, &d3, fp, swap, &rd_chksum) < 0) { return S3_ERROR; } assert(d1 == n_cb); assert(d2 == n_feat); assert(d3 == n_density); if (do_chk) { /* See if the checksum in the file matches that which was computed from the read data */ if (bio_fread(&sv_chksum, sizeof(uint32), 1, fp, swap, &ignore) != 1) { s3close(fp); return S3_ERROR; } if (sv_chksum != rd_chksum) { E_FATAL("Checksum error; read corrupt data.\n"); } } s3close(fp); *out_wt_mean = wt_mean; *out_wt_var = wt_var; *out_pass2var = pass2var; *out_dnom = dnom; *out_n_cb = n_cb; *out_n_feat = n_feat; *out_n_density = n_density; *out_veclen = veclen; E_INFO("Read %s%s%s%s [%ux%ux%u vector arrays]\n", fn, (has_means ? " with means" : ""), (has_vars ? " with vars" : ""), (has_vars && pass2var ? " (2pass)" : ""), n_cb, n_feat, n_density); return S3_SUCCESS; }
static int32 read_mixw(s2_semi_mgau_t * s, char const *file_name, double SmoothMin) { char **argname, **argval; char eofchk; FILE *fp; int32 byteswap, chksum_present; uint32 chksum; float32 *pdf; int32 i, f, c, n; int32 n_sen; int32 n_feat; int32 n_comp; int32 n_err; E_INFO("Reading mixture weights file '%s'\n", file_name); if ((fp = fopen(file_name, "rb")) == NULL) E_FATAL("fopen(%s,rb) failed\n", file_name); /* Read header, including argument-value info and 32-bit byteorder magic */ if (bio_readhdr(fp, &argname, &argval, &byteswap) < 0) E_FATAL("bio_readhdr(%s) failed\n", file_name); /* Parse argument-value list */ chksum_present = 0; for (i = 0; argname[i]; i++) { if (strcmp(argname[i], "version") == 0) { if (strcmp(argval[i], MGAU_MIXW_VERSION) != 0) E_WARN("Version mismatch(%s): %s, expecting %s\n", file_name, argval[i], MGAU_MIXW_VERSION); } else if (strcmp(argname[i], "chksum0") == 0) { chksum_present = 1; /* Ignore the associated value */ } } bio_hdrarg_free(argname, argval); argname = argval = NULL; chksum = 0; /* Read #senones, #features, #codewords, arraysize */ if ((bio_fread(&n_sen, sizeof(int32), 1, fp, byteswap, &chksum) != 1) || (bio_fread(&n_feat, sizeof(int32), 1, fp, byteswap, &chksum) != 1) || (bio_fread(&n_comp, sizeof(int32), 1, fp, byteswap, &chksum) != 1) || (bio_fread(&n, sizeof(int32), 1, fp, byteswap, &chksum) != 1)) { E_FATAL("bio_fread(%s) (arraysize) failed\n", file_name); } if (n_feat != s->n_feat) E_FATAL("#Features streams(%d) != %d\n", n_feat, s->n_feat); if (n != n_sen * n_feat * n_comp) { E_FATAL ("%s: #float32s(%d) doesn't match header dimensions: %d x %d x %d\n", file_name, i, n_sen, n_feat, n_comp); } /* n_sen = number of mixture weights per codeword, which is * fixed at the number of senones since we have only one codebook. */ s->n_sen = n_sen; /* Quantized mixture weight arrays. */ s->mixw = ckd_calloc_3d(s->n_feat, s->n_density, n_sen, sizeof(***s->mixw)); /* Temporary structure to read in floats before conversion to (int32) logs3 */ pdf = (float32 *) ckd_calloc(n_comp, sizeof(float32)); /* Read senone probs data, normalize, floor, convert to logs3, truncate to 8 bits */ n_err = 0; for (i = 0; i < n_sen; i++) { for (f = 0; f < n_feat; f++) { if (bio_fread((void *) pdf, sizeof(float32), n_comp, fp, byteswap, &chksum) != n_comp) { E_FATAL("bio_fread(%s) (arraydata) failed\n", file_name); } /* Normalize and floor */ if (vector_sum_norm(pdf, n_comp) <= 0.0) n_err++; vector_floor(pdf, n_comp, SmoothMin); vector_sum_norm(pdf, n_comp); /* Convert to LOG, quantize, and transpose */ for (c = 0; c < n_comp; c++) { int32 qscr; qscr = -logmath_log(s->lmath_8b, pdf[c]); if ((qscr > MAX_NEG_MIXW) || (qscr < 0)) qscr = MAX_NEG_MIXW; s->mixw[f][c][i] = qscr; } } } if (n_err > 0) E_WARN("Weight normalization failed for %d senones\n", n_err); ckd_free(pdf); if (chksum_present) bio_verify_chksum(fp, byteswap, chksum); if (fread(&eofchk, 1, 1, fp) == 1) E_FATAL("More data than expected in %s\n", file_name); fclose(fp); E_INFO("Read %d x %d x %d mixture weights\n", n_sen, n_feat, n_comp); return n_sen; }
/* Read a Sphinx3 mean or variance file. */ static int32 s3_read_mgau(s2_semi_mgau_t *s, const char *file_name, float32 ***out_cb) { char tmp; FILE *fp; int32 i, blk, n; int32 n_mgau; int32 n_feat; int32 n_density; int32 *veclen; int32 byteswap, chksum_present; char **argname, **argval; uint32 chksum; E_INFO("Reading S3 mixture gaussian file '%s'\n", file_name); if ((fp = fopen(file_name, "rb")) == NULL) E_FATAL("fopen(%s,rb) failed\n", file_name); /* Read header, including argument-value info and 32-bit byteorder magic */ if (bio_readhdr(fp, &argname, &argval, &byteswap) < 0) E_FATAL("bio_readhdr(%s) failed\n", file_name); /* Parse argument-value list */ chksum_present = 0; for (i = 0; argname[i]; i++) { if (strcmp(argname[i], "version") == 0) { if (strcmp(argval[i], MGAU_PARAM_VERSION) != 0) E_WARN("Version mismatch(%s): %s, expecting %s\n", file_name, argval[i], MGAU_PARAM_VERSION); } else if (strcmp(argname[i], "chksum0") == 0) { chksum_present = 1; /* Ignore the associated value */ } } bio_hdrarg_free(argname, argval); argname = argval = NULL; chksum = 0; /* #Codebooks */ if (bio_fread(&n_mgau, sizeof(int32), 1, fp, byteswap, &chksum) != 1) E_FATAL("fread(%s) (#codebooks) failed\n", file_name); if (n_mgau != 1) { E_ERROR("%s: #codebooks (%d) != 1\n", file_name, n_mgau); fclose(fp); return -1; } /* #Features/codebook */ if (bio_fread(&n_feat, sizeof(int32), 1, fp, byteswap, &chksum) != 1) E_FATAL("fread(%s) (#features) failed\n", file_name); if (s->n_feat == 0) s->n_feat = n_feat; else if (n_feat != s->n_feat) E_FATAL("#Features streams(%d) != %d\n", n_feat, s->n_feat); /* #Gaussian densities/feature in each codebook */ if (bio_fread(&n_density, sizeof(int32), 1, fp, byteswap, &chksum) != 1) E_FATAL("fread(%s) (#density/codebook) failed\n", file_name); if (s->n_density == 0) s->n_density = n_density; else if (n_density != s->n_density) E_FATAL("%s: Number of densities per feature(%d) != %d\n", file_name, n_mgau, s->n_density); /* Vector length of feature stream */ veclen = ckd_calloc(s->n_feat, sizeof(int32)); if (bio_fread(veclen, sizeof(int32), s->n_feat, fp, byteswap, &chksum) != s->n_feat) E_FATAL("fread(%s) (feature vector-length) failed\n", file_name); for (i = 0, blk = 0; i < s->n_feat; ++i) { if (s->veclen[i] == 0) s->veclen[i] = veclen[i]; else if (veclen[i] != s->veclen[i]) E_FATAL("feature stream length %d is inconsistent (%d != %d)\n", i, veclen[i], s->veclen[i]); blk += veclen[i]; } /* #Floats to follow; for the ENTIRE SET of CODEBOOKS */ if (bio_fread(&n, sizeof(int32), 1, fp, byteswap, &chksum) != 1) E_FATAL("fread(%s) (total #floats) failed\n", file_name); if (n != n_mgau * n_density * blk) E_FATAL ("%s: #float32s(%d) doesn't match dimensions: %d x %d x %d\n", file_name, n, n_mgau, n_density, blk); *out_cb = ckd_calloc(s->n_feat, sizeof(float32 *)); for (i = 0; i < s->n_feat; ++i) { (*out_cb)[i] = (float32 *) ckd_calloc(n_density * veclen[i], sizeof(float32)); if (bio_fread ((*out_cb)[i], sizeof(float32), n_density * veclen[i], fp, byteswap, &chksum) != n_density * veclen[i]) E_FATAL("fread(%s, %d) of feat %d failed\n", file_name, n_density * veclen[i], i); } ckd_free(veclen); if (chksum_present) bio_verify_chksum(fp, byteswap, chksum); if (fread(&tmp, 1, 1, fp) == 1) E_FATAL("%s: More data than expected\n", file_name); fclose(fp); E_INFO("%d mixture Gaussians, %d components, %d feature streams, veclen %d\n", n_mgau, n_density, n_feat, blk); return n; }
int s3map_read(const char *fn, void **out_map, uint32 *out_n_dom, uint32 *out_n_rng, size_t map_elem_size) { uint32 rd_chksum = 0; uint32 sv_chksum; uint32 ignore; char *ver; char *do_chk; FILE *fp; uint32 swap; fp = s3open(fn, "rb", &swap); if (fp == NULL) { return S3_ERROR; } /* check version id */ ver = s3get_gvn_fattr("version"); if (ver) { if (strcmp(ver, MAP_FILE_VERSION) != 0) { E_FATAL("Version mismatch for %s, file ver: %s != reader ver: %s\n", fn, ver, MAP_FILE_VERSION); } } else { E_FATAL("No version attribute for %s\n", fn); } do_chk = s3get_gvn_fattr("chksum0"); if (bio_fread(out_n_rng, sizeof(uint32), 1, fp, swap, &rd_chksum) != 1) { s3close(fp); return S3_ERROR; } if (bio_fread_1d(out_map, map_elem_size, out_n_dom, fp, swap, &rd_chksum) < 0) { s3close(fp); return S3_ERROR; } if (do_chk) { if (bio_fread(&sv_chksum, sizeof(uint32), 1, fp, swap, &ignore) != 1) { s3close(fp); return S3_ERROR; } if (sv_chksum != rd_chksum) { E_FATAL("Checksum error; read corrupted data.\n"); } } E_INFO("Read %s [%u mappings to %u]\n", fn, *out_n_dom, *out_n_rng); return S3_SUCCESS; }
/** Reading a regression map */ static int s3map_read(const char *fn, /**< The file name */ void **out_map, int32 * out_n_dom, int32 * out_n_rng, size_t map_elem_size) { uint32 rd_chksum = 0; uint32 sv_chksum; uint32 ignore; char *ver; char *do_chk; FILE *fp; int32 swap; char **argname, **argval; int i; if ((fp = fopen(fn, "rb")) == NULL) E_FATAL_SYSTEM("fopen(%s,rb) failed\n", fn); /* Read header, including argument-value info and 32-bit byteorder magic */ if (bio_readhdr(fp, &argname, &argval, &swap) < 0) E_FATAL("bio_readhdr(%s) failed\n", fn); /* Parse argument-value list */ ver = do_chk = NULL; for (i = 0; argname[i]; i++) { if (strcmp(argname[i], "version") == 0) { if (strcmp(argval[i], MAP_FILE_VERSION) != 0) { E_FATAL("Version mismatch(%s): %s, expecting %s\n", fn, argval[i], MAP_FILE_VERSION); } ver = argval[i]; } else if (strcmp(argname[i], "chksum0") == 0) { do_chk = argval[i]; } } if (ver == NULL) E_FATAL("No version attribute for %s\n", fn); bio_hdrarg_free(argname, argval); argname = argval = NULL; if (bio_fread(out_n_rng, sizeof(uint32), 1, fp, swap, &rd_chksum) != 1) { fclose(fp); return S3_ERROR; } if (bio_fread_1d(out_map, map_elem_size, out_n_dom, fp, swap, &rd_chksum) < 0) { fclose(fp); return S3_ERROR; } if (do_chk) { if (bio_fread(&sv_chksum, sizeof(uint32), 1, fp, swap, &ignore) != 1) { fclose(fp); return S3_ERROR; } if (sv_chksum != rd_chksum) { E_FATAL("Checksum error; read corrupted data.\n"); } } E_INFO("Read %s [%u mappings to %u]\n", fn, *out_n_dom, *out_n_rng); return S3_SUCCESS; }
int s3mixw_read(const char *fn, float32 ****out_mixw, uint32 *out_n_mixw, uint32 *out_n_feat, uint32 *out_n_density) { uint32 rd_chksum = 0; uint32 sv_chksum; uint32 swap; uint32 ignore; char *ver; char *do_chk; FILE *fp; fp = s3open(fn, "rb", &swap); if (fp == NULL) return S3_ERROR; /* check version id */ ver = s3get_gvn_fattr("version"); if (ver) { if (strcmp(ver, MIXW_FILE_VERSION) != 0) { E_FATAL("Version mismatch for %s, file ver: %s != reader ver: %s\n", fn, ver, MIXW_FILE_VERSION); } } else { E_FATAL("No version attribute for %s\n", fn); } /* if do_chk is non-NULL, there is a checksum after the data in the file */ do_chk = s3get_gvn_fattr("chksum0"); /* Read the mixing weight array */ if (bio_fread_3d((void ****)out_mixw, sizeof(float32), out_n_mixw, out_n_feat, out_n_density, fp, swap, &rd_chksum) < 0) { s3close(fp); return S3_ERROR; } if (do_chk) { /* See if the checksum in the file matches that which was computed from the read data */ if (bio_fread(&sv_chksum, sizeof(uint32), 1, fp, swap, &ignore) != 1) { s3close(fp); return S3_ERROR; } if (sv_chksum != rd_chksum) { E_FATAL("Checksum error; read corrupt data.\n"); } } s3close(fp); E_INFO("Read %s [%ux%ux%u array]\n", fn, *out_n_mixw, *out_n_feat, *out_n_density); return S3_SUCCESS; }
static int32 senone_mgau_map_read (senone_t *s, const char *file_name) { FILE *fp; int32 byteswap, chksum_present, n_mgau_present; uint32 chksum; int32 i; char eofchk; char **argname, **argval; float32 v; E_INFO("Reading senone-codebook map file: %s\n", file_name); if ((fp = fopen(file_name, "rb")) == NULL) E_FATAL_SYSTEM("fopen(%s,rb) failed\n", file_name); /* Read header, including argument-value info and 32-bit byteorder magic */ if (bio_readhdr (fp, &argname, &argval, &byteswap) < 0) E_FATAL("bio_readhdr(%s) failed\n", file_name); /* Parse argument-value list */ chksum_present = 0; n_mgau_present = 0; for (i = 0; argname[i]; i++) { if (strcmp (argname[i], "version") == 0) { if (strcmp(argval[i], SPDEF_PARAM_VERSION) != 0) { E_WARN("Version mismatch(%s): %s, expecting %s\n", file_name, argval[i], SPDEF_PARAM_VERSION); } /* HACK!! Convert version# to float32 and take appropriate action */ if (sscanf (argval[i], "%f", &v) != 1) E_FATAL("%s: Bad version no. string: %s\n", file_name, argval[i]); n_mgau_present = (v > 1.1) ? 1 : 0; } else if (strcmp (argname[i], "chksum0") == 0) { chksum_present = 1; /* Ignore the associated value */ } } bio_hdrarg_free (argname, argval); argname = argval = NULL; chksum = 0; /* Read #gauden (if version matches) */ if (n_mgau_present) { if (bio_fread (&(s->n_mgau), sizeof(int32), 1, fp, byteswap, &chksum) != 1) E_FATAL("fread(%s) (#gauden) failed\n", file_name); } /* Read 1d array data; s->sen2mgau allocated by called function */ if (bio_fread_1d ((void **)(&s->sen2mgau), sizeof(int32), &(s->n_sen), fp, byteswap, &chksum) < 0) { E_FATAL("bio_fread_1d(%s) failed\n", file_name); } /* Infer n_mgau if not present in this version */ if (! n_mgau_present) { s->n_mgau = 1; for (i = 0; i < s->n_sen; i++) { if (s->sen2mgau[i] >= s->n_mgau) s->n_mgau = s->sen2mgau[i]+1; } } if (s->n_sen >= MAX_SENID) E_FATAL("%s: #senones (%d) exceeds limit (%d)\n", file_name, s->n_sen, MAX_SENID); if (s->n_mgau >= MAX_MGAUID) E_FATAL("%s: #gauden (%d) exceeds limit (%d)\n", file_name, s->n_mgau, MAX_MGAUID); /* Check for validity of mappings */ for (i = 0; i < s->n_sen; i++) { if ((s->sen2mgau[i] >= s->n_mgau) || NOT_MGAUID(s->sen2mgau[i])) E_FATAL("Bad sen2mgau[%d]= %d, out of range [0, %d)\n", i, s->sen2mgau[i], s->n_mgau); } if (chksum_present) bio_verify_chksum (fp, byteswap, chksum); if (fread (&eofchk, 1, 1, fp) == 1) E_FATAL("More data than expected in %s\n", file_name); fclose(fp); E_INFO("Read %d->%d senone-codebook mappings\n", s->n_sen, s->n_mgau); return 0; }
static int32 senone_mixw_read(logmath_t * logmath, senone_t *s, const char *file_name, float64 mixwfloor) { FILE *fp; char **argname, **argval; int32 byteswap, chksum_present; uint32 chksum; float32 *pdf; int32 i, j, f, m, c, p, n_sen, n_err, n_cw, nval; char eofchk; mixw_t *fw; E_INFO("Reading senone mixture weights: %s\n", file_name); if ((fp = fopen(file_name, "rb")) == NULL) E_FATAL_SYSTEM("fopen(%s,rb) failed\n", file_name); /* Read header, including argument-value info and 32-bit byteorder magic */ if (bio_readhdr (fp, &argname, &argval, &byteswap) < 0) E_FATAL("bio_readhdr(%s) failed\n", file_name); /* Parse argument-value list */ chksum_present = 0; for (i = 0; argname[i]; i++) { if (strcmp (argname[i], "version") == 0) { if (strcmp(argval[i], MIXW_PARAM_VERSION) != 0) E_WARN("Version mismatch(%s): %s, expecting %s\n", file_name, argval[i], MIXW_PARAM_VERSION); } else if (strcmp (argname[i], "chksum0") == 0) { chksum_present = 1; /* Ignore the associated value */ } } bio_hdrarg_free (argname, argval); argname = argval = NULL; chksum = 0; /* Read #senones, #features, #codewords, arraysize */ n_sen = s->n_sen; if ((bio_fread (&(s->n_sen), sizeof(int32), 1, fp, byteswap, &chksum) != 1) || (bio_fread (&(s->n_feat), sizeof(int32), 1, fp, byteswap, &chksum) != 1) || (bio_fread (&(n_cw), sizeof(int32), 1, fp, byteswap, &chksum) != 1) || (bio_fread (&nval, sizeof(int32), 1, fp, byteswap, &chksum) != 1)) { E_FATAL("bio_fread(%s) (arraysize) failed\n", file_name); } if ((n_sen != 0) && (s->n_sen != n_sen)) E_FATAL("#senones(%d) conflict with mapping file(%d)\n", s->n_sen, n_sen); if (s->n_sen >= MAX_SENID) E_FATAL("%s: #senones (%d) exceeds limit (%d)\n", file_name, s->n_sen, MAX_SENID); if (s->n_feat <= 0) E_FATAL("Bad #features: %d\n", s->n_feat); if (n_cw <= 0) E_FATAL("Bad #mixing-wts/senone: %d\n", n_cw); /* Allocate sen2mgau map if not yet done so (i.e. no explicit mapping file given */ if (! s->sen2mgau) { assert ((s->n_mgau == 0) || (s->n_mgau == 1)); s->sen2mgau = (uint32 *) ckd_calloc (s->n_sen, sizeof(int32)); if (s->n_mgau == 1) { /* Semicontinuous mode; all senones map to single, shared gaussian: 0 */ for (i = 0; i < s->n_sen; i++) s->sen2mgau[i] = 0; } else { /* Fully continuous mode; each senone maps to own parent gaussian */ s->n_mgau = s->n_sen; for (i = 0; i < s->n_sen; i++) s->sen2mgau[i] = i; } } else assert (s->n_mgau != 0); if (s->n_mgau >= MAX_MGAUID) E_FATAL("%s: #gauden (%d) exceeds limit (%d)\n", file_name, s->n_mgau, MAX_MGAUID); if (nval != s->n_sen * s->n_feat * n_cw) { E_FATAL("%s: #float32 values(%d) doesn't match dimensions: %d x %d x %d\n", file_name, nval, s->n_sen, s->n_feat, n_cw); } /* * Compute #LSB bits to be dropped to represent mixwfloor with 8 bits. * All PDF values will be truncated (in the LSB positions) by these many bits. */ if ((mixwfloor <= 0.0) || (mixwfloor >= 1.0)) E_FATAL("mixwfloor (%e) not in range (0, 1)\n", mixwfloor); /* Allocate memory for s->mgau2sen and senone PDF data */ build_mgau2sen (s, n_cw); /* Temporary structure to read in floats */ pdf = (float32 *) ckd_calloc (n_cw, sizeof(float32)); /* Read senone probs data, normalize, floor, convert to logs3, truncate to 8 bits */ n_err = 0; for (i = 0; i < s->n_sen; i++) { m = s->sen2mgau[i]; /* Parent mgau */ j = s->mgau2sen_idx[i]; /* Index of senone i within list of senones for mgau m */ fw = s->mgau2sen[m].feat_mixw; for (f = 0; f < s->n_feat; f++) { if (bio_fread((void *)pdf, sizeof(float32), n_cw, fp, byteswap, &chksum) != n_cw) { E_FATAL("bio_fread(%s) (arraydata) failed\n", file_name); } /* Normalize and floor */ if (vector_sum_norm (pdf, n_cw) == 0.0) n_err++; vector_floor (pdf, n_cw, mixwfloor); vector_sum_norm (pdf, n_cw); /* Convert to logs3, truncate to 8 bits, and store in s->pdf */ for (c = 0; c < n_cw; c++) { p = -logmath_log(logmath, pdf[c]); printf ("%f %d\n", pdf[c], p); fw[f].prob[j][c] = p; } } } if (n_err > 0) E_WARN("Weight normalization failed for %d senones\n", n_err); ckd_free (pdf); if (chksum_present) bio_verify_chksum (fp, byteswap, chksum); if (fread (&eofchk, 1, 1, fp) == 1) E_FATAL("More data than expected in %s\n", file_name); fclose(fp); E_INFO("Read mixture weights for %d senones: %d features x %d codewords\n", s->n_sen, s->n_feat, n_cw); return 0; }