static int32
senone_mixw_read(senone_t * s, char const *file_name, logmath_t *lmath)
{
char eofchk;
FILE *fp;
int32 byteswap, chksum_present;
uint32 chksum;
float32 *pdf;
int32 i, f, c, p, n_err;
char **argname, **argval;
E_INFO("Reading senone mixture weights: %s\n", file_name);
if ((fp = fopen(file_name, "rb")) == NULL)
E_FATAL_SYSTEM("Failed to open mixture weights file '%s' for reading", file_name);
/* Read header, including argument-value info and 32-bit byteorder magic */
if (bio_readhdr(fp, &argname, &argval, &byteswap) < 0)
E_FATAL("Failed to read header from file '%s'\n", file_name);
/* Parse argument-value list */
chksum_present = 0;
for (i = 0; argname[i]; i++) {
if (strcmp(argname[i], "version") == 0) {
if (strcmp(argval[i], MIXW_PARAM_VERSION) != 0)
E_WARN("Version mismatch(%s): %s, expecting %s\n",
file_name, argval[i], MIXW_PARAM_VERSION);
}
else if (strcmp(argname[i], "chksum0") == 0) {
chksum_present = 1; /* Ignore the associated value */
}
}
bio_hdrarg_free(argname, argval);
argname = argval = NULL;
chksum = 0;
/* Read #senones, #features, #codewords, arraysize */
if ((bio_fread(&(s->n_sen), sizeof(int32), 1, fp, byteswap, &chksum) !=
1)
||
(bio_fread(&(s->n_feat), sizeof(int32), 1, fp, byteswap, &chksum)
!= 1)
|| (bio_fread(&(s->n_cw), sizeof(int32), 1, fp, byteswap, &chksum)
!= 1)
|| (bio_fread(&i, sizeof(int32), 1, fp, byteswap, &chksum) != 1)) {
E_FATAL("bio_fread(%s) (arraysize) failed\n", file_name);
}
if (i != s->n_sen * s->n_feat * s->n_cw) {
E_FATAL
("%s: #float32s(%d) doesn't match dimensions: %d x %d x %d\n",
file_name, i, s->n_sen, s->n_feat, s->n_cw);
}
/*
* Compute #LSB bits to be dropped to represent mixwfloor with 8 bits.
* All PDF values will be truncated (in the LSB positions) by these many bits.
*/
if ((s->mixwfloor <= 0.0) || (s->mixwfloor >= 1.0))
E_FATAL("mixwfloor (%e) not in range (0, 1)\n", s->mixwfloor);
/* Use a fixed shift for compatibility with everything else. */
E_INFO("Truncating senone logs3(pdf) values by %d bits\n", SENSCR_SHIFT);
/*
* Allocate memory for senone PDF data. Organize normally or transposed depending on
* s->n_gauden.
*/
if (s->n_gauden > 1) {
E_INFO("Not transposing mixture weights in memory\n");
s->pdf =
(senprob_t ***) ckd_calloc_3d(s->n_sen, s->n_feat, s->n_cw,
sizeof(senprob_t));
}
else {
E_INFO("Transposing mixture weights in memory\n");
s->pdf =
(senprob_t ***) ckd_calloc_3d(s->n_feat, s->n_cw, s->n_sen,
sizeof(senprob_t));
}
/* Temporary structure to read in floats */
pdf = (float32 *) ckd_calloc(s->n_cw, sizeof(float32));
/* Read senone probs data, normalize, floor, convert to logs3, truncate to 8 bits */
n_err = 0;
for (i = 0; i < s->n_sen; i++) {
for (f = 0; f < s->n_feat; f++) {
if (bio_fread
((void *) pdf, sizeof(float32), s->n_cw, fp, byteswap,
&chksum)
!= s->n_cw) {
E_FATAL("bio_fread(%s) (arraydata) failed\n", file_name);
}
/* Normalize and floor */
if (vector_sum_norm(pdf, s->n_cw) <= 0.0)
n_err++;
vector_floor(pdf, s->n_cw, s->mixwfloor);
vector_sum_norm(pdf, s->n_cw);
/* Convert to logs3, truncate to 8 bits, and store in s->pdf */
for (c = 0; c < s->n_cw; c++) {
p = -(logmath_log(lmath, pdf[c]));
p += (1 << (SENSCR_SHIFT - 1)) - 1; /* Rounding before truncation */
if (s->n_gauden > 1)
s->pdf[i][f][c] =
(p < (255 << SENSCR_SHIFT)) ? (p >> SENSCR_SHIFT) : 255;
else
s->pdf[f][c][i] =
(p < (255 << SENSCR_SHIFT)) ? (p >> SENSCR_SHIFT) : 255;
}
}
}
static int32
read_mixw(s2_semi_mgau_t * s, char const *file_name, double SmoothMin)
{
char **argname, **argval;
char eofchk;
FILE *fp;
int32 byteswap, chksum_present;
uint32 chksum;
float32 *pdf;
int32 i, f, c, n;
int32 n_sen;
int32 n_feat;
int32 n_comp;
int32 n_err;
E_INFO("Reading mixture weights file '%s'\n", file_name);
if ((fp = fopen(file_name, "rb")) == NULL)
E_FATAL("fopen(%s,rb) failed\n", file_name);
/* Read header, including argument-value info and 32-bit byteorder magic */
if (bio_readhdr(fp, &argname, &argval, &byteswap) < 0)
E_FATAL("bio_readhdr(%s) failed\n", file_name);
/* Parse argument-value list */
chksum_present = 0;
for (i = 0; argname[i]; i++) {
if (strcmp(argname[i], "version") == 0) {
if (strcmp(argval[i], MGAU_MIXW_VERSION) != 0)
E_WARN("Version mismatch(%s): %s, expecting %s\n",
file_name, argval[i], MGAU_MIXW_VERSION);
}
else if (strcmp(argname[i], "chksum0") == 0) {
chksum_present = 1; /* Ignore the associated value */
}
}
bio_hdrarg_free(argname, argval);
argname = argval = NULL;
chksum = 0;
/* Read #senones, #features, #codewords, arraysize */
if ((bio_fread(&n_sen, sizeof(int32), 1, fp, byteswap, &chksum) != 1)
|| (bio_fread(&n_feat, sizeof(int32), 1, fp, byteswap, &chksum) !=
1)
|| (bio_fread(&n_comp, sizeof(int32), 1, fp, byteswap, &chksum) !=
1)
|| (bio_fread(&n, sizeof(int32), 1, fp, byteswap, &chksum) != 1)) {
E_FATAL("bio_fread(%s) (arraysize) failed\n", file_name);
}
if (n_feat != s->n_feat)
E_FATAL("#Features streams(%d) != %d\n", n_feat, s->n_feat);
if (n != n_sen * n_feat * n_comp) {
E_FATAL
("%s: #float32s(%d) doesn't match header dimensions: %d x %d x %d\n",
file_name, i, n_sen, n_feat, n_comp);
}
/* n_sen = number of mixture weights per codeword, which is
* fixed at the number of senones since we have only one codebook.
*/
s->n_sen = n_sen;
/* Quantized mixture weight arrays. */
s->mixw = ckd_calloc_3d(s->n_feat, s->n_density, n_sen, sizeof(***s->mixw));
/* Temporary structure to read in floats before conversion to (int32) logs3 */
pdf = (float32 *) ckd_calloc(n_comp, sizeof(float32));
/* Read senone probs data, normalize, floor, convert to logs3, truncate to 8 bits */
n_err = 0;
for (i = 0; i < n_sen; i++) {
for (f = 0; f < n_feat; f++) {
if (bio_fread((void *) pdf, sizeof(float32),
n_comp, fp, byteswap, &chksum) != n_comp) {
E_FATAL("bio_fread(%s) (arraydata) failed\n", file_name);
}
/* Normalize and floor */
if (vector_sum_norm(pdf, n_comp) <= 0.0)
n_err++;
vector_floor(pdf, n_comp, SmoothMin);
vector_sum_norm(pdf, n_comp);
/* Convert to LOG, quantize, and transpose */
for (c = 0; c < n_comp; c++) {
int32 qscr;
qscr = -logmath_log(s->lmath_8b, pdf[c]);
if ((qscr > MAX_NEG_MIXW) || (qscr < 0))
qscr = MAX_NEG_MIXW;
s->mixw[f][c][i] = qscr;
}
}
}
if (n_err > 0)
E_WARN("Weight normalization failed for %d senones\n", n_err);
ckd_free(pdf);
if (chksum_present)
bio_verify_chksum(fp, byteswap, chksum);
if (fread(&eofchk, 1, 1, fp) == 1)
E_FATAL("More data than expected in %s\n", file_name);
fclose(fp);
E_INFO("Read %d x %d x %d mixture weights\n", n_sen, n_feat, n_comp);
return n_sen;
}
tmat_t *tmat_init (char *file_name, float64 tpfloor)
{
char tmp;
int32 n_src, n_dst;
FILE *fp;
int32 byteswap, chksum_present;
uint32 chksum;
float32 **tp;
int32 i, j, k, tp_per_tmat;
char **argname, **argval;
tmat_t *t;
E_INFO("Reading HMM transition probability matrices: %s\n", file_name);
t = (tmat_t *) ckd_calloc (1, sizeof(tmat_t));
if ((fp = fopen(file_name, "rb")) == NULL)
E_FATAL_SYSTEM("fopen(%s,rb) failed\n", file_name);
/* Read header, including argument-value info and 32-bit byteorder magic */
if (bio_readhdr (fp, &argname, &argval, &byteswap) < 0)
E_FATAL("bio_readhdr(%s) failed\n", file_name);
/* Parse argument-value list */
chksum_present = 0;
for (i = 0; argname[i]; i++) {
if (strcmp (argname[i], "version") == 0) {
if (strcmp(argval[i], TMAT_PARAM_VERSION) != 0)
E_WARN("Version mismatch(%s): %s, expecting %s\n",
file_name, argval[i], TMAT_PARAM_VERSION);
} else if (strcmp (argname[i], "chksum0") == 0) {
chksum_present = 1; /* Ignore the associated value */
}
}
bio_hdrarg_free (argname, argval);
argname = argval = NULL;
chksum = 0;
/* Read #tmat, #from-states, #to-states, arraysize */
if ((bio_fread (&(t->n_tmat), sizeof(int32), 1, fp, byteswap, &chksum) != 1) ||
(bio_fread (&n_src, sizeof(int32), 1, fp, byteswap, &chksum) != 1) ||
(bio_fread (&n_dst, sizeof(int32), 1, fp, byteswap, &chksum) != 1) ||
(bio_fread (&i, sizeof(int32), 1, fp, byteswap, &chksum) != 1)) {
E_FATAL("bio_fread(%s) (arraysize) failed\n", file_name);
}
if (t->n_tmat >= MAX_S3TMATID)
E_FATAL("%s: #tmat (%d) exceeds limit (%d)\n", file_name, t->n_tmat, MAX_S3TMATID);
if (n_dst != n_src+1)
E_FATAL("%s: #from-states(%d) != #to-states(%d)-1\n", file_name, n_src, n_dst);
t->n_state = n_src;
if (i != t->n_tmat * n_src * n_dst) {
E_FATAL("%s: #float32s(%d) doesn't match dimensions: %d x %d x %d\n",
file_name, i, t->n_tmat, n_src, n_dst);
}
/* Allocate memory for tmat data */
t->tp = (int32 ***) ckd_calloc_3d (t->n_tmat, n_src, n_dst, sizeof(int32));
/* Temporary structure to read in the float data */
tp = (float32 **) ckd_calloc_2d (n_src, n_dst, sizeof(float32));
/* Read transition matrices, normalize and floor them, and convert to logs3 domain */
tp_per_tmat = n_src * n_dst;
for (i = 0; i < t->n_tmat; i++) {
if (bio_fread (tp[0], sizeof(float32), tp_per_tmat, fp,
byteswap, &chksum) != tp_per_tmat) {
E_FATAL("fread(%s) (arraydata) failed\n", file_name);
}
/* Normalize and floor */
for (j = 0; j < n_src; j++) {
if (vector_sum_norm (tp[j], n_dst) == 0.0)
E_ERROR("Normalization failed for tmat %d from state %d\n", i, j);
vector_nz_floor (tp[j], n_dst, tpfloor);
vector_sum_norm (tp[j], n_dst);
/* Convert to logs3. Take care of special case when tp = 0.0! */
for (k = 0; k < n_dst; k++)
t->tp[i][j][k] = (tp[j][k] == 0.0) ? S3_LOGPROB_ZERO : logs3(tp[j][k]);
}
}
ckd_free_2d ((void **) tp);
if (chksum_present)
bio_verify_chksum (fp, byteswap, chksum);
if (fread (&tmp, 1, 1, fp) == 1)
E_ERROR("Non-empty file beyond end of data\n");
fclose(fp);
E_INFO("Read %d transition matrices of size %dx%d\n",
t->n_tmat, t->n_state, t->n_state+1);
if (tmat_chk_uppertri (t) < 0)
E_FATAL("Tmat not upper triangular\n");
return t;
}
static int32 senone_mixw_read(logmath_t * logmath, senone_t *s, const char *file_name, float64 mixwfloor)
{
FILE *fp;
char **argname, **argval;
int32 byteswap, chksum_present;
uint32 chksum;
float32 *pdf;
int32 i, j, f, m, c, p, n_sen, n_err, n_cw, nval;
char eofchk;
mixw_t *fw;
E_INFO("Reading senone mixture weights: %s\n", file_name);
if ((fp = fopen(file_name, "rb")) == NULL)
E_FATAL_SYSTEM("fopen(%s,rb) failed\n", file_name);
/* Read header, including argument-value info and 32-bit byteorder magic */
if (bio_readhdr (fp, &argname, &argval, &byteswap) < 0)
E_FATAL("bio_readhdr(%s) failed\n", file_name);
/* Parse argument-value list */
chksum_present = 0;
for (i = 0; argname[i]; i++) {
if (strcmp (argname[i], "version") == 0) {
if (strcmp(argval[i], MIXW_PARAM_VERSION) != 0)
E_WARN("Version mismatch(%s): %s, expecting %s\n",
file_name, argval[i], MIXW_PARAM_VERSION);
} else if (strcmp (argname[i], "chksum0") == 0) {
chksum_present = 1; /* Ignore the associated value */
}
}
bio_hdrarg_free (argname, argval);
argname = argval = NULL;
chksum = 0;
/* Read #senones, #features, #codewords, arraysize */
n_sen = s->n_sen;
if ((bio_fread (&(s->n_sen), sizeof(int32), 1, fp, byteswap, &chksum) != 1) ||
(bio_fread (&(s->n_feat), sizeof(int32), 1, fp, byteswap, &chksum) != 1) ||
(bio_fread (&(n_cw), sizeof(int32), 1, fp, byteswap, &chksum) != 1) ||
(bio_fread (&nval, sizeof(int32), 1, fp, byteswap, &chksum) != 1)) {
E_FATAL("bio_fread(%s) (arraysize) failed\n", file_name);
}
if ((n_sen != 0) && (s->n_sen != n_sen))
E_FATAL("#senones(%d) conflict with mapping file(%d)\n", s->n_sen, n_sen);
if (s->n_sen >= MAX_SENID)
E_FATAL("%s: #senones (%d) exceeds limit (%d)\n", file_name, s->n_sen, MAX_SENID);
if (s->n_feat <= 0)
E_FATAL("Bad #features: %d\n", s->n_feat);
if (n_cw <= 0)
E_FATAL("Bad #mixing-wts/senone: %d\n", n_cw);
/* Allocate sen2mgau map if not yet done so (i.e. no explicit mapping file given */
if (! s->sen2mgau) {
assert ((s->n_mgau == 0) || (s->n_mgau == 1));
s->sen2mgau = (uint32 *) ckd_calloc (s->n_sen, sizeof(int32));
if (s->n_mgau == 1) {
/* Semicontinuous mode; all senones map to single, shared gaussian: 0 */
for (i = 0; i < s->n_sen; i++)
s->sen2mgau[i] = 0;
} else {
/* Fully continuous mode; each senone maps to own parent gaussian */
s->n_mgau = s->n_sen;
for (i = 0; i < s->n_sen; i++)
s->sen2mgau[i] = i;
}
} else
assert (s->n_mgau != 0);
if (s->n_mgau >= MAX_MGAUID)
E_FATAL("%s: #gauden (%d) exceeds limit (%d)\n", file_name, s->n_mgau, MAX_MGAUID);
if (nval != s->n_sen * s->n_feat * n_cw) {
E_FATAL("%s: #float32 values(%d) doesn't match dimensions: %d x %d x %d\n",
file_name, nval, s->n_sen, s->n_feat, n_cw);
}
/*
* Compute #LSB bits to be dropped to represent mixwfloor with 8 bits.
* All PDF values will be truncated (in the LSB positions) by these many bits.
*/
if ((mixwfloor <= 0.0) || (mixwfloor >= 1.0))
E_FATAL("mixwfloor (%e) not in range (0, 1)\n", mixwfloor);
/* Allocate memory for s->mgau2sen and senone PDF data */
build_mgau2sen (s, n_cw);
/* Temporary structure to read in floats */
pdf = (float32 *) ckd_calloc (n_cw, sizeof(float32));
/* Read senone probs data, normalize, floor, convert to logs3, truncate to 8 bits */
n_err = 0;
for (i = 0; i < s->n_sen; i++) {
m = s->sen2mgau[i]; /* Parent mgau */
j = s->mgau2sen_idx[i]; /* Index of senone i within list of senones for mgau m */
fw = s->mgau2sen[m].feat_mixw;
for (f = 0; f < s->n_feat; f++) {
if (bio_fread((void *)pdf, sizeof(float32), n_cw, fp, byteswap, &chksum)
!= n_cw) {
E_FATAL("bio_fread(%s) (arraydata) failed\n", file_name);
}
/* Normalize and floor */
if (vector_sum_norm (pdf, n_cw) == 0.0)
n_err++;
vector_floor (pdf, n_cw, mixwfloor);
vector_sum_norm (pdf, n_cw);
/* Convert to logs3, truncate to 8 bits, and store in s->pdf */
for (c = 0; c < n_cw; c++) {
p = -logmath_log(logmath, pdf[c]);
printf ("%f %d\n", pdf[c], p);
fw[f].prob[j][c] = p;
}
}
}
if (n_err > 0)
E_WARN("Weight normalization failed for %d senones\n", n_err);
ckd_free (pdf);
if (chksum_present)
bio_verify_chksum (fp, byteswap, chksum);
if (fread (&eofchk, 1, 1, fp) == 1)
E_FATAL("More data than expected in %s\n", file_name);
fclose(fp);
E_INFO("Read mixture weights for %d senones: %d features x %d codewords\n",
s->n_sen, s->n_feat, n_cw);
return 0;
}
