static int normalize() { char file_name[MAXPATHLEN+1]; float32 ***mixw_acc = NULL; float32 ***in_mixw = NULL; float64 s; uint32 n_mixw; uint32 n_stream; uint32 n_mllr_class; uint32 n_density; float32 ***tmat_acc = NULL; uint32 n_tmat; uint32 n_state_pm; uint32 i, j, k; vector_t ***in_mean = NULL; vector_t ***wt_mean = NULL; vector_t ***in_var = NULL; vector_t ***wt_var = NULL; vector_t ****in_fullvar = NULL; vector_t ****wt_fullvar = NULL; int32 pass2var = FALSE; int32 var_is_full = FALSE; float32 ***dnom = NULL; uint32 n_mgau; uint32 n_gau_stream; uint32 n_gau_density; const uint32 *veclen = NULL; const char **accum_dir; const char *oaccum_dir; const char *in_mixw_fn; const char *out_mixw_fn; const char *out_tmat_fn; const char *in_mean_fn; const char *out_mean_fn; const char *in_var_fn; const char *out_var_fn; const char *out_dcount_fn; int err; uint32 mllr_mult; uint32 mllr_add; float32 *****regl = NULL; float32 ****regr = NULL; uint32 no_retries=0; accum_dir = cmd_ln_str_list("-accumdir"); oaccum_dir = cmd_ln_str("-oaccumdir"); out_mixw_fn = cmd_ln_str("-mixwfn"); out_tmat_fn = cmd_ln_str("-tmatfn"); out_mean_fn = cmd_ln_str("-meanfn"); out_var_fn = cmd_ln_str("-varfn"); in_mixw_fn = cmd_ln_str("-inmixwfn"); in_mean_fn = cmd_ln_str("-inmeanfn"); in_var_fn = cmd_ln_str("-invarfn"); out_dcount_fn = cmd_ln_str("-dcountfn"); var_is_full = cmd_ln_int32("-fullvar"); /* must be at least one accum dir */ assert(accum_dir[0] != NULL); if (out_mixw_fn == NULL) { E_INFO("No -mixwfn specified, will skip if any\n"); } if (out_tmat_fn == NULL) { E_INFO("No -tmatfn specified, will skip if any\n"); } if (out_mean_fn == NULL) { E_INFO("No -meanfn specified, will skip if any\n"); } if (out_var_fn == NULL) { E_INFO("No -varfn specified, will skip if any\n"); } if (in_mixw_fn != NULL) { E_INFO("Selecting unseen mixing weight parameters from %s\n", in_mixw_fn); } if (in_mean_fn != NULL) { E_INFO("Selecting unseen density mean parameters from %s\n", in_mean_fn); if (s3gau_read(in_mean_fn, &in_mean, &n_mgau, &n_gau_stream, &n_gau_density, &veclen) != S3_SUCCESS) { E_FATAL_SYSTEM("Couldn't read %s", in_mean_fn); } ckd_free((void *)veclen); veclen = NULL; } if (in_var_fn != NULL) { E_INFO("Selecting unseen density variance parameters from %s\n", in_var_fn); if (var_is_full) { if (s3gau_read_full(in_var_fn, &in_fullvar, &n_mgau, &n_gau_stream, &n_gau_density, &veclen) != S3_SUCCESS) { E_FATAL_SYSTEM("Couldn't read %s", in_var_fn); } } else { if (s3gau_read(in_var_fn, &in_var, &n_mgau, &n_gau_stream, &n_gau_density, &veclen) != S3_SUCCESS) { E_FATAL_SYSTEM("Couldn't read %s", in_var_fn); } } ckd_free((void *)veclen); veclen = NULL; } n_stream = 0; for (i = 0; accum_dir[i]; i++) { E_INFO("Reading and accumulating counts from %s\n", accum_dir[i]); if (out_mixw_fn) { rdacc_mixw(accum_dir[i], &mixw_acc, &n_mixw, &n_stream, &n_density); } if (out_tmat_fn) { rdacc_tmat(accum_dir[i], &tmat_acc, &n_tmat, &n_state_pm); } if (out_mean_fn || out_var_fn) { if (var_is_full) rdacc_den_full(accum_dir[i], &wt_mean, &wt_fullvar, &pass2var, &dnom, &n_mgau, &n_gau_stream, &n_gau_density, &veclen); else rdacc_den(accum_dir[i], &wt_mean, &wt_var, &pass2var, &dnom, &n_mgau, &n_gau_stream, &n_gau_density, &veclen); if (out_mixw_fn) { if (n_stream != n_gau_stream) { E_ERROR("mixw inconsistent w/ densities WRT # " "streams (%u != %u)\n", n_stream, n_gau_stream); } if (n_density != n_gau_density) { E_ERROR("mixw inconsistent w/ densities WRT # " "den/mix (%u != %u)\n", n_density, n_gau_density); } } else { n_stream = n_gau_stream; n_density = n_gau_density; } } } if (oaccum_dir && mixw_acc) { /* write the total mixing weight reest. accumulators */ err = 0; sprintf(file_name, "%s/mixw_counts", oaccum_dir); if (in_mixw_fn) { if (s3mixw_read(in_mixw_fn, &in_mixw, &i, &j, &k) != S3_SUCCESS) { E_FATAL_SYSTEM("Unable to read %s", in_mixw_fn); } if (i != n_mixw) { E_FATAL("# mixw in input mixw file != # mixw in output mixw file\n"); } if (j != n_stream) { E_FATAL("# stream in input mixw file != # stream in output mixw file\n"); } if (k != n_density) { E_FATAL("# density in input mixw file != # density in output mixw file\n"); } for (i = 0; i < n_mixw; i++) { for (j = 0; j < n_stream; j++) { for (k = 0, s = 0; k < n_density; k++) { s += mixw_acc[i][j][k]; } if ((s == 0) && in_mixw) { for (k = 0, s = 0; k < n_density; k++) { mixw_acc[i][j][k] = in_mixw[i][j][k]; } E_INFO("set mixw %u stream %u to input mixw value\n", i, j); } } } } do { /* Write out the accumulated reestimation sums */ if (s3mixw_write(file_name, mixw_acc, n_mixw, n_stream, n_density) != S3_SUCCESS) { if (err == 0) { E_ERROR("Unable to write %s; Retrying...\n", file_name); } ++err; sleep(3); no_retries++; if(no_retries>10){ E_FATAL("Failed to get the files after 10 retries(about 30 seconds).\n "); } } } while (err > 1); } if (pass2var) E_INFO("-2passvar yes\n"); if (oaccum_dir && (wt_mean || wt_var || wt_fullvar)) { /* write the total mixing Gau. den reest. accumulators */ err = 0; sprintf(file_name, "%s/gauden_counts", oaccum_dir); do { int32 rv; if (var_is_full) rv = s3gaucnt_write_full(file_name, wt_mean, wt_fullvar, pass2var, dnom, n_mgau, n_gau_stream, n_gau_density, veclen); else rv = s3gaucnt_write(file_name, wt_mean, wt_var, pass2var, dnom, n_mgau, n_gau_stream, n_gau_density, veclen); if (rv != S3_SUCCESS) { if (err == 0) { E_ERROR("Unable to write %s; Retrying...\n", file_name); } ++err; sleep(3); no_retries++; if(no_retries>10){ E_FATAL("Failed to get the files after 10 retries(about 5 minutes).\n "); } } } while (err > 1); } if (oaccum_dir && tmat_acc) { /* write the total transition matrix reest. accumulators */ err = 0; sprintf(file_name, "%s/tmat_counts", oaccum_dir); do { if (s3tmat_write(file_name, tmat_acc, n_tmat, n_state_pm) != S3_SUCCESS) { if (err == 0) { E_ERROR("Unable to write %s; Retrying...\n", file_name); } ++err; sleep(3); no_retries++; if(no_retries>10){ E_FATAL("Failed to get the files after 10 retries(about 5 minutes).\n "); } } } while (err > 1); } if (oaccum_dir && regr && regl) { /* write the total MLLR regression matrix accumulators */ err = 0; sprintf(file_name, "%s/regmat_counts", oaccum_dir); do { if (s3regmatcnt_write(file_name, regr, regl, n_mllr_class, n_stream, veclen, mllr_mult, mllr_add) != S3_SUCCESS) { if (err == 0) { E_ERROR("Unable to write %s; Retrying...\n", file_name); } ++err; sleep(3); no_retries++; if(no_retries>10){ E_FATAL("Failed to get the files after 10 retries(about 5 minutes).\n "); } } } while (err > 1); } if (wt_mean || wt_var || wt_fullvar) { if (out_mean_fn) { E_INFO("Normalizing mean for n_mgau= %u, n_stream= %u, n_density= %u\n", n_mgau, n_stream, n_density); gauden_norm_wt_mean(in_mean, wt_mean, dnom, n_mgau, n_stream, n_density, veclen); } else { if (wt_mean) { E_INFO("Ignoring means since -meanfn not specified\n"); } } if (out_var_fn) { if (var_is_full) { if (wt_fullvar) { E_INFO("Normalizing fullvar\n"); gauden_norm_wt_fullvar(in_fullvar, wt_fullvar, pass2var, dnom, wt_mean, /* wt_mean now just mean */ n_mgau, n_stream, n_density, veclen, cmd_ln_boolean("-tiedvar")); } } else { if (wt_var) { E_INFO("Normalizing var\n"); gauden_norm_wt_var(in_var, wt_var, pass2var, dnom, wt_mean, /* wt_mean now just mean */ n_mgau, n_stream, n_density, veclen, cmd_ln_boolean("-tiedvar")); } } } else { if (wt_var || wt_fullvar) { E_INFO("Ignoring variances since -varfn not specified\n"); } } } else { E_INFO("No means or variances to normalize\n"); } /* * Write the parameters to files */ if (out_mixw_fn) { if (mixw_acc) { if (s3mixw_write(out_mixw_fn, mixw_acc, n_mixw, n_stream, n_density) != S3_SUCCESS) { return S3_ERROR; } } else { E_WARN("NO mixing weight accumulators seen, but -mixwfn specified.\n"); } } else { if (mixw_acc) { E_INFO("Mixing weight accumulators seen, but -mixwfn NOT specified.\n"); } } if (out_tmat_fn) { if (tmat_acc) { if (s3tmat_write(out_tmat_fn, tmat_acc, n_tmat, n_state_pm) != S3_SUCCESS) { return S3_ERROR; } } else { E_WARN("NO transition matrix accumulators seen, but -tmatfn specified.\n"); } } else { if (tmat_acc) E_INFO("Transition matrix accumulators seen, but -tmatfn NOT specified\n"); } if (out_mean_fn) { if (wt_mean) { if (s3gau_write(out_mean_fn, (const vector_t ***)wt_mean, n_mgau, n_stream, n_density, veclen) != S3_SUCCESS) return S3_ERROR; if (out_dcount_fn) { if (s3gaudnom_write(out_dcount_fn, dnom, n_mgau, n_stream, n_density) != S3_SUCCESS) return S3_ERROR; } } else E_WARN("NO reestimated means seen, but -meanfn specified\n"); } else { if (wt_mean) { E_INFO("Reestimated means seen, but -meanfn NOT specified\n"); } } if (out_var_fn) { if (var_is_full) { if (wt_fullvar) { if (s3gau_write_full(out_var_fn, (const vector_t ****)wt_fullvar, n_mgau, n_stream, n_density, veclen) != S3_SUCCESS) return S3_ERROR; } else E_WARN("NO reestimated variances seen, but -varfn specified\n"); } else { if (wt_var) { if (s3gau_write(out_var_fn, (const vector_t ***)wt_var, n_mgau, n_stream, n_density, veclen) != S3_SUCCESS) return S3_ERROR; } else E_WARN("NO reestimated variances seen, but -varfn specified\n"); } } else { if (wt_var) { E_INFO("Reestimated variances seen, but -varfn NOT specified\n"); } } if (veclen) ckd_free((void *)veclen); return S3_SUCCESS; }
/********************************************************************* * * Function: * accum_dump * * Description: * This functions writes the reestimation numerators and * denominators to files. * * This routine may be called repeatedly until success to * try to cope with exceptional conditions like insufficient disk * space, etc. * * Function Inputs: * const char *out_dir - * model_inventory_t *inv - * int32 mixw_reest - * int32 tmat_reest - * int32 mean_reest - * int32 var_reest - * * Global Inputs: * None * * Return Values: * S3_SUCCESS - The values were written to files successfully. * S3_ERROR - The values were not completely written to files. * * Global Outputs: * None * * Errors: * *********************************************************************/ int32 accum_dump(const char *out_dir, model_inventory_t *inv, int32 mixw_reest, int32 tmat_reest, int32 mean_reest, int32 var_reest, int32 pass2var, int32 var_is_full, int ckpt) /* checkpoint dump flag */ { char fn[MAXPATHLEN+1]; gauden_t *g; /* run over the accumulators and report anything exceptional */ if (!ckpt) accum_stat(inv, mixw_reest); g = inv->gauden; mk_bkp(mixw_reest, tmat_reest, mean_reest, var_reest, out_dir); if (mixw_reest) { sprintf(fn, "%s/mixw_counts", out_dir); if (s3mixw_write(fn, inv->mixw_acc, inv->n_mixw, g->n_feat, g->n_density) != S3_SUCCESS) { revert_bkp(mixw_reest, tmat_reest, mean_reest, var_reest, out_dir); return S3_ERROR; } } else { if (!ckpt) E_INFO("Mixture weights not reestimated. " "No %s/mixw_counts produced.\n", out_dir); } if (tmat_reest) { sprintf(fn, "%s/tmat_counts", out_dir); if (s3tmat_write(fn, inv->tmat_acc, inv->n_tmat, inv->n_state_pm) != S3_SUCCESS) { revert_bkp(mixw_reest, tmat_reest, mean_reest, var_reest, out_dir); return S3_ERROR; } } else { if (!ckpt) E_INFO("Transition matrices not reestimated. " "No %s/tmat_counts produced.\n", out_dir); } if (mean_reest || var_reest) { int32 rv; sprintf(fn, "%s/gauden_counts", out_dir); if (var_is_full) rv = s3gaucnt_write_full(fn, (mean_reest ? g->macc : NULL), (var_reest ? g->fullvacc : NULL), pass2var, g->dnom, g->n_mgau, g->n_feat, g->n_density, g->veclen); else rv = s3gaucnt_write(fn, (mean_reest ? g->macc : NULL), (var_reest ? g->vacc : NULL), pass2var, g->dnom, g->n_mgau, g->n_feat, g->n_density, g->veclen); if (rv != S3_SUCCESS) { revert_bkp(mixw_reest, tmat_reest, mean_reest, var_reest, out_dir); return S3_ERROR; } } else { if (!ckpt) E_INFO("means and variances not reestimated. " "No %s/gauden_counts produced.\n", out_dir); } return commit(mixw_reest, tmat_reest, mean_reest, var_reest, ckpt, out_dir); }