int s3lambcnt_write(const char *fn, const uint32 *lambda_cnt, uint32 n_lambda_cnt) { FILE *fp; uint32 chksum = 0; uint32 ignore = 0; s3clr_fattr(); s3add_fattr("version", LAMBDACNT_FILE_VERSION, TRUE); s3add_fattr("chksum0", "yes", TRUE); fp = s3open(fn, "wb", NULL); if (fp == NULL) return S3_ERROR; if (s3write_1d((void *)lambda_cnt, sizeof(uint32), n_lambda_cnt, fp, &chksum) != S3_SUCCESS) { s3close(fp); return S3_ERROR; } if (s3write(&chksum, sizeof(uint32), 1, fp, &ignore) != 1) { s3close(fp); return S3_ERROR; } s3close(fp); E_INFO("Wrote %s [%u array]\n", fn, n_lambda_cnt); return S3_SUCCESS; }
int s3mixw_write(const char *fn, float32 ***mixw, uint32 n_mixw, uint32 n_feat, uint32 n_density) { FILE *fp; uint32 chksum = 0; uint32 ignore = 0; uint32 i, j, k; s3clr_fattr(); s3add_fattr("version", MIXW_FILE_VERSION, TRUE); s3add_fattr("chksum0", "yes", TRUE); fp = s3open(fn, "wb", NULL); if (fp == NULL) return S3_ERROR; for (i = 0; i < n_mixw; i++) { for (j = 0; j < n_feat; j++) { for (k = 0; k < n_density; k++) { if (mixw[i][j][k] < 0) { E_ERROR("mixw[%u][%u][%u] < 0 (%e)\n", i,j,k,mixw[i][j][k]); } } } } /* floor all non-zero entries to this value to make sure that results are compatible between machines */ floor_nz_3d(mixw, n_mixw, n_feat, n_density, MIN_POS_FLOAT32); if (bio_fwrite_3d((void ***)mixw, sizeof(float32), n_mixw, n_feat, n_density, fp, &chksum) < 0) { s3close(fp); return S3_ERROR; } if (bio_fwrite(&chksum, sizeof(uint32), 1, fp, 0, &ignore) != 1) { s3close(fp); return S3_ERROR; } s3close(fp); E_INFO("Wrote %s [%ux%ux%u array]\n", fn, n_mixw, n_feat, n_density); return S3_SUCCESS; }
int s3gau_write_full(const char *fn, const vector_t ****out, uint32 n_mgau, uint32 n_feat, uint32 n_density, const uint32 *veclen) { FILE *fp = NULL; uint32 chksum = 0; uint32 blk, i, ignore = 0; s3clr_fattr(); s3add_fattr("version", GAU_FILE_VERSION, TRUE); s3add_fattr("chksum0", "yes", TRUE); fp = s3open(fn, "wb", NULL); if (fp == NULL) return S3_ERROR; if (bio_fwrite(&n_mgau, sizeof(uint32), 1, fp, 0, &chksum) != 1) { goto error; } if (bio_fwrite(&n_feat, sizeof(uint32), 1, fp, 0, &chksum) != 1) { goto error; } if (bio_fwrite(&n_density, sizeof(uint32), 1, fp, 0, &chksum) != 1) { goto error; } if (bio_fwrite(veclen, sizeof(uint32), n_feat, fp, 0, &chksum) != n_feat) { goto error; } for (blk = 0, i = 0; i < n_feat; i++) blk += veclen[i] * veclen[i]; if (bio_fwrite_1d(out[0][0][0][0], sizeof(float32), n_mgau*n_density*blk, fp, &chksum) < 0) { goto error; } if (bio_fwrite(&chksum, sizeof(uint32), 1, fp, 0, &ignore) != 1) { goto error; } s3close(fp); E_INFO("Wrote %s [%ux%ux%u array of full matrices]\n", fn, n_mgau, n_feat, n_density); return S3_SUCCESS; error: if (fp) s3close(fp); return S3_ERROR; }
int s3tmat_write(const char *fn, float32 ***tmat, uint32 n_tmat, uint32 n_state) { FILE *fp; uint32 chksum = 0; uint32 ignore = 0; int t, i, j; s3clr_fattr(); s3add_fattr("version", TMAT_FILE_VERSION, TRUE); s3add_fattr("chksum0", "yes", TRUE); fp = s3open(fn, "wb", NULL); if (fp == NULL) return S3_ERROR; for (t = 0; t < n_tmat; t++) { for (i = 0; i < n_state-1; i++) { for (j = 0; j < n_state; j++) { if (tmat[t][i][j] < 0) { E_ERROR("tmat[%u][%u][%u] < 0 (%e)\n", t, i, j, tmat[t][i][j]); } } } } /* floor all non-zero entries to this value to make sure that results are compatible between machines */ floor_nz_3d(tmat, n_tmat, n_state-1, n_state, MIN_POS_FLOAT32); if (s3write_3d((void ***)tmat, sizeof(float32), n_tmat, n_state-1, n_state, fp, &chksum) != S3_SUCCESS) { s3close(fp); return S3_ERROR; } if (s3write(&chksum, sizeof(uint32), 1, fp, &ignore) != 1) { s3close(fp); return S3_ERROR; } s3close(fp); E_INFO("Wrote %s [%ux%ux%u array]\n", fn, n_tmat, n_state-1, n_state); return S3_SUCCESS; }
int s3mixw_intv_read(const char *fn, uint32 mixw_s, uint32 mixw_e, float32 ****out_mixw, uint32 *out_n_mixw, uint32 *out_n_feat, uint32 *out_n_density) { uint32 ignore; char *ver; char *do_chk; FILE *fp; uint32 swap; fp = s3open(fn, "rb", &swap); if (fp == NULL) return S3_ERROR; /* check version id */ ver = s3get_gvn_fattr("version"); if (ver) { if (strcmp(ver, MIXW_FILE_VERSION) != 0) { E_FATAL("Version mismatch for %s, file ver: %s != reader ver: %s\n", fn, ver, MIXW_FILE_VERSION); } } else { E_FATAL("No version attribute for %s\n", fn); } /* if do_chk is non-NULL, there is a checksum after the data in the file */ do_chk = s3get_gvn_fattr("chksum0"); /* Read the mixing weight array */ if (bio_fread_intv_3d((void ****)out_mixw, sizeof(float32), mixw_s, mixw_e, out_n_mixw, out_n_feat, out_n_density, fp, swap, &ignore) != S3_SUCCESS) { s3close(fp); return S3_ERROR; } s3close(fp); E_INFO("Read %s [%ux%ux%u array]\n", fn, *out_n_mixw, *out_n_feat, *out_n_density); return S3_SUCCESS; }
static FILE * open_dmp(const char *fn) { FILE *fp; s3clr_fattr(); fp = s3open(fn, "wb", NULL); if (fp == NULL) { E_ERROR_SYSTEM("Unable to open %s for writing.", fn); } return fp; }
int s3map_write(const char *fn, void *map, uint32 n_dom, uint32 n_rng, size_t map_elem_size) { FILE *fp; uint32 chksum = 0; uint32 ignore = 0; s3clr_fattr(); s3add_fattr("version", MAP_FILE_VERSION, TRUE); s3add_fattr("chksum0", "yes", TRUE); fp = s3open(fn, "wb", NULL); if (fp == NULL) return S3_ERROR; if (bio_fwrite(&n_rng, sizeof(uint32), 1, fp, &chksum) != 1) { s3close(fp); return S3_ERROR; } if (bio_fwrite_1d(map, map_elem_size, n_dom, fp, &chksum) < 0) { s3close(fp); return S3_ERROR; } if (bio_fwrite(&chksum, sizeof(uint32), 1, fp, &ignore) != 1) { s3close(fp); return S3_ERROR; } s3close(fp); E_INFO("Wrote %s [%u mappings to %u]\n", fn, n_dom, n_rng); return S3_SUCCESS; }
static int init_state(const char *obsdmp, const char *obsidx, uint32 n_density, uint32 n_stream, const uint32 *veclen, int reest, const char *mixwfn, const char *meanfn, const char *varfn, uint32 ts_off, uint32 ts_cnt, uint32 n_ts, uint32 n_d_ts) { uint32 blksz; vector_t ***mean; vector_t ***var = NULL; vector_t ****fullvar = NULL; float32 ***mixw = NULL; uint32 n_frame; uint32 ignore = 0; codew_t *label; uint32 n_corpus = 0; float64 sqerr; float64 tot_sqerr; segdmp_type_t t; uint32 i, j, ts, n; timing_t *km_timer; timing_t *var_timer; timing_t *em_timer; int32 full_covar; km_timer = timing_get("km"); var_timer = timing_get("var"); em_timer = timing_get("em"); blksz = feat_blksize(); full_covar = cmd_ln_int32("-fullvar"); /* fully-continuous for now */ mean = gauden_alloc_param(ts_cnt, n_stream, n_density, veclen); if (full_covar) fullvar = gauden_alloc_param_full(ts_cnt, n_stream, n_density, veclen); else var = gauden_alloc_param(ts_cnt, n_stream, n_density, veclen); if (mixwfn) mixw = (float32 ***)ckd_calloc_3d(ts_cnt, n_stream, n_density, sizeof(float32)); if ((const char *)cmd_ln_access("-segidxfn")) { E_INFO("Multi-class dump\n"); if (segdmp_open_read((const char **)cmd_ln_access("-segdmpdirs"), (const char *)cmd_ln_access("-segdmpfn"), (const char *)cmd_ln_access("-segidxfn"), &n, &t) != S3_SUCCESS) { E_FATAL("Unable to open dumps\n"); } if (n != n_d_ts) { E_FATAL("Expected %u tied-states in dump, but apparently %u\n", n_d_ts, n); } if (t != SEGDMP_TYPE_FEAT) { E_FATAL("Expected feature dump, but instead saw %u\n", t); } multiclass = TRUE; } else { E_INFO("1-class dump file\n"); multiclass = FALSE; dmp_fp = s3open((const char *)cmd_ln_access("-segdmpfn"), "rb", &dmp_swp); if (dmp_fp == NULL) { E_ERROR_SYSTEM("Unable to open dump file %s for reading\n", (const char *)cmd_ln_access("-segdmpfn")); return S3_ERROR; } if (s3read(&n_frame, sizeof(uint32), 1, dmp_fp, dmp_swp, &ignore) != 1) { E_ERROR_SYSTEM("Unable to open dump file %s for reading\n", (const char *)cmd_ln_access("-segdmpfn")); return S3_ERROR; } data_offset = ftell(dmp_fp); } tot_sqerr = 0; for (i = 0; i < ts_cnt; i++) { ts = ts_off + i; /* stride not accounted for yet */ if (o2d == NULL) { if (multiclass) n_frame = segdmp_n_seg(ts); } else { for (j = 0, n_frame = 0; j < n_o2d[ts]; j++) { n_frame += segdmp_n_seg(o2d[ts][j]); } } E_INFO("Corpus %u: sz==%u frames%s\n", ts, n_frame, (n_frame > *(uint32 *)cmd_ln_access("-vartiethr") ? "" : " tied var")); if (n_frame == 0) { continue; } E_INFO("Convergence ratios are abs(cur - prior) / abs(prior)\n"); /* Do some variety of k-means clustering */ if (km_timer) timing_start(km_timer); sqerr = cluster(ts, n_stream, n_frame, veclen, mean[i], n_density, &label); if (km_timer) timing_stop(km_timer); if (sqerr < 0) { E_ERROR("Unable to do k-means for state %u; skipping...\n", ts); continue; } /* Given the k-means and assuming equal prior liklihoods * compute the variances */ if (var_timer) timing_start(var_timer); if (full_covar) full_variances(ts, mean[i], fullvar[i], n_density, veclen, n_frame, n_stream, label); else variances(ts, mean[i], var[i], n_density, veclen, n_frame, n_stream, label); if (var_timer) timing_stop(var_timer); if (mixwfn) { /* initialize the mixing weights by counting # of occurrances * of the top codeword over the corpus and normalizing */ init_mixw(mixw[i], mean[i], n_density, veclen, n_frame, n_stream, label); ckd_free(label); if (reest == TRUE && full_covar) E_ERROR("EM re-estimation is not yet supported for full covariances\n"); else if (reest == TRUE) { if (em_timer) timing_start(em_timer); /* Do iterations of EM to estimate the mixture densities */ reest_sum(ts, mean[i], var[i], mixw[i], n_density, n_stream, n_frame, veclen, *(uint32 *)cmd_ln_access("-niter"), FALSE, *(uint32 *)cmd_ln_access("-vartiethr")); if (em_timer) timing_stop(em_timer); } } ++n_corpus; tot_sqerr += sqerr; E_INFO("sqerr [%u] == %e\n", ts, sqerr); } if (n_corpus > 0) { E_INFO("sqerr = %e tot %e rms\n", tot_sqerr, sqrt(tot_sqerr/n_corpus)); } if (!multiclass) s3close(dmp_fp); if (meanfn) { if (s3gau_write(meanfn, (const vector_t ***)mean, ts_cnt, n_stream, n_density, veclen) != S3_SUCCESS) { return S3_ERROR; } } else { E_INFO("No mean file given; none written\n"); } if (varfn) { if (full_covar) { if (s3gau_write_full(varfn, (const vector_t ****)fullvar, ts_cnt, n_stream, n_density, veclen) != S3_SUCCESS) return S3_ERROR; } else { if (s3gau_write(varfn, (const vector_t ***)var, ts_cnt, n_stream, n_density, veclen) != S3_SUCCESS) return S3_ERROR; } } else { E_INFO("No variance file given; none written\n"); } if (mixwfn) { if (s3mixw_write(mixwfn, mixw, ts_cnt, n_stream, n_density) != S3_SUCCESS) { return S3_ERROR; } } else { E_INFO("No mixing weight file given; none written\n"); } return S3_SUCCESS; }
int s3regmatcnt_read(const char *fn, float32 *****out_regr, float32 ******out_regl, uint32 *out_n_class, uint32 *out_n_feat, const uint32 **out_veclen, uint32 *out_mllr_mult, uint32 *out_mllr_add) { uint32 rd_chksum = 0; uint32 sv_chksum; uint32 ignore; char *ver; char *do_chk; FILE *fp; uint32 n_feat, n_class; const uint32 *veclen; float32 ****l_regr; float32 *****l_regl; uint32 d1, d2, d3, m, f; uint32 swap; fp = s3open(fn, "rb", &swap); if (fp == NULL) return S3_ERROR; /* check version id */ ver = s3get_gvn_fattr("version"); if (ver) { if (strcmp(ver, REGMATCNT_FILE_VERSION) != 0) { E_FATAL("Version mismatch for %s, file ver: %s != reader ver: %s\n", fn, ver, REGMATCNT_FILE_VERSION); } } else { E_FATAL("No version attribute for %s\n", fn); } /* if do_chk is non-NULL, there is a checksum after the data in the file */ do_chk = s3get_gvn_fattr("chksum0"); if (s3read_1d((void **)&veclen, sizeof(uint32), &n_feat, fp, swap, &rd_chksum) != S3_SUCCESS) { return S3_ERROR; } if (s3read((void *)&n_class, sizeof(uint32), 1, fp, swap, &rd_chksum) != 1) { return S3_ERROR; } if (s3read((void *)out_mllr_mult, sizeof(uint32), 1, fp, swap, &rd_chksum) != 1) { return S3_ERROR; } if (s3read((void *)out_mllr_add, sizeof(uint32), 1, fp, swap, &rd_chksum) != 1) { return S3_ERROR; } l_regr = (float32 ****)ckd_calloc_2d(n_class, n_feat, sizeof(float32 ***)); l_regl = (float32 *****)ckd_calloc_2d(n_class, n_feat, sizeof(float32 ****)); for (m = 0; m < n_class; m++) { for (f = 0; f < n_feat; f++) { if (s3read_3d((void ****)&l_regl[m][f], sizeof(float32), &d1, &d2, &d3, fp, swap, &rd_chksum) != S3_SUCCESS) { return S3_ERROR; } if (d1 != veclen[f]) { E_FATAL("left-hand side regression matrices inconsistent w/ feature set. Should be %ux%ux%u, but %ux%ux%u\n", veclen[f], veclen[f]+1, veclen[f]+1, d1, d2, d3); } if ((d2 != d1 + 1) || (d3 != d1 + 1)) { E_FATAL("left-hand side regression matrices should be %ux%ux%u, but %ux%ux%u\n", d1, d1+1, d1+1, d1, d2, d3); } if (s3read_2d((void ***)&l_regr[m][f], sizeof(float32), &d1, &d2, fp, swap, &rd_chksum) != S3_SUCCESS) { return S3_ERROR; } if (d1 != veclen[f]) { E_FATAL("right-hand side regression matrices inconsistent w/ feature set. Should be %ux%u, but %ux%u\n", veclen[f], veclen[f]+1, d1, d2); } if (d2 != d1 + 1) { E_FATAL("right-hand side regression matrices should be %ux%u, but %ux%u\n", d1, d1+1, d1, d2); } } } if (do_chk) { /* See if the checksum in the file matches that which was computed from the read data */ if (s3read(&sv_chksum, sizeof(uint32), 1, fp, swap, &ignore) != 1) { s3close(fp); return S3_ERROR; } if (sv_chksum != rd_chksum) { E_FATAL("Checksum error; read corrupt data.\n"); } } s3close(fp); *out_regl = l_regl; *out_regr = l_regr; *out_n_class = n_class; *out_n_feat = n_feat; *out_veclen = veclen; /* other two output args done above */ E_INFO("Read %s %u*%u regl and regr arrays\n", fn, n_class, n_feat); return S3_SUCCESS; }
int s3regmatcnt_write(const char *fn, float32 ****regr, float32 *****regl, uint32 n_class, uint32 n_feat, const uint32 *veclen, uint32 mllr_mult, uint32 mllr_add) { FILE *fp; uint32 chksum = 0; uint32 ignore = 0; uint32 m, f; s3clr_fattr(); s3add_fattr("version", REGMATCNT_FILE_VERSION, TRUE); s3add_fattr("chksum0", "yes", TRUE); fp = s3open(fn, "wb", NULL); if (fp == NULL) return S3_ERROR; if (s3write_1d((void *)veclen, sizeof(uint32), n_feat, fp, &chksum) != S3_SUCCESS) { return S3_ERROR; } if (s3write((void *)&n_class, sizeof(uint32), 1, fp, &chksum) != 1) { return S3_ERROR; } if (s3write((void *)&mllr_mult, sizeof(uint32), 1, fp, &chksum) != 1) { return S3_ERROR; } if (s3write((void *)&mllr_add, sizeof(uint32), 1, fp, &chksum) != 1) { return S3_ERROR; } for (m = 0; m < n_class; m++) { for (f = 0; f < n_feat; f++) { if (s3write_3d((void ***)regl[m][f], sizeof(float32), veclen[f], veclen[f]+1, veclen[f]+1, fp, &chksum) != S3_SUCCESS) { return S3_ERROR; } if (s3write_2d((void **)regr[m][f], sizeof(float32), veclen[f], veclen[f]+1, fp, &chksum) != S3_SUCCESS) { return S3_ERROR; } } } if (s3write(&chksum, sizeof(uint32), 1, fp, &ignore) != 1) { s3close(fp); return S3_ERROR; } s3close(fp); E_INFO("Wrote %s %u*%u regl and regl arrays.\n", fn, n_class, n_feat); return S3_SUCCESS; }
int s3tmat_read(const char *fn, float32 ****out_tmat, uint32 *out_n_tmat, uint32 *out_n_state) { uint32 rd_chksum = 0; uint32 sv_chksum; uint32 ignore; uint32 tmp; char *ver; char *do_chk; FILE *fp; uint32 swap; fp = s3open(fn, "rb", &swap); if (fp == NULL) return S3_ERROR; /* check version id */ ver = s3get_gvn_fattr("version"); if (ver) { if (strcmp(ver, TMAT_FILE_VERSION) != 0) { E_FATAL("Version mismatch for %s, file ver: %s != reader ver: %s\n", fn, ver, TMAT_FILE_VERSION); } } else { E_FATAL("No version attribute for %s\n", fn); } /* if do_chk is non-NULL, there is a checksum after the data in the file */ do_chk = s3get_gvn_fattr("chksum0"); if (s3read_3d((void ****)out_tmat, sizeof(float32), out_n_tmat, &tmp, out_n_state, fp, swap, &rd_chksum) != S3_SUCCESS) { s3close(fp); return S3_ERROR; } if (do_chk) { if (s3read(&sv_chksum, sizeof(uint32), 1, fp, swap, &ignore) != 1) { s3close(fp); return S3_ERROR; } if (sv_chksum != rd_chksum) { E_FATAL("Checksum error; read corrupted data.\n"); } } s3close(fp); E_INFO("Read %s [%ux%ux%u array]\n", fn, *out_n_tmat, (*out_n_state)-1, *out_n_state); return S3_SUCCESS; }
int s3mixw_read(const char *fn, float32 ****out_mixw, uint32 *out_n_mixw, uint32 *out_n_feat, uint32 *out_n_density) { uint32 rd_chksum = 0; uint32 sv_chksum; uint32 swap; uint32 ignore; char *ver; char *do_chk; FILE *fp; fp = s3open(fn, "rb", &swap); if (fp == NULL) return S3_ERROR; /* check version id */ ver = s3get_gvn_fattr("version"); if (ver) { if (strcmp(ver, MIXW_FILE_VERSION) != 0) { E_FATAL("Version mismatch for %s, file ver: %s != reader ver: %s\n", fn, ver, MIXW_FILE_VERSION); } } else { E_FATAL("No version attribute for %s\n", fn); } /* if do_chk is non-NULL, there is a checksum after the data in the file */ do_chk = s3get_gvn_fattr("chksum0"); /* Read the mixing weight array */ if (bio_fread_3d((void ****)out_mixw, sizeof(float32), out_n_mixw, out_n_feat, out_n_density, fp, swap, &rd_chksum) < 0) { s3close(fp); return S3_ERROR; } if (do_chk) { /* See if the checksum in the file matches that which was computed from the read data */ if (bio_fread(&sv_chksum, sizeof(uint32), 1, fp, swap, &ignore) != 1) { s3close(fp); return S3_ERROR; } if (sv_chksum != rd_chksum) { E_FATAL("Checksum error; read corrupt data.\n"); } } s3close(fp); E_INFO("Read %s [%ux%ux%u array]\n", fn, *out_n_mixw, *out_n_feat, *out_n_density); return S3_SUCCESS; }
int s3gau_read_maybe_full(const char *fn, vector_t *****out, uint32 *out_n_mgau, uint32 *out_n_feat, uint32 *out_n_density, uint32 **out_veclen, uint32 need_full) { FILE *fp; const char *do_chk; const char *ver; uint32 n_mgau, n_feat, n_density; uint32 *veclen, maxveclen; uint32 blk, i, j, k, l, r, n; uint32 chksum = 0; uint32 sv_chksum, ignore = 0; float32 *raw; vector_t ****o; uint32 swap; fp = s3open(fn, "rb", &swap); if (fp == NULL) return S3_ERROR; /* check version id */ ver = s3get_gvn_fattr("version"); if (ver) { if (strcmp(ver, GAU_FILE_VERSION) != 0) { E_FATAL("Version mismatch for %s, file ver: %s != reader ver: %s\n", fn, ver, GAU_FILE_VERSION); } } else { E_FATAL("No version attribute for %s\n", fn); } /* if do_chk is non-NULL, there is a checksum after the data in the file */ do_chk = s3get_gvn_fattr("chksum0"); if (do_chk && !strcmp(do_chk, "no")) { do_chk = NULL; } if (bio_fread(&n_mgau, sizeof(uint32), 1, fp, swap, &chksum) != 1) { goto error; } if (bio_fread(&n_feat, sizeof(uint32), 1, fp, swap, &chksum) != 1) { goto error; } if (bio_fread(&n_density, sizeof(uint32), 1, fp, swap, &chksum) != 1) { goto error; } veclen = ckd_calloc(n_feat, sizeof(uint32)); if (bio_fread(veclen, sizeof(uint32), n_feat, fp, swap, &chksum) != n_feat) { goto error; } if (bio_fread_1d((void **)&raw, sizeof(float32), &n, fp, swap, &chksum) < 0) { ckd_free(veclen); goto error; } for (i = 0, blk = 0, maxveclen = 0; i < n_feat; i++) { blk += veclen[i] * veclen[i]; if (veclen[i] > maxveclen) maxveclen = veclen[i]; } if (n != n_mgau * n_density * blk) { if (need_full) E_ERROR("Failed to read full covariance file %s (expected %d values, got %d)\n", fn, n_mgau * n_density * blk, n); goto error; } o = (vector_t ****)ckd_calloc_4d(n_mgau, n_feat, n_density, maxveclen, sizeof(vector_t)); for (i = 0, r = 0; i < n_mgau; i++) { for (j = 0; j < n_feat; j++) { for (k = 0; k < n_density; k++) { for (l = 0; l < veclen[j]; l++) { o[i][j][k][l] = &raw[r]; r += veclen[j]; } } } } if (do_chk) { /* See if the checksum in the file matches that which was computed from the read data */ if (bio_fread(&sv_chksum, sizeof(uint32), 1, fp, swap, &ignore) != 1) { goto error; } if (sv_chksum != chksum) { E_FATAL("Checksum error; read corrupt data.\n"); } } *out = o; *out_n_mgau = n_mgau; *out_n_feat = n_feat; *out_n_density = n_density; *out_veclen = veclen; s3close(fp); E_INFO("Read %s [%ux%ux%u array of full matrices]\n", fn, n_mgau, n_feat, n_density); return S3_SUCCESS; error: if (fp) s3close(fp); return S3_ERROR; }
int s3gaucnt_write_full(const char *fn, vector_t ***wt_mean, vector_t ****wt_var, int32 pass2var, float32 ***dnom, uint32 n_cb, uint32 n_feat, uint32 n_density, const uint32 *veclen) { FILE *fp; int32 chksum = 0; int32 ignore = 0; uint32 n_elem, blk, j, has_means, has_vars; s3clr_fattr(); s3add_fattr("version", GAUCNT_FILE_VERSION, TRUE); s3add_fattr("chksum0", "yes", TRUE); fp = s3open(fn, "wb", NULL); if (fp == NULL) return S3_ERROR; if (wt_mean != NULL) has_means = TRUE; else has_means = FALSE; if (bio_fwrite((void *)&has_means, sizeof(uint32), 1, fp, 0, &chksum) != 1) { return S3_ERROR; } if (wt_var != NULL) has_vars = TRUE; else has_vars = FALSE; if (bio_fwrite((void *)&has_vars, sizeof(uint32), 1, fp, 0, &chksum) != 1) { return S3_ERROR; } if (bio_fwrite((void *)&pass2var, sizeof(uint32), 1, fp, 0, &chksum) != 1) { return S3_ERROR; } if (bio_fwrite((void *)&n_cb, sizeof(uint32), 1, fp, 0, &chksum) != 1) { return S3_ERROR; } if (bio_fwrite((void *)&n_density, sizeof(uint32), 1, fp, 0, &chksum) != 1) { return S3_ERROR; } if (bio_fwrite_1d((void *)veclen, sizeof(uint32), n_feat, fp, &chksum) < 0) { return S3_ERROR; } for (j = 0, blk = 0; j < n_feat; j++) blk += veclen[j]; n_elem = n_cb * n_density * blk; if (has_means) { band_nz_1d(wt_mean[0][0][0], n_elem, MIN_POS_FLOAT32); if (bio_fwrite_1d((void *)wt_mean[0][0][0], sizeof(float32), n_elem, fp, &chksum) < 0) return S3_ERROR; } if (has_vars) { /* Don't floor full variances!!! */ if (bio_fwrite_1d((void *)wt_var[0][0][0][0], sizeof(float32), n_elem * blk, fp, &chksum) < 0) return S3_ERROR; } /* floor all non-zero entries to this value to make sure that results are compatible between machines */ floor_nz_3d(dnom, n_cb, n_feat, n_density, MIN_POS_FLOAT32); if (bio_fwrite_3d((void ***)dnom, sizeof(float32), n_cb, n_feat, n_density, fp, &chksum) < 0) { return S3_ERROR; } if (bio_fwrite(&chksum, sizeof(uint32), 1, fp, 0, &ignore) != 1) { s3close(fp); return S3_ERROR; } s3close(fp); E_INFO("Wrote %s%s%s%s [%ux%ux%u vector/matrix arrays]\n", fn, (has_means ? " with means" : ""), (has_vars ? " with full vars" : ""), (has_vars && pass2var ? " (2pass)" : ""), n_cb, n_feat, n_density); return S3_SUCCESS; }
int s3gaucnt_read_full(const char *fn, vector_t ****out_wt_mean, vector_t *****out_wt_var, int32 *out_pass2var, float32 ****out_dnom, uint32 *out_n_cb, uint32 *out_n_feat, uint32 *out_n_density, uint32 **out_veclen) { uint32 rd_chksum = 0; uint32 sv_chksum; uint32 ignore; char *ver; char *do_chk; FILE *fp; uint32 swap; uint32 has_means; uint32 has_vars; uint32 pass2var; uint32 n_cb; uint32 n_feat; uint32 n_density; uint32 *veclen; float32 *buf; float32 ***dnom; uint32 n, i, b_i, j, k, l, d1, d2, d3; vector_t ***wt_mean = NULL; vector_t ****wt_var = NULL; fp = s3open(fn, "rb", &swap); if (fp == NULL) return S3_ERROR; /* check version id */ ver = s3get_gvn_fattr("version"); if (ver) { if (strcmp(ver, GAUCNT_FILE_VERSION) != 0) { E_FATAL("Version mismatch for %s, file ver: %s != reader ver: %s\n", fn, ver, GAUCNT_FILE_VERSION); } } else { E_FATAL("No version attribute for %s\n", fn); } /* if do_chk is non-NULL, there is a checksum after the data in the file */ do_chk = s3get_gvn_fattr("chksum0"); if (bio_fread((void *)&has_means, sizeof(uint32), 1, fp, swap, &rd_chksum) != 1) { return S3_ERROR; } if (bio_fread((void *)&has_vars, sizeof(uint32), 1, fp, swap, &rd_chksum) != 1) { return S3_ERROR; } if (bio_fread((void *)&pass2var, sizeof(uint32), 1, fp, swap, &rd_chksum) != 1) { return S3_ERROR; } if (bio_fread((void *)&n_cb, sizeof(uint32), 1, fp, swap, &rd_chksum) != 1) { return S3_ERROR; } if (bio_fread((void *)&n_density, sizeof(uint32), 1, fp, swap, &rd_chksum) != 1) { return S3_ERROR; } if (bio_fread_1d((void **)&veclen, sizeof(uint32), &n_feat, fp, swap, &rd_chksum) < 0) { return S3_ERROR; } if (has_means) { if (bio_fread_1d((void *)&buf, sizeof(float32), &n, fp, swap, &rd_chksum) < 0) { return S3_ERROR; } wt_mean = (vector_t ***)ckd_calloc_3d(n_cb, n_feat, n_density, sizeof(vector_t)); for (i = 0, b_i = 0; i < n_cb; i++) { for (j = 0; j < n_feat; j++) { for (k = 0; k < n_density; k++) { wt_mean[i][j][k] = &buf[b_i]; b_i += veclen[j]; } } } } if (has_vars) { uint32 blk, maxveclen; for (i = 0, blk = 0, maxveclen = 0; i < n_feat; i++) { blk += veclen[i]; if (veclen[i] > maxveclen) maxveclen = veclen[i]; } if (bio_fread_1d((void *)&buf, sizeof(float32), &n, fp, swap, &rd_chksum) < 0) { return S3_ERROR; } assert(n == n_cb * n_density * blk * blk); wt_var = (vector_t ****)ckd_calloc_4d(n_cb, n_feat, n_density, maxveclen, sizeof(vector_t)); for (i = 0, b_i = 0; i < n_cb; i++) { for (j = 0; j < n_feat; j++) { for (k = 0; k < n_density; k++) { for (l = 0; l < veclen[j]; l++) { wt_var[i][j][k][l] = &buf[b_i]; b_i += veclen[j]; } } } } } if (bio_fread_3d((void ****)&dnom, sizeof(float32), &d1, &d2, &d3, fp, swap, &rd_chksum) < 0) { return S3_ERROR; } assert(d1 == n_cb); assert(d2 == n_feat); assert(d3 == n_density); if (do_chk) { /* See if the checksum in the file matches that which was computed from the read data */ if (bio_fread(&sv_chksum, sizeof(uint32), 1, fp, swap, &ignore) != 1) { s3close(fp); return S3_ERROR; } if (sv_chksum != rd_chksum) { E_FATAL("Checksum error; read corrupt data.\n"); } } s3close(fp); *out_wt_mean = wt_mean; *out_wt_var = wt_var; *out_pass2var = pass2var; *out_dnom = dnom; *out_n_cb = n_cb; *out_n_feat = n_feat; *out_n_density = n_density; *out_veclen = veclen; E_INFO("Read %s%s%s%s [%ux%ux%u vector arrays]\n", fn, (has_means ? " with means" : ""), (has_vars ? " with vars" : ""), (has_vars && pass2var ? " (2pass)" : ""), n_cb, n_feat, n_density); return S3_SUCCESS; }
int s3lamb_read(const char *fn, float32 **out_lambda, uint32 *out_n_lambda) { FILE *fp; const char *ver; const char *do_chk; uint32 rd_chksum = 0, sv_chksum, ignore; float32 *lambda; uint32 n_lambda; uint32 swap; fp = s3open(fn, "rb", &swap); if (fp == NULL) return S3_ERROR; /* check version io */ ver = s3get_gvn_fattr("version"); if (ver) { if (strcmp(ver, LAMBDA_FILE_VERSION) != 0) { E_FATAL("Version mismatch for %s, file ver: %s != reader ver: %s\n", fn, ver, LAMBDA_FILE_VERSION); } } else { E_FATAL("No version attribute for %s\n", fn); } /* if do_chk is non-NULL, there is a checksum after the data in the file */ do_chk = s3get_gvn_fattr("chksum0"); if (s3read_1d((void **)&lambda, sizeof(float32), &n_lambda, fp, swap, &rd_chksum) != S3_SUCCESS) return S3_ERROR; if (do_chk) { /* See if the checksum in the file matches that which was computed from the read data */ if (s3read(&sv_chksum, sizeof(uint32), 1, fp, swap, &ignore) != 1) { s3close(fp); return S3_ERROR; } if (sv_chksum != rd_chksum) { E_FATAL("Checksum error; read corrupt data.\n"); } } s3close(fp); *out_lambda = lambda; *out_n_lambda = n_lambda; E_INFO("Read %s [%u array]\n", fn, n_lambda); return S3_SUCCESS; }
int s3map_read(const char *fn, void **out_map, uint32 *out_n_dom, uint32 *out_n_rng, size_t map_elem_size) { uint32 rd_chksum = 0; uint32 sv_chksum; uint32 ignore; char *ver; char *do_chk; FILE *fp; uint32 swap; fp = s3open(fn, "rb", &swap); if (fp == NULL) { return S3_ERROR; } /* check version id */ ver = s3get_gvn_fattr("version"); if (ver) { if (strcmp(ver, MAP_FILE_VERSION) != 0) { E_FATAL("Version mismatch for %s, file ver: %s != reader ver: %s\n", fn, ver, MAP_FILE_VERSION); } } else { E_FATAL("No version attribute for %s\n", fn); } do_chk = s3get_gvn_fattr("chksum0"); if (bio_fread(out_n_rng, sizeof(uint32), 1, fp, swap, &rd_chksum) != 1) { s3close(fp); return S3_ERROR; } if (bio_fread_1d(out_map, map_elem_size, out_n_dom, fp, swap, &rd_chksum) < 0) { s3close(fp); return S3_ERROR; } if (do_chk) { if (bio_fread(&sv_chksum, sizeof(uint32), 1, fp, swap, &ignore) != 1) { s3close(fp); return S3_ERROR; } if (sv_chksum != rd_chksum) { E_FATAL("Checksum error; read corrupted data.\n"); } } E_INFO("Read %s [%u mappings to %u]\n", fn, *out_n_dom, *out_n_rng); return S3_SUCCESS; }