int
s3lambcnt_write(const char *fn,
const uint32 *lambda_cnt,
uint32 n_lambda_cnt)
{
FILE *fp;
uint32 chksum = 0;
uint32 ignore = 0;
s3clr_fattr();
s3add_fattr("version", LAMBDACNT_FILE_VERSION, TRUE);
s3add_fattr("chksum0", "yes", TRUE);
fp = s3open(fn, "wb", NULL);
if (fp == NULL)
return S3_ERROR;
if (s3write_1d((void *)lambda_cnt, sizeof(uint32), n_lambda_cnt, fp, &chksum) != S3_SUCCESS) {
s3close(fp);
return S3_ERROR;
}
if (s3write(&chksum, sizeof(uint32), 1, fp, &ignore) != 1) {
s3close(fp);
return S3_ERROR;
}
s3close(fp);
E_INFO("Wrote %s [%u array]\n", fn, n_lambda_cnt);
return S3_SUCCESS;
}
int
s3gau_write_full(const char *fn,
const vector_t ****out,
uint32 n_mgau,
uint32 n_feat,
uint32 n_density,
const uint32 *veclen)
{
FILE *fp = NULL;
uint32 chksum = 0;
uint32 blk, i, ignore = 0;
s3clr_fattr();
s3add_fattr("version", GAU_FILE_VERSION, TRUE);
s3add_fattr("chksum0", "yes", TRUE);
fp = s3open(fn, "wb", NULL);
if (fp == NULL)
return S3_ERROR;
if (bio_fwrite(&n_mgau, sizeof(uint32), 1, fp, 0, &chksum) != 1) {
goto error;
}
if (bio_fwrite(&n_feat, sizeof(uint32), 1, fp, 0, &chksum) != 1) {
goto error;
}
if (bio_fwrite(&n_density, sizeof(uint32), 1, fp, 0, &chksum) != 1) {
goto error;
}
if (bio_fwrite(veclen, sizeof(uint32), n_feat, fp, 0, &chksum) != n_feat) {
goto error;
}
for (blk = 0, i = 0; i < n_feat; i++)
blk += veclen[i] * veclen[i];
if (bio_fwrite_1d(out[0][0][0][0], sizeof(float32),
n_mgau*n_density*blk, fp, &chksum) < 0) {
goto error;
}
if (bio_fwrite(&chksum, sizeof(uint32), 1, fp, 0, &ignore) != 1) {
goto error;
}
s3close(fp);
E_INFO("Wrote %s [%ux%ux%u array of full matrices]\n",
fn, n_mgau, n_feat, n_density);
return S3_SUCCESS;
error:
if (fp) s3close(fp);
return S3_ERROR;
}
int
s3mixw_write(const char *fn,
float32 ***mixw,
uint32 n_mixw,
uint32 n_feat,
uint32 n_density)
{
FILE *fp;
uint32 chksum = 0;
uint32 ignore = 0;
uint32 i, j, k;
s3clr_fattr();
s3add_fattr("version", MIXW_FILE_VERSION, TRUE);
s3add_fattr("chksum0", "yes", TRUE);
fp = s3open(fn, "wb", NULL);
if (fp == NULL)
return S3_ERROR;
for (i = 0; i < n_mixw; i++) {
for (j = 0; j < n_feat; j++) {
for (k = 0; k < n_density; k++) {
if (mixw[i][j][k] < 0) {
E_ERROR("mixw[%u][%u][%u] < 0 (%e)\n",
i,j,k,mixw[i][j][k]);
}
}
}
}
/* floor all non-zero entries to this value to make sure
that results are compatible between machines */
floor_nz_3d(mixw, n_mixw, n_feat, n_density, MIN_POS_FLOAT32);
if (bio_fwrite_3d((void ***)mixw,
sizeof(float32),
n_mixw,
n_feat,
n_density,
fp,
&chksum) < 0) {
s3close(fp);
return S3_ERROR;
}
if (bio_fwrite(&chksum, sizeof(uint32), 1, fp, 0, &ignore) != 1) {
s3close(fp);
return S3_ERROR;
}
s3close(fp);
E_INFO("Wrote %s [%ux%ux%u array]\n",
fn, n_mixw, n_feat, n_density);
return S3_SUCCESS;
}
int
s3tmat_write(const char *fn,
float32 ***tmat,
uint32 n_tmat,
uint32 n_state)
{
FILE *fp;
uint32 chksum = 0;
uint32 ignore = 0;
int t, i, j;
s3clr_fattr();
s3add_fattr("version", TMAT_FILE_VERSION, TRUE);
s3add_fattr("chksum0", "yes", TRUE);
fp = s3open(fn, "wb", NULL);
if (fp == NULL)
return S3_ERROR;
for (t = 0; t < n_tmat; t++) {
for (i = 0; i < n_state-1; i++) {
for (j = 0; j < n_state; j++) {
if (tmat[t][i][j] < 0) {
E_ERROR("tmat[%u][%u][%u] < 0 (%e)\n",
t, i, j, tmat[t][i][j]);
}
}
}
}
/* floor all non-zero entries to this value to make sure
that results are compatible between machines */
floor_nz_3d(tmat, n_tmat, n_state-1, n_state, MIN_POS_FLOAT32);
if (s3write_3d((void ***)tmat,
sizeof(float32),
n_tmat,
n_state-1,
n_state,
fp,
&chksum) != S3_SUCCESS) {
s3close(fp);
return S3_ERROR;
}
if (s3write(&chksum, sizeof(uint32), 1, fp, &ignore) != 1) {
s3close(fp);
return S3_ERROR;
}
s3close(fp);
E_INFO("Wrote %s [%ux%ux%u array]\n",
fn, n_tmat, n_state-1, n_state);
return S3_SUCCESS;
}
int
s3mixw_intv_read(const char *fn,
uint32 mixw_s,
uint32 mixw_e,
float32 ****out_mixw,
uint32 *out_n_mixw,
uint32 *out_n_feat,
uint32 *out_n_density)
{
uint32 ignore;
char *ver;
char *do_chk;
FILE *fp;
uint32 swap;
fp = s3open(fn, "rb", &swap);
if (fp == NULL)
return S3_ERROR;
/* check version id */
ver = s3get_gvn_fattr("version");
if (ver) {
if (strcmp(ver, MIXW_FILE_VERSION) != 0) {
E_FATAL("Version mismatch for %s, file ver: %s != reader ver: %s\n",
fn, ver, MIXW_FILE_VERSION);
}
}
else {
E_FATAL("No version attribute for %s\n", fn);
}
/* if do_chk is non-NULL, there is a checksum after the data in the file */
do_chk = s3get_gvn_fattr("chksum0");
/* Read the mixing weight array */
if (bio_fread_intv_3d((void ****)out_mixw,
sizeof(float32),
mixw_s,
mixw_e,
out_n_mixw,
out_n_feat,
out_n_density,
fp,
swap,
&ignore) != S3_SUCCESS) {
s3close(fp);
return S3_ERROR;
}
s3close(fp);
E_INFO("Read %s [%ux%ux%u array]\n",
fn, *out_n_mixw, *out_n_feat, *out_n_density);
return S3_SUCCESS;
}
int
s3map_write(const char *fn,
void *map,
uint32 n_dom,
uint32 n_rng,
size_t map_elem_size)
{
FILE *fp;
uint32 chksum = 0;
uint32 ignore = 0;
s3clr_fattr();
s3add_fattr("version", MAP_FILE_VERSION, TRUE);
s3add_fattr("chksum0", "yes", TRUE);
fp = s3open(fn, "wb", NULL);
if (fp == NULL)
return S3_ERROR;
if (bio_fwrite(&n_rng, sizeof(uint32), 1, fp, &chksum) != 1) {
s3close(fp);
return S3_ERROR;
}
if (bio_fwrite_1d(map,
map_elem_size,
n_dom,
fp,
&chksum) < 0) {
s3close(fp);
return S3_ERROR;
}
if (bio_fwrite(&chksum, sizeof(uint32), 1, fp, &ignore) != 1) {
s3close(fp);
return S3_ERROR;
}
s3close(fp);
E_INFO("Wrote %s [%u mappings to %u]\n", fn, n_dom, n_rng);
return S3_SUCCESS;
}
static int
init_state(const char *obsdmp,
const char *obsidx,
uint32 n_density,
uint32 n_stream,
const uint32 *veclen,
int reest,
const char *mixwfn,
const char *meanfn,
const char *varfn,
uint32 ts_off,
uint32 ts_cnt,
uint32 n_ts,
uint32 n_d_ts)
{
uint32 blksz;
vector_t ***mean;
vector_t ***var = NULL;
vector_t ****fullvar = NULL;
float32 ***mixw = NULL;
uint32 n_frame;
uint32 ignore = 0;
codew_t *label;
uint32 n_corpus = 0;
float64 sqerr;
float64 tot_sqerr;
segdmp_type_t t;
uint32 i, j, ts, n;
timing_t *km_timer;
timing_t *var_timer;
timing_t *em_timer;
int32 full_covar;
km_timer = timing_get("km");
var_timer = timing_get("var");
em_timer = timing_get("em");
blksz = feat_blksize();
full_covar = cmd_ln_int32("-fullvar");
/* fully-continuous for now */
mean = gauden_alloc_param(ts_cnt, n_stream, n_density, veclen);
if (full_covar)
fullvar = gauden_alloc_param_full(ts_cnt, n_stream, n_density, veclen);
else
var = gauden_alloc_param(ts_cnt, n_stream, n_density, veclen);
if (mixwfn)
mixw = (float32 ***)ckd_calloc_3d(ts_cnt,
n_stream,
n_density,
sizeof(float32));
if ((const char *)cmd_ln_access("-segidxfn")) {
E_INFO("Multi-class dump\n");
if (segdmp_open_read((const char **)cmd_ln_access("-segdmpdirs"),
(const char *)cmd_ln_access("-segdmpfn"),
(const char *)cmd_ln_access("-segidxfn"),
&n,
&t) != S3_SUCCESS) {
E_FATAL("Unable to open dumps\n");
}
if (n != n_d_ts) {
E_FATAL("Expected %u tied-states in dump, but apparently %u\n",
n_d_ts, n);
}
if (t != SEGDMP_TYPE_FEAT) {
E_FATAL("Expected feature dump, but instead saw %u\n", t);
}
multiclass = TRUE;
}
else {
E_INFO("1-class dump file\n");
multiclass = FALSE;
dmp_fp = s3open((const char *)cmd_ln_access("-segdmpfn"), "rb",
&dmp_swp);
if (dmp_fp == NULL) {
E_ERROR_SYSTEM("Unable to open dump file %s for reading\n",
(const char *)cmd_ln_access("-segdmpfn"));
return S3_ERROR;
}
if (s3read(&n_frame, sizeof(uint32), 1, dmp_fp, dmp_swp, &ignore) != 1) {
E_ERROR_SYSTEM("Unable to open dump file %s for reading\n",
(const char *)cmd_ln_access("-segdmpfn"));
return S3_ERROR;
}
data_offset = ftell(dmp_fp);
}
tot_sqerr = 0;
for (i = 0; i < ts_cnt; i++) {
ts = ts_off + i;
/* stride not accounted for yet */
if (o2d == NULL) {
if (multiclass)
n_frame = segdmp_n_seg(ts);
}
else {
for (j = 0, n_frame = 0; j < n_o2d[ts]; j++) {
n_frame += segdmp_n_seg(o2d[ts][j]);
}
}
E_INFO("Corpus %u: sz==%u frames%s\n",
ts, n_frame,
(n_frame > *(uint32 *)cmd_ln_access("-vartiethr") ? "" : " tied var"));
if (n_frame == 0) {
continue;
}
E_INFO("Convergence ratios are abs(cur - prior) / abs(prior)\n");
/* Do some variety of k-means clustering */
if (km_timer)
timing_start(km_timer);
sqerr = cluster(ts, n_stream, n_frame, veclen, mean[i], n_density, &label);
if (km_timer)
timing_stop(km_timer);
if (sqerr < 0) {
E_ERROR("Unable to do k-means for state %u; skipping...\n", ts);
continue;
}
/* Given the k-means and assuming equal prior liklihoods
* compute the variances */
if (var_timer)
timing_start(var_timer);
if (full_covar)
full_variances(ts, mean[i], fullvar[i], n_density, veclen,
n_frame, n_stream, label);
else
variances(ts, mean[i], var[i], n_density, veclen, n_frame, n_stream, label);
if (var_timer)
timing_stop(var_timer);
if (mixwfn) {
/* initialize the mixing weights by counting # of occurrances
* of the top codeword over the corpus and normalizing */
init_mixw(mixw[i], mean[i], n_density, veclen, n_frame, n_stream, label);
ckd_free(label);
if (reest == TRUE && full_covar)
E_ERROR("EM re-estimation is not yet supported for full covariances\n");
else if (reest == TRUE) {
if (em_timer)
timing_start(em_timer);
/* Do iterations of EM to estimate the mixture densities */
reest_sum(ts, mean[i], var[i], mixw[i], n_density, n_stream,
n_frame, veclen,
*(uint32 *)cmd_ln_access("-niter"),
FALSE,
*(uint32 *)cmd_ln_access("-vartiethr"));
if (em_timer)
timing_stop(em_timer);
}
}
++n_corpus;
tot_sqerr += sqerr;
E_INFO("sqerr [%u] == %e\n", ts, sqerr);
}
if (n_corpus > 0) {
E_INFO("sqerr = %e tot %e rms\n", tot_sqerr, sqrt(tot_sqerr/n_corpus));
}
if (!multiclass)
s3close(dmp_fp);
if (meanfn) {
if (s3gau_write(meanfn,
(const vector_t ***)mean,
ts_cnt,
n_stream,
n_density,
veclen) != S3_SUCCESS) {
return S3_ERROR;
}
}
else {
E_INFO("No mean file given; none written\n");
}
if (varfn) {
if (full_covar) {
if (s3gau_write_full(varfn,
(const vector_t ****)fullvar,
ts_cnt,
n_stream,
n_density,
veclen) != S3_SUCCESS)
return S3_ERROR;
}
else {
if (s3gau_write(varfn,
(const vector_t ***)var,
ts_cnt,
n_stream,
n_density,
veclen) != S3_SUCCESS)
return S3_ERROR;
}
}
else {
E_INFO("No variance file given; none written\n");
}
if (mixwfn) {
if (s3mixw_write(mixwfn,
mixw,
ts_cnt,
n_stream,
n_density) != S3_SUCCESS) {
return S3_ERROR;
}
}
else {
E_INFO("No mixing weight file given; none written\n");
}
return S3_SUCCESS;
}
int
s3regmatcnt_read(const char *fn,
float32 *****out_regr,
float32 ******out_regl,
uint32 *out_n_class,
uint32 *out_n_feat,
const uint32 **out_veclen,
uint32 *out_mllr_mult,
uint32 *out_mllr_add)
{
uint32 rd_chksum = 0;
uint32 sv_chksum;
uint32 ignore;
char *ver;
char *do_chk;
FILE *fp;
uint32 n_feat, n_class;
const uint32 *veclen;
float32 ****l_regr;
float32 *****l_regl;
uint32 d1, d2, d3, m, f;
uint32 swap;
fp = s3open(fn, "rb", &swap);
if (fp == NULL)
return S3_ERROR;
/* check version id */
ver = s3get_gvn_fattr("version");
if (ver) {
if (strcmp(ver, REGMATCNT_FILE_VERSION) != 0) {
E_FATAL("Version mismatch for %s, file ver: %s != reader ver: %s\n",
fn, ver, REGMATCNT_FILE_VERSION);
}
}
else {
E_FATAL("No version attribute for %s\n", fn);
}
/* if do_chk is non-NULL, there is a checksum after the data in the file */
do_chk = s3get_gvn_fattr("chksum0");
if (s3read_1d((void **)&veclen, sizeof(uint32), &n_feat, fp, swap, &rd_chksum) != S3_SUCCESS) {
return S3_ERROR;
}
if (s3read((void *)&n_class, sizeof(uint32), 1, fp, swap, &rd_chksum) != 1) {
return S3_ERROR;
}
if (s3read((void *)out_mllr_mult, sizeof(uint32), 1, fp, swap, &rd_chksum) != 1) {
return S3_ERROR;
}
if (s3read((void *)out_mllr_add, sizeof(uint32), 1, fp, swap, &rd_chksum) != 1) {
return S3_ERROR;
}
l_regr = (float32 ****)ckd_calloc_2d(n_class, n_feat, sizeof(float32 ***));
l_regl = (float32 *****)ckd_calloc_2d(n_class, n_feat, sizeof(float32 ****));
for (m = 0; m < n_class; m++) {
for (f = 0; f < n_feat; f++) {
if (s3read_3d((void ****)&l_regl[m][f],
sizeof(float32), &d1, &d2, &d3, fp, swap, &rd_chksum) != S3_SUCCESS) {
return S3_ERROR;
}
if (d1 != veclen[f]) {
E_FATAL("left-hand side regression matrices inconsistent w/ feature set. Should be %ux%ux%u, but %ux%ux%u\n", veclen[f], veclen[f]+1, veclen[f]+1, d1, d2, d3);
}
if ((d2 != d1 + 1) ||
(d3 != d1 + 1)) {
E_FATAL("left-hand side regression matrices should be %ux%ux%u, but %ux%ux%u\n", d1, d1+1, d1+1, d1, d2, d3);
}
if (s3read_2d((void ***)&l_regr[m][f], sizeof(float32), &d1, &d2, fp, swap, &rd_chksum) != S3_SUCCESS) {
return S3_ERROR;
}
if (d1 != veclen[f]) {
E_FATAL("right-hand side regression matrices inconsistent w/ feature set. Should be %ux%u, but %ux%u\n", veclen[f], veclen[f]+1, d1, d2);
}
if (d2 != d1 + 1) {
E_FATAL("right-hand side regression matrices should be %ux%u, but %ux%u\n", d1, d1+1, d1, d2);
}
}
}
if (do_chk) {
/* See if the checksum in the file matches that which
was computed from the read data */
if (s3read(&sv_chksum, sizeof(uint32), 1, fp, swap, &ignore) != 1) {
s3close(fp);
return S3_ERROR;
}
if (sv_chksum != rd_chksum) {
E_FATAL("Checksum error; read corrupt data.\n");
}
}
s3close(fp);
*out_regl = l_regl;
*out_regr = l_regr;
*out_n_class = n_class;
*out_n_feat = n_feat;
*out_veclen = veclen;
/* other two output args done above */
E_INFO("Read %s %u*%u regl and regr arrays\n",
fn, n_class, n_feat);
return S3_SUCCESS;
}
int
s3regmatcnt_write(const char *fn,
float32 ****regr,
float32 *****regl,
uint32 n_class,
uint32 n_feat,
const uint32 *veclen,
uint32 mllr_mult,
uint32 mllr_add)
{
FILE *fp;
uint32 chksum = 0;
uint32 ignore = 0;
uint32 m, f;
s3clr_fattr();
s3add_fattr("version", REGMATCNT_FILE_VERSION, TRUE);
s3add_fattr("chksum0", "yes", TRUE);
fp = s3open(fn, "wb", NULL);
if (fp == NULL)
return S3_ERROR;
if (s3write_1d((void *)veclen, sizeof(uint32), n_feat, fp, &chksum) != S3_SUCCESS) {
return S3_ERROR;
}
if (s3write((void *)&n_class, sizeof(uint32), 1, fp, &chksum) != 1) {
return S3_ERROR;
}
if (s3write((void *)&mllr_mult, sizeof(uint32), 1, fp, &chksum) != 1) {
return S3_ERROR;
}
if (s3write((void *)&mllr_add, sizeof(uint32), 1, fp, &chksum) != 1) {
return S3_ERROR;
}
for (m = 0; m < n_class; m++) {
for (f = 0; f < n_feat; f++) {
if (s3write_3d((void ***)regl[m][f], sizeof(float32),
veclen[f], veclen[f]+1, veclen[f]+1, fp, &chksum) != S3_SUCCESS) {
return S3_ERROR;
}
if (s3write_2d((void **)regr[m][f], sizeof(float32),
veclen[f], veclen[f]+1, fp, &chksum) != S3_SUCCESS) {
return S3_ERROR;
}
}
}
if (s3write(&chksum, sizeof(uint32), 1, fp, &ignore) != 1) {
s3close(fp);
return S3_ERROR;
}
s3close(fp);
E_INFO("Wrote %s %u*%u regl and regl arrays.\n",
fn, n_class, n_feat);
return S3_SUCCESS;
}
int
s3tmat_read(const char *fn,
float32 ****out_tmat,
uint32 *out_n_tmat,
uint32 *out_n_state)
{
uint32 rd_chksum = 0;
uint32 sv_chksum;
uint32 ignore;
uint32 tmp;
char *ver;
char *do_chk;
FILE *fp;
uint32 swap;
fp = s3open(fn, "rb", &swap);
if (fp == NULL)
return S3_ERROR;
/* check version id */
ver = s3get_gvn_fattr("version");
if (ver) {
if (strcmp(ver, TMAT_FILE_VERSION) != 0) {
E_FATAL("Version mismatch for %s, file ver: %s != reader ver: %s\n",
fn, ver, TMAT_FILE_VERSION);
}
}
else {
E_FATAL("No version attribute for %s\n", fn);
}
/* if do_chk is non-NULL, there is a checksum after the data in the file */
do_chk = s3get_gvn_fattr("chksum0");
if (s3read_3d((void ****)out_tmat,
sizeof(float32),
out_n_tmat,
&tmp,
out_n_state,
fp,
swap,
&rd_chksum) != S3_SUCCESS) {
s3close(fp);
return S3_ERROR;
}
if (do_chk) {
if (s3read(&sv_chksum, sizeof(uint32), 1, fp, swap, &ignore) != 1) {
s3close(fp);
return S3_ERROR;
}
if (sv_chksum != rd_chksum) {
E_FATAL("Checksum error; read corrupted data.\n");
}
}
s3close(fp);
E_INFO("Read %s [%ux%ux%u array]\n",
fn, *out_n_tmat, (*out_n_state)-1, *out_n_state);
return S3_SUCCESS;
}
int
s3gau_read_maybe_full(const char *fn,
vector_t *****out,
uint32 *out_n_mgau,
uint32 *out_n_feat,
uint32 *out_n_density,
uint32 **out_veclen,
uint32 need_full)
{
FILE *fp;
const char *do_chk;
const char *ver;
uint32 n_mgau, n_feat, n_density;
uint32 *veclen, maxveclen;
uint32 blk, i, j, k, l, r, n;
uint32 chksum = 0;
uint32 sv_chksum, ignore = 0;
float32 *raw;
vector_t ****o;
uint32 swap;
fp = s3open(fn, "rb", &swap);
if (fp == NULL)
return S3_ERROR;
/* check version id */
ver = s3get_gvn_fattr("version");
if (ver) {
if (strcmp(ver, GAU_FILE_VERSION) != 0) {
E_FATAL("Version mismatch for %s, file ver: %s != reader ver: %s\n",
fn, ver, GAU_FILE_VERSION);
}
}
else {
E_FATAL("No version attribute for %s\n", fn);
}
/* if do_chk is non-NULL, there is a checksum after the data in the file */
do_chk = s3get_gvn_fattr("chksum0");
if (do_chk && !strcmp(do_chk, "no")) {
do_chk = NULL;
}
if (bio_fread(&n_mgau, sizeof(uint32), 1, fp, swap, &chksum) != 1) {
goto error;
}
if (bio_fread(&n_feat, sizeof(uint32), 1, fp, swap, &chksum) != 1) {
goto error;
}
if (bio_fread(&n_density, sizeof(uint32), 1, fp, swap, &chksum) != 1) {
goto error;
}
veclen = ckd_calloc(n_feat, sizeof(uint32));
if (bio_fread(veclen, sizeof(uint32), n_feat, fp, swap, &chksum) != n_feat) {
goto error;
}
if (bio_fread_1d((void **)&raw, sizeof(float32), &n, fp, swap, &chksum) < 0) {
ckd_free(veclen);
goto error;
}
for (i = 0, blk = 0, maxveclen = 0; i < n_feat; i++) {
blk += veclen[i] * veclen[i];
if (veclen[i] > maxveclen) maxveclen = veclen[i];
}
if (n != n_mgau * n_density * blk) {
if (need_full)
E_ERROR("Failed to read full covariance file %s (expected %d values, got %d)\n",
fn, n_mgau * n_density * blk, n);
goto error;
}
o = (vector_t ****)ckd_calloc_4d(n_mgau, n_feat, n_density,
maxveclen, sizeof(vector_t));
for (i = 0, r = 0; i < n_mgau; i++) {
for (j = 0; j < n_feat; j++) {
for (k = 0; k < n_density; k++) {
for (l = 0; l < veclen[j]; l++) {
o[i][j][k][l] = &raw[r];
r += veclen[j];
}
}
}
}
if (do_chk) {
/* See if the checksum in the file matches that which
was computed from the read data */
if (bio_fread(&sv_chksum, sizeof(uint32), 1, fp, swap, &ignore) != 1) {
goto error;
}
if (sv_chksum != chksum) {
E_FATAL("Checksum error; read corrupt data.\n");
}
}
*out = o;
*out_n_mgau = n_mgau;
*out_n_feat = n_feat;
*out_n_density = n_density;
*out_veclen = veclen;
s3close(fp);
E_INFO("Read %s [%ux%ux%u array of full matrices]\n",
fn, n_mgau, n_feat, n_density);
return S3_SUCCESS;
error:
if (fp) s3close(fp);
return S3_ERROR;
}
int
s3gaucnt_write_full(const char *fn,
vector_t ***wt_mean,
vector_t ****wt_var,
int32 pass2var,
float32 ***dnom,
uint32 n_cb,
uint32 n_feat,
uint32 n_density,
const uint32 *veclen)
{
FILE *fp;
int32 chksum = 0;
int32 ignore = 0;
uint32 n_elem, blk, j, has_means, has_vars;
s3clr_fattr();
s3add_fattr("version", GAUCNT_FILE_VERSION, TRUE);
s3add_fattr("chksum0", "yes", TRUE);
fp = s3open(fn, "wb", NULL);
if (fp == NULL)
return S3_ERROR;
if (wt_mean != NULL)
has_means = TRUE;
else
has_means = FALSE;
if (bio_fwrite((void *)&has_means, sizeof(uint32), 1, fp, 0, &chksum) != 1) {
return S3_ERROR;
}
if (wt_var != NULL)
has_vars = TRUE;
else
has_vars = FALSE;
if (bio_fwrite((void *)&has_vars, sizeof(uint32), 1, fp, 0, &chksum) != 1) {
return S3_ERROR;
}
if (bio_fwrite((void *)&pass2var, sizeof(uint32), 1, fp, 0, &chksum) != 1) {
return S3_ERROR;
}
if (bio_fwrite((void *)&n_cb, sizeof(uint32), 1, fp, 0, &chksum) != 1) {
return S3_ERROR;
}
if (bio_fwrite((void *)&n_density, sizeof(uint32), 1, fp, 0, &chksum) != 1) {
return S3_ERROR;
}
if (bio_fwrite_1d((void *)veclen, sizeof(uint32), n_feat, fp, &chksum) < 0) {
return S3_ERROR;
}
for (j = 0, blk = 0; j < n_feat; j++)
blk += veclen[j];
n_elem = n_cb * n_density * blk;
if (has_means) {
band_nz_1d(wt_mean[0][0][0], n_elem, MIN_POS_FLOAT32);
if (bio_fwrite_1d((void *)wt_mean[0][0][0], sizeof(float32), n_elem, fp, &chksum) < 0)
return S3_ERROR;
}
if (has_vars) {
/* Don't floor full variances!!! */
if (bio_fwrite_1d((void *)wt_var[0][0][0][0], sizeof(float32),
n_elem * blk, fp, &chksum) < 0)
return S3_ERROR;
}
/* floor all non-zero entries to this value to make sure
that results are compatible between machines */
floor_nz_3d(dnom, n_cb, n_feat, n_density, MIN_POS_FLOAT32);
if (bio_fwrite_3d((void ***)dnom, sizeof(float32),
n_cb, n_feat, n_density, fp, &chksum) < 0) {
return S3_ERROR;
}
if (bio_fwrite(&chksum, sizeof(uint32), 1, fp, 0, &ignore) != 1) {
s3close(fp);
return S3_ERROR;
}
s3close(fp);
E_INFO("Wrote %s%s%s%s [%ux%ux%u vector/matrix arrays]\n",
fn,
(has_means ? " with means" : ""),
(has_vars ? " with full vars" : ""),
(has_vars && pass2var ? " (2pass)" : ""),
n_cb, n_feat, n_density);
return S3_SUCCESS;
}
int
s3gaucnt_read_full(const char *fn,
vector_t ****out_wt_mean,
vector_t *****out_wt_var,
int32 *out_pass2var,
float32 ****out_dnom,
uint32 *out_n_cb,
uint32 *out_n_feat,
uint32 *out_n_density,
uint32 **out_veclen)
{
uint32 rd_chksum = 0;
uint32 sv_chksum;
uint32 ignore;
char *ver;
char *do_chk;
FILE *fp;
uint32 swap;
uint32 has_means;
uint32 has_vars;
uint32 pass2var;
uint32 n_cb;
uint32 n_feat;
uint32 n_density;
uint32 *veclen;
float32 *buf;
float32 ***dnom;
uint32 n, i, b_i, j, k, l, d1, d2, d3;
vector_t ***wt_mean = NULL;
vector_t ****wt_var = NULL;
fp = s3open(fn, "rb", &swap);
if (fp == NULL)
return S3_ERROR;
/* check version id */
ver = s3get_gvn_fattr("version");
if (ver) {
if (strcmp(ver, GAUCNT_FILE_VERSION) != 0) {
E_FATAL("Version mismatch for %s, file ver: %s != reader ver: %s\n",
fn, ver, GAUCNT_FILE_VERSION);
}
}
else {
E_FATAL("No version attribute for %s\n", fn);
}
/* if do_chk is non-NULL, there is a checksum after the data in the file */
do_chk = s3get_gvn_fattr("chksum0");
if (bio_fread((void *)&has_means, sizeof(uint32), 1, fp, swap, &rd_chksum) != 1) {
return S3_ERROR;
}
if (bio_fread((void *)&has_vars, sizeof(uint32), 1, fp, swap, &rd_chksum) != 1) {
return S3_ERROR;
}
if (bio_fread((void *)&pass2var, sizeof(uint32), 1, fp, swap, &rd_chksum) != 1) {
return S3_ERROR;
}
if (bio_fread((void *)&n_cb, sizeof(uint32), 1, fp, swap, &rd_chksum) != 1) {
return S3_ERROR;
}
if (bio_fread((void *)&n_density, sizeof(uint32), 1, fp, swap, &rd_chksum) != 1) {
return S3_ERROR;
}
if (bio_fread_1d((void **)&veclen, sizeof(uint32), &n_feat, fp, swap, &rd_chksum) < 0) {
return S3_ERROR;
}
if (has_means) {
if (bio_fread_1d((void *)&buf, sizeof(float32), &n, fp, swap, &rd_chksum) < 0) {
return S3_ERROR;
}
wt_mean = (vector_t ***)ckd_calloc_3d(n_cb, n_feat, n_density, sizeof(vector_t));
for (i = 0, b_i = 0; i < n_cb; i++) {
for (j = 0; j < n_feat; j++) {
for (k = 0; k < n_density; k++) {
wt_mean[i][j][k] = &buf[b_i];
b_i += veclen[j];
}
}
}
}
if (has_vars) {
uint32 blk, maxveclen;
for (i = 0, blk = 0, maxveclen = 0; i < n_feat; i++) {
blk += veclen[i];
if (veclen[i] > maxveclen) maxveclen = veclen[i];
}
if (bio_fread_1d((void *)&buf, sizeof(float32), &n, fp, swap, &rd_chksum) < 0) {
return S3_ERROR;
}
assert(n == n_cb * n_density * blk * blk);
wt_var = (vector_t ****)ckd_calloc_4d(n_cb, n_feat, n_density,
maxveclen, sizeof(vector_t));
for (i = 0, b_i = 0; i < n_cb; i++) {
for (j = 0; j < n_feat; j++) {
for (k = 0; k < n_density; k++) {
for (l = 0; l < veclen[j]; l++) {
wt_var[i][j][k][l] = &buf[b_i];
b_i += veclen[j];
}
}
}
}
}
if (bio_fread_3d((void ****)&dnom, sizeof(float32), &d1, &d2, &d3, fp, swap, &rd_chksum) < 0) {
return S3_ERROR;
}
assert(d1 == n_cb);
assert(d2 == n_feat);
assert(d3 == n_density);
if (do_chk) {
/* See if the checksum in the file matches that which
was computed from the read data */
if (bio_fread(&sv_chksum, sizeof(uint32), 1, fp, swap, &ignore) != 1) {
s3close(fp);
return S3_ERROR;
}
if (sv_chksum != rd_chksum) {
E_FATAL("Checksum error; read corrupt data.\n");
}
}
s3close(fp);
*out_wt_mean = wt_mean;
*out_wt_var = wt_var;
*out_pass2var = pass2var;
*out_dnom = dnom;
*out_n_cb = n_cb;
*out_n_feat = n_feat;
*out_n_density = n_density;
*out_veclen = veclen;
E_INFO("Read %s%s%s%s [%ux%ux%u vector arrays]\n",
fn,
(has_means ? " with means" : ""),
(has_vars ? " with vars" : ""),
(has_vars && pass2var ? " (2pass)" : ""),
n_cb, n_feat, n_density);
return S3_SUCCESS;
}
int
s3lamb_read(const char *fn,
float32 **out_lambda,
uint32 *out_n_lambda)
{
FILE *fp;
const char *ver;
const char *do_chk;
uint32 rd_chksum = 0, sv_chksum, ignore;
float32 *lambda;
uint32 n_lambda;
uint32 swap;
fp = s3open(fn, "rb", &swap);
if (fp == NULL)
return S3_ERROR;
/* check version io */
ver = s3get_gvn_fattr("version");
if (ver) {
if (strcmp(ver, LAMBDA_FILE_VERSION) != 0) {
E_FATAL("Version mismatch for %s, file ver: %s != reader ver: %s\n",
fn, ver, LAMBDA_FILE_VERSION);
}
}
else {
E_FATAL("No version attribute for %s\n", fn);
}
/* if do_chk is non-NULL, there is a checksum after the data in the file */
do_chk = s3get_gvn_fattr("chksum0");
if (s3read_1d((void **)&lambda,
sizeof(float32),
&n_lambda,
fp,
swap,
&rd_chksum) != S3_SUCCESS)
return S3_ERROR;
if (do_chk) {
/* See if the checksum in the file matches that which
was computed from the read data */
if (s3read(&sv_chksum, sizeof(uint32), 1, fp, swap, &ignore) != 1) {
s3close(fp);
return S3_ERROR;
}
if (sv_chksum != rd_chksum) {
E_FATAL("Checksum error; read corrupt data.\n");
}
}
s3close(fp);
*out_lambda = lambda;
*out_n_lambda = n_lambda;
E_INFO("Read %s [%u array]\n", fn, n_lambda);
return S3_SUCCESS;
}
int
s3map_read(const char *fn,
void **out_map,
uint32 *out_n_dom,
uint32 *out_n_rng,
size_t map_elem_size)
{
uint32 rd_chksum = 0;
uint32 sv_chksum;
uint32 ignore;
char *ver;
char *do_chk;
FILE *fp;
uint32 swap;
fp = s3open(fn, "rb", &swap);
if (fp == NULL) {
return S3_ERROR;
}
/* check version id */
ver = s3get_gvn_fattr("version");
if (ver) {
if (strcmp(ver, MAP_FILE_VERSION) != 0) {
E_FATAL("Version mismatch for %s, file ver: %s != reader ver: %s\n",
fn, ver, MAP_FILE_VERSION);
}
}
else {
E_FATAL("No version attribute for %s\n", fn);
}
do_chk = s3get_gvn_fattr("chksum0");
if (bio_fread(out_n_rng,
sizeof(uint32),
1,
fp,
swap,
&rd_chksum) != 1) {
s3close(fp);
return S3_ERROR;
}
if (bio_fread_1d(out_map,
map_elem_size,
out_n_dom,
fp,
swap,
&rd_chksum) < 0) {
s3close(fp);
return S3_ERROR;
}
if (do_chk) {
if (bio_fread(&sv_chksum, sizeof(uint32), 1, fp, swap, &ignore) != 1) {
s3close(fp);
return S3_ERROR;
}
if (sv_chksum != rd_chksum) {
E_FATAL("Checksum error; read corrupted data.\n");
}
}
E_INFO("Read %s [%u mappings to %u]\n", fn, *out_n_dom, *out_n_rng);
return S3_SUCCESS;
}
int
s3mixw_read(const char *fn,
float32 ****out_mixw,
uint32 *out_n_mixw,
uint32 *out_n_feat,
uint32 *out_n_density)
{
uint32 rd_chksum = 0;
uint32 sv_chksum;
uint32 swap;
uint32 ignore;
char *ver;
char *do_chk;
FILE *fp;
fp = s3open(fn, "rb", &swap);
if (fp == NULL)
return S3_ERROR;
/* check version id */
ver = s3get_gvn_fattr("version");
if (ver) {
if (strcmp(ver, MIXW_FILE_VERSION) != 0) {
E_FATAL("Version mismatch for %s, file ver: %s != reader ver: %s\n",
fn, ver, MIXW_FILE_VERSION);
}
}
else {
E_FATAL("No version attribute for %s\n", fn);
}
/* if do_chk is non-NULL, there is a checksum after the data in the file */
do_chk = s3get_gvn_fattr("chksum0");
/* Read the mixing weight array */
if (bio_fread_3d((void ****)out_mixw,
sizeof(float32),
out_n_mixw,
out_n_feat,
out_n_density,
fp,
swap,
&rd_chksum) < 0) {
s3close(fp);
return S3_ERROR;
}
if (do_chk) {
/* See if the checksum in the file matches that which
was computed from the read data */
if (bio_fread(&sv_chksum, sizeof(uint32), 1, fp, swap, &ignore) != 1) {
s3close(fp);
return S3_ERROR;
}
if (sv_chksum != rd_chksum) {
E_FATAL("Checksum error; read corrupt data.\n");
}
}
s3close(fp);
E_INFO("Read %s [%ux%ux%u array]\n",
fn, *out_n_mixw, *out_n_feat, *out_n_density);
return S3_SUCCESS;
}
