/**
* @brief Write all mixture density data.
*
* The pointers of all mixture densities are first gathered,
* sorted by the address. Then the densities are written
* by the sorted order. The pointers to the lower structure (variance etc.)
* in the data are written in a corresponding scholar id.
* The pointer index of this data will be used later to convert any pointer
* reference to a density data into scholar id.
*
* @param fp [in] file pointer
* @param hmm [in] writing %HMM definition data
*/
static boolean
wt_dens(FILE *fp, HTK_HMM_INFO *hmm)
{
HTK_HMM_Dens *d;
unsigned int idx;
unsigned int vid;
dens_num = hmm->totalmixnum;
dens_index = (HTK_HMM_Dens **)mymalloc(sizeof(HTK_HMM_Dens *) * dens_num);
idx = 0;
for(d = hmm->dnstart; d; d = d->next) dens_index[idx++] = d;
qsort(dens_index, dens_num, sizeof(HTK_HMM_Dens *), (int (*)(const void *, const void *))qsort_dens_index);
wrt(fp, &dens_num, sizeof(unsigned int), 1);
for (idx = 0; idx < dens_num; idx++) {
d = dens_index[idx];
wrt_str(fp, d->name);
wrt(fp, &(d->meanlen), sizeof(short), 1);
wrt(fp, d->mean, sizeof(VECT), d->meanlen);
vid = search_vid(d->var);
/* for debug */
if (d->var != vr_index[vid]) {
jlog("Error: write_binhmm: index not match!!!\n");
return FALSE;
}
wrt(fp, &vid, sizeof(unsigned int), 1);
wrt(fp, &(d->gconst), sizeof(LOGPROB), 1);
}
jlog("Stat: write_binhmm: %d gaussian densities written\n", dens_num);
return TRUE;
}
/**
* @brief Write all mixture pdf data.
*
* The pointers of all mixture pdfs are first gathered,
* sorted by the address. Then the mixture pdfs are written
* by the sorted order. The pointers to the lower structure (variance etc.)
* in the data are written in a corresponding scholar id.
* The pointer index of this data will be used later to convert any pointer
* reference to a data into scholar id.
*
* @param fp [in] file pointer
* @param hmm [in] writing %HMM definition data
*/
static boolean
wt_mpdf(FILE *fp, HTK_HMM_INFO *hmm)
{
HTK_HMM_PDF *m;
unsigned int idx;
mpdf_num = 0;
for(m=hmm->pdfstart;m;m=m->next) mpdf_num++;
mpdf_index = (HTK_HMM_PDF **)mymalloc(sizeof(HTK_HMM_PDF *) * mpdf_num);
idx = 0;
for(m=hmm->pdfstart;m;m=m->next) mpdf_index[idx++] = m;
qsort(mpdf_index, mpdf_num, sizeof(HTK_HMM_PDF *), (int (*)(const void *, const void *))qsort_mpdf_index);
wrt(fp, &mpdf_num, sizeof(unsigned int), 1);
for (idx = 0; idx < mpdf_num; idx++) {
m = mpdf_index[idx];
wrt_str(fp, m->name);
wrt(fp, &(m->stream_id), sizeof(short), 1);
if (wt_pdf_sub(fp, hmm, m) == FALSE) return FALSE;
}
jlog("Stat: write_binhmm: %d mixture PDF written\n", mpdf_num);
return TRUE;
}
/**
* Write %HMM type of mixture tying.
*
* @param fp [in] file pointer
* @param hmm [out] pointer to the writing %HMM definition data
*/
static boolean
wt_type(FILE *fp, HTK_HMM_INFO *hmm)
{
wrt(fp, &(hmm->is_tied_mixture), sizeof(boolean), 1);
wrt(fp, &(hmm->maxmixturenum), sizeof(int), 1);
return TRUE;
}
/**
* @brief Write all transition matrix data.
*
* The pointers of all transition matrixes are first gathered,
* sorted by the address. Then the transition matrix data are written
* by the sorted order. The index will be used later to convert any pointer
* reference to a transition matrix into scholar id.
*
* @param fp [in] file pointer
* @param hmm [in] writing %HMM definition data
*/
static boolean
wt_trans(FILE *fp, HTK_HMM_INFO *hmm)
{
HTK_HMM_Trans *t;
unsigned int idx;
int i;
tr_num = 0;
for(t = hmm->trstart; t; t = t->next) tr_num++;
tr_index = (HTK_HMM_Trans **)mymalloc(sizeof(HTK_HMM_Trans *) * tr_num);
idx = 0;
for(t = hmm->trstart; t; t = t->next) tr_index[idx++] = t;
qsort(tr_index, tr_num, sizeof(HTK_HMM_Trans *), (int (*)(const void *, const void *))qsort_tr_index);
wrt(fp, &tr_num, sizeof(unsigned int), 1);
for (idx = 0; idx < tr_num; idx++) {
t = tr_index[idx];
wrt_str(fp, t->name);
wrt(fp, &(t->statenum), sizeof(short), 1);
for(i=0;i<t->statenum;i++) {
wrt(fp, t->a[i], sizeof(PROB), t->statenum);
}
}
jlog("Stat: write_binhmm: %d transition maxtix written\n", tr_num);
return TRUE;
}
/**
* @brief Write all state data.
*
* The pointers of all states are first gathered,
* sorted by the address. Then the state informations are written
* by the sorted order. The pointers to the lower structure (mixture etc.)
* in the data are written in a corresponding scholar id.
* The pointer index of this data will be used later to convert any pointer
* reference to a state data into scholar id.
*
* @param fp [in] file pointer
* @param hmm [in] writing %HMM definition data
* @param mpdf_macro [in] TRUE if mixture PDFs are already read as separated definitions
*/
static boolean
wt_state(FILE *fp, HTK_HMM_INFO *hmm, boolean mpdf_macro)
{
HTK_HMM_State *s;
unsigned int idx;
unsigned int mid;
unsigned int swid;
int m;
st_num = hmm->totalstatenum;
st_index = (HTK_HMM_State **)mymalloc(sizeof(HTK_HMM_State *) * st_num);
idx = 0;
for(s = hmm->ststart; s; s = s->next) st_index[idx++] = s;
qsort(st_index, st_num, sizeof(HTK_HMM_State *), (int (*)(const void *, const void *))qsort_st_index);
wrt(fp, &st_num, sizeof(unsigned int), 1);
for (idx = 0; idx < st_num; idx++) {
s = st_index[idx];
wrt_str(fp, s->name);
if (mpdf_macro) {
/* mpdf are already written, so write index */
for(m=0;m<s->nstream;m++) {
if (s->pdf[m] == NULL) {
mid = mpdf_num;
} else {
mid = search_mpdfid(s->pdf[m]);
if (s->pdf[m] != mpdf_index[mid]) {
jlog("Error: write_binhmm: index not match!!!\n");
return FALSE;
}
}
wrt(fp, &mid, sizeof(unsigned int), 1);
}
} else {
/* mpdf should be written here */
for(m=0;m<s->nstream;m++) {
/* stream_id will not be written */
if (wt_pdf_sub(fp, hmm, s->pdf[m]) == FALSE) return FALSE;
}
}
if (hmm->opt.stream_info.num > 1) {
/* write steam weight */
if (s->w == NULL) {
swid = streamweight_num;
} else {
swid = search_swid(s->w);
if (s->w != streamweight_index[swid]) {
jlog("Error: write_binhmm: index not match!!!\n");
return FALSE;
}
}
wrt(fp, &swid, sizeof(unsigned int), 1);
}
}
jlog("Stat: write_binhmm: %d states written\n", st_num);
return TRUE;
}
int main(int argc, char *argv[])
{
int sz, i = 0;
scanf("%d", &sz);
int key[sz];
while((scanf("%d", &key[i])) == 1 && i++ < sz);
printf("before\n");
wrt(key, sz);
mergesort(key, sz);
printf("after\n");
wrt(key, sz);
return 0;
}
/**
* @brief Write all model data.
*
* The data of all models are written. The order is not important
* at this top level, since there are no reference to this data.
* The pointers to the lower structure (states, transitions, etc.)
* in the data are written by the corresponding scholar id.
*
* @param fp [in] file pointer
* @param hmm [in] writing %HMM definition data
*/
static boolean
wt_data(FILE *fp, HTK_HMM_INFO *hmm)
{
HTK_HMM_Data *d;
unsigned int md_num;
unsigned int sid, tid;
int i;
md_num = hmm->totalhmmnum;
wrt(fp, &(md_num), sizeof(unsigned int), 1);
for(d = hmm->start; d; d = d->next) {
wrt_str(fp, d->name);
wrt(fp, &(d->state_num), sizeof(short), 1);
for (i=0;i<d->state_num;i++) {
if (d->s[i] != NULL) {
sid = search_stid(d->s[i]);
/* for debug */
if (d->s[i] != st_index[sid]) {
jlog("Error: write_binhmm: index not match!!!\n");
return FALSE;
}
} else {
sid = hmm->totalstatenum + 1; /* error value */
}
wrt(fp, &sid, sizeof(unsigned int), 1);
}
tid = search_trid(d->tr);
/* for debug */
if (d->tr != tr_index[tid]) {
jlog("Error: write_binhmm: index not match!!!\n");
return FALSE;
}
wrt(fp, &tid, sizeof(unsigned int), 1);
}
jlog("Stat: write_binhmm: %d HMM model definition written\n", md_num);
return TRUE;
}
/**
* @brief Write all codebook data.
*
* The pointers of all codebook densities are first gathered,
* sorted by the address. Then the densities are written
* by the sorted order. The pointers to the lower structure (mixture etc.)
* in the data are written by the corresponding scholar id.
* The pointer index of this data will be used later to convert any pointer
* reference to a codebook into scholar id.
*
* @param fp [in] file pointer
* @param hmm [in] writing %HMM definition data
*/
static boolean
wt_tmix(FILE *fp, HTK_HMM_INFO *hmm)
{
GCODEBOOK *tm;
unsigned int idx;
unsigned int did;
int i;
tm_num = hmm->codebooknum;
tm_index = (GCODEBOOK **)mymalloc(sizeof(GCODEBOOK *) * tm_num);
tm_idx = 0;
aptree_traverse_and_do(hmm->codebook_root, tmix_list_callback);
qsort(tm_index, tm_num, sizeof(GCODEBOOK *), (int (*)(const void *, const void *))qsort_tm_index);
wrt(fp, &tm_num, sizeof(unsigned int), 1);
for (idx = 0; idx < tm_num; idx++) {
tm = tm_index[idx];
wrt_str(fp, tm->name);
wrt(fp, &(tm->num), sizeof(int), 1);
for(i=0;i<tm->num;i++) {
if (tm->d[i] == NULL) {
did = dens_num;
} else {
did = search_did(tm->d[i]);
/* for debug */
if (tm->d[i] != dens_index[did]) {
jlog("Error: write_binhmm: index not match!!!\n");
return FALSE;
}
}
wrt(fp, &did, sizeof(unsigned int), 1);
}
}
jlog("Stat: write_binhmm: %d tied-mixture codebooks written\n", tm_num);
return TRUE;
}
/**
* @brief Write all variance data.
*
* The pointers of all variance vectors are first gathered,
* sorted by the address. Then the variance vectors are written
* by the sorted order. The index will be used later to convert any pointer
* reference to a variance vector into scholar id.
*
* @param fp [in] file pointer
* @param hmm [in] writing %HMM definition data
*/
static boolean
wt_var(FILE *fp, HTK_HMM_INFO *hmm)
{
HTK_HMM_Var *v;
unsigned int idx;
vr_num = 0;
for(v = hmm->vrstart; v; v = v->next) vr_num++;
vr_index = (HTK_HMM_Var **)mymalloc(sizeof(HTK_HMM_Var *) * vr_num);
idx = 0;
for(v = hmm->vrstart; v; v = v->next) vr_index[idx++] = v;
qsort(vr_index, vr_num, sizeof(HTK_HMM_Var *), (int (*)(const void *, const void *))qsort_vr_index);
wrt(fp, &vr_num, sizeof(unsigned int), 1);
for (idx = 0; idx < vr_num; idx++) {
v = vr_index[idx];
wrt_str(fp, v->name);
wrt(fp, &(v->len), sizeof(short), 1);
wrt(fp, v->vec, sizeof(VECT), v->len);
}
jlog("Stat: write_binhmm: %d variance written\n", vr_num);
return TRUE;
}
/**
* @brief Write all stream weight data.
*
* The pointers of all stream weights are first gathered,
* sorted by the address. Then the stream weights are written
* by the sorted order. The pointers to the lower structure (variance etc.)
* in the data are written in a corresponding scholar id.
* The pointer index of this data will be used later to convert any pointer
* reference to a data into scholar id.
*
* @param fp [in] file pointer
* @param hmm [in] writing %HMM definition data
*/
static boolean
wt_streamweight(FILE *fp, HTK_HMM_INFO *hmm)
{
HTK_HMM_StreamWeight *sw;
unsigned int idx;
streamweight_num = 0;
for(sw=hmm->swstart;sw;sw=sw->next) streamweight_num++;
streamweight_index = (HTK_HMM_StreamWeight **)mymalloc(sizeof(HTK_HMM_StreamWeight *) * streamweight_num);
idx = 0;
for(sw = hmm->swstart; sw; sw = sw->next) streamweight_index[idx++] = sw;
qsort(streamweight_index, streamweight_num, sizeof(HTK_HMM_StreamWeight *), (int (*)(const void *, const void *))qsort_streamweight_index);
wrt(fp, &streamweight_num, sizeof(unsigned int), 1);
for (idx = 0; idx < streamweight_num; idx++) {
sw = streamweight_index[idx];
wrt_str(fp, sw->name);
wrt(fp, &(sw->len), sizeof(short), 1);
wrt(fp, sw->weight, sizeof(VECT), sw->len);
}
jlog("Stat: write_binhmm: %d stream weights written\n", streamweight_num);
return TRUE;
}
/**
* Write a mixture PDF.
*
* @param fp [in] file pointer
* @param hmm [in] writing %HMM definition data
* @param m [out] mixture PDF to be written
*
* @return TRUE on success, FALSE on error.
*
*/
static boolean
wt_pdf_sub(FILE *fp, HTK_HMM_INFO *hmm, HTK_HMM_PDF *m)
{
unsigned int did;
int i;
short dummy;
if (hmm->is_tied_mixture) {
/* try tmix */
did = search_tmid((GCODEBOOK *)(m->b));
if ((GCODEBOOK *)m->b == tm_index[did]) {
/* tmix */
dummy = -1;
wrt(fp, &dummy, sizeof(short), 1);
wrt(fp, &did, sizeof(unsigned int), 1);
} else {
/* tmix failed -> normal mixture */
wrt(fp, &(m->mix_num), sizeof(short), 1);
for (i=0;i<m->mix_num;i++) {
if (m->b[i] == NULL) {
did = dens_num;
} else {
did = search_did(m->b[i]);
if (m->b[i] != dens_index[did]) {
jlog("Error: write_binhmm: index not match!!!\n");
return FALSE;
}
}
wrt(fp, &did, sizeof(unsigned int), 1);
}
}
} else { /* not tied mixture */
wrt(fp, &(m->mix_num), sizeof(short), 1);
for (i=0;i<m->mix_num;i++) {
if (m->b[i] == NULL) {
did = dens_num;
} else {
did = search_did(m->b[i]);
if (m->b[i] != dens_index[did]) {
jlog("Error: write_binhmm: index not match!!!\n");
return FALSE;
}
}
wrt(fp, &did, sizeof(unsigned int), 1);
}
}
wrt(fp, m->bweight, sizeof(PROB), m->mix_num);
return TRUE;
}
/**
* Write %HMM option specifications
*
* @param fp [in] file pointer
* @param opt [out] pointer to the %HMM option structure that holds the values.
*/
static boolean
wt_opt(FILE *fp, HTK_HMM_Options *opt)
{
wrt(fp, &(opt->stream_info.num), sizeof(short), 1);
wrt(fp, opt->stream_info.vsize, sizeof(short), MAXSTREAMNUM);
wrt(fp, &(opt->vec_size), sizeof(short), 1);
wrt(fp, &(opt->cov_type), sizeof(short), 1);
wrt(fp, &(opt->dur_type), sizeof(short), 1);
wrt(fp, &(opt->param_type), sizeof(short), 1);
return TRUE;
}
/**
* Write header information, with identifier string.
*
* @param fp [in] file pointer
* @param str [in] user header string (any string within BINGRAM_HDSIZE
* bytes is allowed)
* @param version [in] file format version id
*/
static boolean
write_header(FILE *fp, char *str)
{
char buf[BINGRAM_HDSIZE];
int i, totallen;
for(i=0;i<BINGRAM_HDSIZE;i++) buf[i] = EOF;
totallen = strlen(BINGRAM_IDSTR_V5) + 1 + strlen(BINGRAM_SIZESTR_HEAD) + strlen(BINGRAM_SIZESTR_BODY) + 1 + strlen(BINGRAM_BYTEORDER_HEAD) + strlen(BINGRAM_NATURAL_BYTEORDER) + 1 + strlen(str);
if (totallen >= BINGRAM_HDSIZE) {
jlog("Warning: write_bingram: header too long, last will be truncated\n");
i = strlen(str) - (totallen - BINGRAM_HDSIZE);
str[i] = '\0';
}
sprintf(buf, "%s\n%s%s %s%s\n%s", BINGRAM_IDSTR_V5, BINGRAM_SIZESTR_HEAD, BINGRAM_SIZESTR_BODY, BINGRAM_BYTEORDER_HEAD, BINGRAM_NATURAL_BYTEORDER, str);
wrt(fp, buf, 1, BINGRAM_HDSIZE);
return TRUE;
}
int CDBAPI_CacheTest::Run(void)
{
cout << "Run CDBAPI_CacheTest test" << endl << endl;
try {
auto_ptr<ICache> blob_cache( MakeICache() );
if ( blob_cache.get() == NULL ) {
return 1;
}
int top = blob_cache->GetTimeStampPolicy();
blob_cache->SetTimeStampPolicy(top, 30);
blob_cache->Purge("", "", 30);
const char* szTest = "01234567890";
blob_cache->Store("key_1",
1,
"",
szTest,
strlen(szTest));
size_t blob_size = blob_cache->GetSize("key_1",
1,
"");
int s = strlen(szTest);
assert(blob_size == size_t(s));
char blob_buf[1024] = {0,};
blob_cache->Read("key_1",
1,
"",
blob_buf,
1024);
int cmp = strcmp(blob_buf, szTest);
assert(cmp == 0);
{
auto_ptr<IWriter> wrt(blob_cache->GetWriteStream("key_3", 1, "sk1"));
size_t bytes_written;
int s = strlen(szTest);
wrt->Write(szTest, s, &bytes_written);
wrt->Flush();
}
{
char str[1024] = {0,};
char* sp = str;
auto_ptr<IReader> rdr(blob_cache->GetReadStream("key_3", 1, "sk1"));
char buf[1024];
ERW_Result r;
size_t rd;
do {
r = rdr->Read(buf, 1, &rd);
if (r != eRW_Success)
break;
*sp++ = buf[0];
} while (rd > 0);
int c = strcmp(szTest, str);
assert(c == 0);
}
// dump huge BLOB
{
unsigned bsize = 10 * 1024 * 1024;
int* big_blob = new int[bsize];
for (unsigned i = 0; i < bsize; ++i) {
big_blob[i] = i;
}
{
auto_ptr<IWriter> wrt(blob_cache->GetWriteStream("key_big", 1, ""));
unsigned written = 0;
int* p = big_blob;
while (written < bsize) {
size_t bytes_written;
unsigned elements_to_write = 10;
unsigned chunk_size = elements_to_write * sizeof(int);
wrt->Write(p, chunk_size, &bytes_written);
p += elements_to_write;
written += elements_to_write;
assert(bytes_written == chunk_size);
} // while
wrt->Flush();
}
auto_ptr<IReader> rdr(blob_cache->GetReadStream("key_big", 1, ""));
ERW_Result r;
size_t rd;
unsigned int cnt = 0;
do {
int buf[1];
r = rdr->Read(buf, sizeof(int), &rd);
if (r != eRW_Success)
break;
if (cnt != (unsigned int)buf[0]) {
cerr << "BLOB comparison error element idx = " << cnt <<
" value=" << buf[0] << endl;
assert(cnt == (unsigned int)buf[0]);
}
cnt++;
} while (rd > 0);
}
blob_cache->Remove("key_big", 1, "");
} catch( CDB_Exception& dbe ) {
NcbiCerr << dbe.what() << " " << dbe.Message()
<< NcbiEndl;
return 1;
}
cout << endl;
cout << "TEST execution completed successfully!" << endl << endl;
return 0;
}
int CTestTransmission::Run(void)
{
LOG_POST("Test IReader/IWriter transmission");
const int test_buf_size = 1024;
unsigned char buf[test_buf_size * 10];
unsigned char buf2[test_buf_size * 10];
// ---------------------------------------
// write the test sequence
ERW_Result res;
size_t sz;
{{
CTestMemWriter wrt(buf, sizeof(buf));
s_SaveTestData(&wrt, test_buf_size/2);
sz = wrt.SizeWritten();
assert(sz == test_buf_size/2);
}}
// ---------------------------------------
// read the whole test sequence
{{
#if defined(NCBI_XCODE_BUILD) && defined(__OPTIMIZE__) && NCBI_COMPILER_VERSION == 421
// Our Xcode builds use gcc-4.2 -Os, which passes in a
// CTestMemWriter(!) object without this workaround. :-/
unique_ptr<CTestMemReader> rdr(new CTestMemReader(buf, sz));
s_ReadCheck1(rdr.get(), buf, buf2, sz * 2, sz);
#else
CTestMemReader rdr(buf, sz);
s_ReadCheck1(&rdr, buf, buf2, sz * 2, sz);
#endif
}}
// ---------------------------------------
// read the test sequence char by char
{{
CTestMemReader rdr(buf, sz);
for (size_t i = 0; i < sz * 2; ++i) {
buf2[i] = 0;
size_t read;
res = rdr.Read(buf2 + i, 1, &read);
if (res == eRW_Eof) {
assert(i == sz);
assert(read == 0);
break;
}
assert(read == 1);
assert(res == eRW_Success);
assert (buf2[i] == buf[i]);
}
}}
// ---------------------------------------
// read the test sequence by 7 char
{{
CTestMemReader rdr(buf, sz);
size_t read = 0;
for (size_t i = 0; i < sz * 2; i+=read) {
unsigned char buf3[10] = {0,};
res = rdr.Read(buf3, 7, &read);
if (res == eRW_Eof) {
assert(read == 0);
break;
}
assert(read <= 7);
assert(res == eRW_Success);
for (size_t j = 0; j < read; ++j) {
assert(buf3[j] == (unsigned char)(i + j));
}
}
}}
// Transmission test
memset(buf, 0, sizeof(buf));
memset(buf2, 0, sizeof(buf2));
{{
CTestMemWriter wrt(buf, sizeof(buf));
CTransmissionWriter twrt(&wrt);
char b = 35;
twrt.Write(&b, 1);
CTestMemReader rdr(buf, sz);
CTransmissionReader trdr(&rdr);
char c = 0;
trdr.Read(&c, 1);
assert(b==c);
}}
size_t tsize = 3;
size_t byte_size;
{{
CTestMemWriter wrt(buf, sizeof(buf));
CTransmissionWriter twrt(&wrt);
s_SaveTestData(&twrt, tsize);//test_buf_size/2);
byte_size = wrt.SizeWritten();
}}
// ---------------------------------------
// read the whole test sequence
{{
CTestMemReader rdr(buf, byte_size);
CTransmissionReader trdr(&rdr);
s_ReadCheck1(&trdr, buf, buf2, sz * 2, tsize);//test_buf_size/2);
}}
memset(buf, 0, sizeof(buf));
memset(buf2, 0, sizeof(buf2));
tsize = test_buf_size/2;
{{
CTestMemWriter wrt(buf, sizeof(buf));
CTransmissionWriter twrt(&wrt);
s_SaveTestData(&twrt, tsize);
byte_size = wrt.SizeWritten();
}}
// ---------------------------------------
// read the whole test sequence
{{
CTestMemReader rdr(buf, byte_size);
CTransmissionReader trdr(&rdr);
s_ReadCheck1(&trdr, buf, buf2, tsize * 2, tsize);
}}
// ---------------------------------------
// read the test sequence char by char
{{
CTestMemReader rdr(buf, byte_size);
CTransmissionReader trdr(&rdr);
size_t i;
for (i = 0; i < tsize * 2; ++i) {
buf2[i] = 0;
size_t read;
res = trdr.Read(buf2 + i, 1, &read);
if (res == eRW_Eof) {
assert(i == sz);
assert(read == 0);
break;
}
assert(read == 1);
assert(res == eRW_Success);
unsigned char ch = (unsigned char) i;
assert (buf2[i] == ch);
}
assert(i == tsize);
}}
// ---------------------------------------
// read the test sequence by 7 char
{{
CTestMemReader rdr(buf, byte_size);
CTransmissionReader trdr(&rdr);
size_t read = 0;
size_t total_read = 0;
for (size_t i = 0; i < tsize * 2; i+=read) {
unsigned char buf3[10] = {0,};
res = trdr.Read(buf3, 7, &read);
if (res == eRW_Eof) {
assert(read == 0);
break;
}
total_read += read;
assert(read <= 7);
assert(res == eRW_Success);
for (size_t j = 0; j < read; ++j) {
assert(buf3[j] == (unsigned char)(i + j));
}
}
assert(total_read = tsize);
}}
// ---------------------------
// test eof packet
{{
CTestMemWriter wrt(buf, sizeof(buf));
char b = 35;
{
CTransmissionWriter twrt(&wrt, eNoOwnership, CTransmissionWriter::eSendEofPacket);
twrt.Write(&b, 1);
}
CTestMemReader rdr(buf, sz);
CTransmissionReader trdr(&rdr);
char c = 0;
trdr.Read(&c, 1);
assert(b==c);
res = trdr.Read(&c,1);
assert(res==eRW_Eof);
}}
LOG_POST("OK");
return 0;
}
/**
* Write acoustic analysis configration parameters into header of binary HMM.
*
* @param fp [in] file pointer
* @param para [in] acoustic analysis configration parameters
*/
static boolean
wt_para(FILE *fp, Value *para)
{
short version;
version = VALUE_VERSION;
wrt(fp, &version, sizeof(short), 1);
wrt(fp, &(para->smp_period), sizeof(long), 1);
wrt(fp, &(para->smp_freq), sizeof(long), 1);
wrt(fp, &(para->framesize), sizeof(int), 1);
wrt(fp, &(para->frameshift), sizeof(int), 1);
wrt(fp, &(para->preEmph), sizeof(float), 1);
wrt(fp, &(para->lifter), sizeof(int), 1);
wrt(fp, &(para->fbank_num), sizeof(int), 1);
wrt(fp, &(para->delWin), sizeof(int), 1);
wrt(fp, &(para->accWin), sizeof(int), 1);
wrt(fp, &(para->silFloor), sizeof(float), 1);
wrt(fp, &(para->escale), sizeof(float), 1);
wrt(fp, &(para->hipass), sizeof(int), 1);
wrt(fp, &(para->lopass), sizeof(int), 1);
wrt(fp, &(para->enormal), sizeof(int), 1);
wrt(fp, &(para->raw_e), sizeof(int), 1);
wrt(fp, &(para->zmeanframe), sizeof(int), 1);
wrt(fp, &(para->usepower), sizeof(int), 1);
return TRUE;
}
std::vector<uint8_t> BsonSerializer::writeDocument(Bundle& value) const
{
std::vector<uint8_t> res;
BufferWriter wrt(res);
// Document length
wrt.writeInt32(0);
for (auto& elem : value)
{
auto info = elem.second.type_info();
if (info.isArithmetic())
{
if (info.isFloatingPoint())
{
wrt.writeUInt8(0x01); // double
wrt.writeCString(elem.first);
wrt.writeDouble(elem.second.as<double>());
}
else {
wrt.writeUInt8(0x12); // int64
wrt.writeCString(elem.first);
wrt.writeInt64(elem.second.as<int64_t>());
}
}
else if (elem.second.isSerializable())
{
wrt.writeUInt8(0x03);
wrt.writeCString(elem.first);
Bundle v = elem.second.serialize().as<Bundle>();
auto val = this->writeDocument(v);
wrt.writeBytes(val.data(), val.size());
}
else if (elem.second.isType<std::vector<AnyValue>>())
{
wrt.writeUInt8(0x04);
wrt.writeCString(elem.first);
std::vector<AnyValue> v = elem.second.as<std::vector<AnyValue>>();
Bundle b;
for (size_t i = 0; i < v.size(); i++) b.set(std::to_string(i), v[i]);
auto val = this->writeDocument(b);
wrt.writeBytes(val.data(), val.size());
}
else if (elem.second.isType<std::vector<uint8_t>>())
{
wrt.writeUInt8(0x05);
wrt.writeCString(elem.first);
std::vector<uint8_t> v = elem.second.as<std::vector<uint8_t>>();
if (v.size() > INT32_MAX) throw std::runtime_error("Data too large");
wrt.writeInt32((int32_t)v.size()); // Binary size
wrt.writeUInt8(0x00); // Subtype
wrt.writeBytes(v.data(), v.size()); // Data
}
else if (elem.second.isType<bool>())
{
wrt.writeUInt8(0x08);
wrt.writeCString(elem.first);
bool v = elem.second.as<bool>();
wrt.writeUInt8(v ? 0x01 : 0x00);
}
else if (elem.second.isType<std::nullptr_t>())
{
wrt.writeUInt8(0x0A);
wrt.writeCString(elem.first);
}
else if (elem.second.isConvertibleTo<std::string>())
{
wrt.writeUInt8(0x02);
wrt.writeCString(elem.first);
std::string v = elem.second.as<std::string>();
if (v.size() > INT32_MAX - 1) throw std::runtime_error("String too large");
wrt.writeInt32((int32_t)v.size() + 1);
wrt.writeCString(v);
}
}
wrt.writeUInt8(0x00);
wrt.setPosition(0);
wrt.setOverwrite(true);
// Update Size value
if (wrt.getSize() > INT32_MAX) throw std::runtime_error("Document to large");
wrt.writeInt32((int32_t)wrt.getSize());
return res;
}
/**
* Write a whole N-gram data in binary format.
*
* @param fp [in] file pointer
* @param ndata [in] N-gram data to write
* @param headerstr [in] user header string
*
* @return TRUE on success, FALSE on failure
*/
boolean
ngram_write_bin(FILE *fp, NGRAM_INFO *ndata, char *headerstr)
{
int i,n;
unsigned int len;
int wlen;
NGRAM_TUPLE_INFO *t;
reset_wrt_counter();
/* write initial header */
if (write_header(fp, headerstr) == FALSE) return FALSE;
/* swap not needed any more */
need_swap = FALSE;
/* write some header info */
wrt(fp, &(ndata->n), sizeof(int), 1);
wrt(fp, &(ndata->dir), sizeof(int), 1);
wrt(fp, &(ndata->bigram_index_reversed), sizeof(boolean), 1);
/* write total info */
for(n=0;n<ndata->n;n++) {
wrt(fp, &(ndata->d[n].totalnum), sizeof(NNID), 1);
/*jlog("ngram %d=%d\n",n+1,ndata->ngram_num[n]);*/
}
/* unk_*, isopen, max_word_num are set after read, so need not save */
/* write wname */
wlen = 0;
for(i=0;i<ndata->max_word_num;i++) {
wlen += strlen(ndata->wname[i]) + 1;
}
wrt(fp, &wlen, sizeof(int), 1);
for(i=0;i<ndata->max_word_num;i++) {
wrt(fp, ndata->wname[i], 1, strlen(ndata->wname[i]) + 1); /* include \0 */
}
/* write N-gram */
for(n=0;n<ndata->n;n++) {
t = &(ndata->d[n]);
wrt(fp, &(t->is24bit), sizeof(boolean), 1);
wrt(fp, &(t->ct_compaction), sizeof(boolean), 1);
wrt(fp, &(t->bgnlistlen), sizeof(NNID), 1);
wrt(fp, &(t->context_num), sizeof(NNID), 1);
if (n > 0) {
if (t->is24bit) {
wrt(fp, t->bgn_upper, sizeof(NNID_UPPER), t->bgnlistlen);
wrt(fp, t->bgn_lower, sizeof(NNID_LOWER), t->bgnlistlen);
} else {
wrt(fp, t->bgn, sizeof(NNID), t->bgnlistlen);
}
wrt(fp, t->num, sizeof(WORD_ID), t->bgnlistlen);
wrt(fp, t->nnid2wid, sizeof(WORD_ID), t->totalnum);
}
wrt(fp, t->prob, sizeof(LOGPROB), t->totalnum);
if (t->bo_wt) {
i = 1;
wrt(fp, &i, sizeof(int), 1);
wrt(fp, t->bo_wt, sizeof(LOGPROB), t->context_num);
} else {
i = 0;
wrt(fp, &i, sizeof(int), 1);
}
if (t->nnid2ctid_upper) {
i = 1;
wrt(fp, &i, sizeof(int), 1);
wrt(fp, t->nnid2ctid_upper, sizeof(NNID_UPPER), t->totalnum);
wrt(fp, t->nnid2ctid_lower, sizeof(NNID_LOWER), t->totalnum);
} else {
i = 0;
wrt(fp, &i, sizeof(int), 1);
}
}
/* write additional LR 2-gram */
if (ndata->bo_wt_1) {
i = 1;
wrt(fp, &i, sizeof(int), 1);
wrt(fp, ndata->bo_wt_1, sizeof(LOGPROB), ndata->d[0].context_num);
} else {
i = 0;
wrt(fp, &i, sizeof(int), 1);
}
if (ndata->p_2) {
i = 1;
wrt(fp, &i, sizeof(int), 1);
wrt(fp, ndata->p_2, sizeof(LOGPROB), ndata->d[1].totalnum);
} else {
i = 0;
wrt(fp, &i, sizeof(int), 1);
}
len = get_wrt_counter();
jlog("Stat: ngram_write_bin: wrote %lu bytes (%.1f MB)\n", len, len / 1048576.0);
return TRUE;
}
shared_ptr<WRTravmap> Mapper2d::
CreateWRTravmap()
{
shared_ptr<WRTravmap> wrt(new WRTravmap(m_travmap));
return wrt;
}