int
ps_add_word(ps_decoder_t *ps,
char const *word,
char const *phones,
int update)
{
int32 wid, lmwid;
ngram_model_t *lmset;
s3cipid_t *pron;
char **phonestr, *tmp;
int np, i, rv;
/* Parse phones into an array of phone IDs. */
tmp = ckd_salloc(phones);
np = str2words(tmp, NULL, 0);
phonestr = ckd_calloc(np, sizeof(*phonestr));
str2words(tmp, phonestr, np);
pron = ckd_calloc(np, sizeof(*pron));
for (i = 0; i < np; ++i) {
pron[i] = bin_mdef_ciphone_id(ps->acmod->mdef, phonestr[i]);
if (pron[i] == -1) {
E_ERROR("Unknown phone %s in phone string %s\n",
phonestr[i], tmp);
ckd_free(phonestr);
ckd_free(tmp);
ckd_free(pron);
return -1;
}
}
/* No longer needed. */
ckd_free(phonestr);
ckd_free(tmp);
/* Add it to the dictionary. */
if ((wid = dict_add_word(ps->dict, word, pron, np)) == -1) {
ckd_free(pron);
return -1;
}
/* No longer needed. */
ckd_free(pron);
/* Now we also have to add it to dict2pid. */
dict2pid_add_word(ps->d2p, wid);
if ((lmset = ps_get_lmset(ps)) != NULL) {
/* Add it to the LM set (meaning, the current LM). In a perfect
* world, this would result in the same WID, but because of the
* weird way that word IDs are handled, it doesn't. */
if ((lmwid = ngram_model_add_word(lmset, word, 1.0))
== NGRAM_INVALID_WID)
return -1;
}
/* Rebuild the widmap and search tree if requested. */
if (update) {
if ((rv = ps_search_reinit(ps->search, ps->dict, ps->d2p) < 0))
return rv;
}
return wid;
}
static s3cipid_t
dict_ciphone_id(dict_t * d, const char *str)
{
if (d->nocase)
return bin_mdef_ciphone_id_nocase(d->mdef, str);
else
return bin_mdef_ciphone_id(d->mdef, str);
}
int
dict_add_g2p_word(dict_t * dict, char const *word)
{
int32 wid = 0;
s3cipid_t *pron;
char **phonestr, *tmp;
int np, i;
char *phones;
phones = dict_g2p(word, dict->ngram_g2p_model);
if (phones == NULL)
return 0;
E_INFO("Adding phone %s for word %s \n", phones, word);
tmp = ckd_salloc(phones);
np = str2words(tmp, NULL, 0);
phonestr = ckd_calloc(np, sizeof(*phonestr));
str2words(tmp, phonestr, np);
pron = ckd_calloc(np, sizeof(*pron));
for (i = 0; i < np; ++i) {
pron[i] = bin_mdef_ciphone_id(dict->mdef, phonestr[i]);
if (pron[i] == -1) {
E_ERROR("Unknown phone %s in phone string %s\n",
phonestr[i], tmp);
ckd_free(phonestr);
ckd_free(tmp);
ckd_free(pron);
ckd_free(phones);
return -1;
}
}
ckd_free(phonestr);
ckd_free(tmp);
ckd_free(phones);
if ((wid = dict_add_word(dict, word, pron, np)) == -1) {
ckd_free(pron);
return -1;
}
ckd_free(pron);
return wid;
}
int
ps_add_word(ps_decoder_t *ps,
char const *word,
char const *phones,
int update)
{
int32 wid;
s3cipid_t *pron;
hash_iter_t *search_it;
char **phonestr, *tmp;
int np, i, rv;
/* Parse phones into an array of phone IDs. */
tmp = ckd_salloc(phones);
np = str2words(tmp, NULL, 0);
phonestr = ckd_calloc(np, sizeof(*phonestr));
str2words(tmp, phonestr, np);
pron = ckd_calloc(np, sizeof(*pron));
for (i = 0; i < np; ++i) {
pron[i] = bin_mdef_ciphone_id(ps->acmod->mdef, phonestr[i]);
if (pron[i] == -1) {
E_ERROR("Unknown phone %s in phone string %s\n",
phonestr[i], tmp);
ckd_free(phonestr);
ckd_free(tmp);
ckd_free(pron);
return -1;
}
}
/* No longer needed. */
ckd_free(phonestr);
ckd_free(tmp);
/* Add it to the dictionary. */
if ((wid = dict_add_word(ps->dict, word, pron, np)) == -1) {
ckd_free(pron);
return -1;
}
/* No longer needed. */
ckd_free(pron);
/* Now we also have to add it to dict2pid. */
dict2pid_add_word(ps->d2p, wid);
/* TODO: we definitely need to refactor this */
for (search_it = hash_table_iter(ps->searches); search_it;
search_it = hash_table_iter_next(search_it)) {
ps_search_t *search = hash_entry_val(search_it->ent);
if (!strcmp(PS_SEARCH_NGRAM, ps_search_name(search))) {
ngram_model_t *lmset = ((ngram_search_t *) search)->lmset;
if (ngram_model_add_word(lmset, word, 1.0) == NGRAM_INVALID_WID) {
hash_table_iter_free(search_it);
return -1;
}
}
if (update) {
if ((rv = ps_search_reinit(search, ps->dict, ps->d2p) < 0)) {
hash_table_iter_free(search_it);
return rv;
}
}
}
/* Rebuild the widmap and search tree if requested. */
return wid;
}
bin_mdef_t *
bin_mdef_read(cmd_ln_t *config, const char *filename)
{
bin_mdef_t *m;
FILE *fh;
size_t tree_start;
int32 val, i, swap, pos, end;
int32 *sseq_size;
int do_mmap;
/* Try to read it as text first. */
if ((m = bin_mdef_read_text(config, filename)) != NULL)
return m;
E_INFO("Reading binary model definition: %s\n", filename);
if ((fh = fopen(filename, "rb")) == NULL)
return NULL;
if (fread(&val, 4, 1, fh) != 1) {
fclose(fh);
E_ERROR_SYSTEM("Failed to read byte-order marker from %s\n",
filename);
return NULL;
}
swap = 0;
if (val == BIN_MDEF_OTHER_ENDIAN) {
swap = 1;
E_INFO("Must byte-swap %s\n", filename);
}
if (fread(&val, 4, 1, fh) != 1) {
fclose(fh);
E_ERROR_SYSTEM("Failed to read version from %s\n", filename);
return NULL;
}
if (swap)
SWAP_INT32(&val);
if (val > BIN_MDEF_FORMAT_VERSION) {
E_ERROR("File format version %d for %s is newer than library\n",
val, filename);
fclose(fh);
return NULL;
}
if (fread(&val, 4, 1, fh) != 1) {
fclose(fh);
E_ERROR_SYSTEM("Failed to read header length from %s\n", filename);
return NULL;
}
if (swap)
SWAP_INT32(&val);
/* Skip format descriptor. */
fseek(fh, val, SEEK_CUR);
/* Finally allocate it. */
m = ckd_calloc(1, sizeof(*m));
m->refcnt = 1;
/* Check these, to make gcc/glibc shut up. */
#define FREAD_SWAP32_CHK(dest) \
if (fread((dest), 4, 1, fh) != 1) { \
fclose(fh); \
ckd_free(m); \
E_ERROR_SYSTEM("Failed to read %s from %s\n", #dest, filename); \
return NULL; \
} \
if (swap) SWAP_INT32(dest);
FREAD_SWAP32_CHK(&m->n_ciphone);
FREAD_SWAP32_CHK(&m->n_phone);
FREAD_SWAP32_CHK(&m->n_emit_state);
FREAD_SWAP32_CHK(&m->n_ci_sen);
FREAD_SWAP32_CHK(&m->n_sen);
FREAD_SWAP32_CHK(&m->n_tmat);
FREAD_SWAP32_CHK(&m->n_sseq);
FREAD_SWAP32_CHK(&m->n_ctx);
FREAD_SWAP32_CHK(&m->n_cd_tree);
FREAD_SWAP32_CHK(&m->sil);
/* CI names are first in the file. */
m->ciname = ckd_calloc(m->n_ciphone, sizeof(*m->ciname));
/* Decide whether to read in the whole file or mmap it. */
do_mmap = config ? cmd_ln_boolean_r(config, "-mmap") : TRUE;
if (swap) {
E_WARN("-mmap specified, but mdef is other-endian. Will not memory-map.\n");
do_mmap = FALSE;
}
/* Actually try to mmap it. */
if (do_mmap) {
m->filemap = mmio_file_read(filename);
if (m->filemap == NULL)
do_mmap = FALSE;
}
pos = ftell(fh);
if (do_mmap) {
/* Get the base pointer from the memory map. */
m->ciname[0] = (char *)mmio_file_ptr(m->filemap) + pos;
/* Success! */
m->alloc_mode = BIN_MDEF_ON_DISK;
}
else {
/* Read everything into memory. */
m->alloc_mode = BIN_MDEF_IN_MEMORY;
fseek(fh, 0, SEEK_END);
end = ftell(fh);
fseek(fh, pos, SEEK_SET);
m->ciname[0] = ckd_malloc(end - pos);
if (fread(m->ciname[0], 1, end - pos, fh) != end - pos)
E_FATAL("Failed to read %d bytes of data from %s\n", end - pos, filename);
}
for (i = 1; i < m->n_ciphone; ++i)
m->ciname[i] = m->ciname[i - 1] + strlen(m->ciname[i - 1]) + 1;
/* Skip past the padding. */
tree_start =
m->ciname[i - 1] + strlen(m->ciname[i - 1]) + 1 - m->ciname[0];
tree_start = (tree_start + 3) & ~3;
m->cd_tree = (cd_tree_t *) (m->ciname[0] + tree_start);
if (swap) {
for (i = 0; i < m->n_cd_tree; ++i) {
SWAP_INT16(&m->cd_tree[i].ctx);
SWAP_INT16(&m->cd_tree[i].n_down);
SWAP_INT32(&m->cd_tree[i].c.down);
}
}
m->phone = (mdef_entry_t *) (m->cd_tree + m->n_cd_tree);
if (swap) {
for (i = 0; i < m->n_phone; ++i) {
SWAP_INT32(&m->phone[i].ssid);
SWAP_INT32(&m->phone[i].tmat);
}
}
sseq_size = (int32 *) (m->phone + m->n_phone);
if (swap)
SWAP_INT32(sseq_size);
m->sseq = ckd_calloc(m->n_sseq, sizeof(*m->sseq));
m->sseq[0] = (uint16 *) (sseq_size + 1);
if (swap) {
for (i = 0; i < *sseq_size; ++i)
SWAP_INT16(m->sseq[0] + i);
}
if (m->n_emit_state) {
for (i = 1; i < m->n_sseq; ++i)
m->sseq[i] = m->sseq[0] + i * m->n_emit_state;
}
else {
m->sseq_len = (uint8 *) (m->sseq[0] + *sseq_size);
for (i = 1; i < m->n_sseq; ++i)
m->sseq[i] = m->sseq[i - 1] + m->sseq_len[i - 1];
}
/* Now build the CD-to-CI mappings using the senone sequences.
* This is the only really accurate way to do it, though it is
* still inaccurate in the case of heterogeneous topologies or
* cross-state tying. */
m->cd2cisen = (int16 *) ckd_malloc(m->n_sen * sizeof(*m->cd2cisen));
m->sen2cimap = (int16 *) ckd_malloc(m->n_sen * sizeof(*m->sen2cimap));
/* Default mappings (identity, none) */
for (i = 0; i < m->n_ci_sen; ++i)
m->cd2cisen[i] = i;
for (; i < m->n_sen; ++i)
m->cd2cisen[i] = -1;
for (i = 0; i < m->n_sen; ++i)
m->sen2cimap[i] = -1;
for (i = 0; i < m->n_phone; ++i) {
int32 j, ssid = m->phone[i].ssid;
for (j = 0; j < bin_mdef_n_emit_state_phone(m, i); ++j) {
int s = bin_mdef_sseq2sen(m, ssid, j);
int ci = bin_mdef_pid2ci(m, i);
/* Take the first one and warn if we have cross-state tying. */
if (m->sen2cimap[s] == -1)
m->sen2cimap[s] = ci;
if (m->sen2cimap[s] != ci)
E_WARN
("Senone %d is shared between multiple base phones\n",
s);
if (j > bin_mdef_n_emit_state_phone(m, ci))
E_WARN("CD phone %d has fewer states than CI phone %d\n",
i, ci);
else
m->cd2cisen[s] =
bin_mdef_sseq2sen(m, m->phone[ci].ssid, j);
}
}
/* Set the silence phone. */
m->sil = bin_mdef_ciphone_id(m, S3_SILENCE_CIPHONE);
E_INFO
("%d CI-phone, %d CD-phone, %d emitstate/phone, %d CI-sen, %d Sen, %d Sen-Seq\n",
m->n_ciphone, m->n_phone - m->n_ciphone, m->n_emit_state,
m->n_ci_sen, m->n_sen, m->n_sseq);
fclose(fh);
return m;
}
