/*
* Reads function-like macro arguments. Returns true if the argument list ends
* with "...". Otherwise false.
*/
static bool read_funclike_define_args(CppContext *ctx, Dict *param) {
for (;;) {
Token *tok = read_cpp_token(ctx);
if (is_punct(tok, ')'))
return false;
if (dict_size(param)) {
if (!is_punct(tok, ','))
error_token(tok, "',' expected, but got '%s'", token_to_string(tok));
tok = read_cpp_token(ctx);
}
if (!tok || tok->toktype == TOKTYPE_NEWLINE)
error_token(tok, "missing ')' in macro parameter list");
if (is_punct(tok, KEYWORD_THREEDOTS)) {
Token *subst = make_token(ctx, TOKTYPE_MACRO_PARAM, (TokenValue)dict_size(param));
dict_put(param, to_string("__VA_ARGS__"), subst);
Token *tok1 = read_cpp_token(ctx);
if (!is_punct(tok1, ')'))
error_token(tok1, "')' expected, but got '%s'", token_to_string(tok1));
return true;
}
if (tok->toktype != TOKTYPE_IDENT)
error_token(tok, "identifier expected, but got '%s'", token_to_string(tok));
Token *subst = make_token(ctx, TOKTYPE_MACRO_PARAM, (TokenValue)dict_size(param));
dict_put(param, tok->val.str, subst);
}
}
void dictExample() {
dict_t *d = (dict_t *)malloc(sizeof(dict_t));
char *var;
char *key1 = strToHeap("key1");
char *key2 = strToHeap("key2");
char *key3 = strToHeap("key3");
char *value1 = strToHeap("var1");
char *value2 = strToHeap("var2");
char *value3 = strToHeap("var3");
dict_init(d);
dict_set(d, key1, value1);
dict_set(d, key2, value2);
dict_set(d, key3, value3);
dict_get(d, "key1", (void **)&var);
printf("key1=>%s\n", var);
dict_get(d, "key2", (void **)&var);
printf("key1=>%s\n", var);
dict_get(d, "key3", (void **)&var);
printf("key1=>%s\n", var);
printf("dict size:%d\n", dict_size(d));
if (1 == dict_del(d, key3)) {
key3 = NULL;
value3 = NULL;
printf("del key3 done\n");
}
char **ks = (char **)malloc(dict_size(d)*sizeof(char*));
int i;
dict_keys(d, ks);
for(i = 0; i < dict_size(d); i++)
printf("%s ",*(ks+i) );
printf("\n");
char *k, *v;
while(dict_iter(d, &k, (void **)&v))
printf("%s = >%s\n", k, v);
dict_reset(d);
dict_destory(d);
free(d);
}
void
ps_search_init(ps_search_t *search, ps_searchfuncs_t *vt,
cmd_ln_t *config, acmod_t *acmod, dict_t *dict,
dict2pid_t *d2p)
{
search->vt = vt;
search->config = config;
search->acmod = acmod;
if (d2p)
search->d2p = dict2pid_retain(d2p);
else
search->d2p = NULL;
if (dict) {
search->dict = dict_retain(dict);
search->start_wid = dict_startwid(dict);
search->finish_wid = dict_finishwid(dict);
search->silence_wid = dict_silwid(dict);
search->n_words = dict_size(dict);
}
else {
search->dict = NULL;
search->start_wid = search->finish_wid = search->silence_wid = -1;
search->n_words = 0;
}
}
void _setup_number_lexer(void) {
lexa = setup_with_scanners();
lexa_add_scanner(lexa, "number");
ck_assert_int_eq(dict_size(lexa -> scanners), 4);
lexa_build_lexer(lexa);
ck_assert_ptr_ne(lexa -> config, NULL);
}
int
ngram_fwdflat_reinit(ngram_search_t *ngs)
{
/* Reallocate things that depend on the number of words. */
int n_words;
ckd_free(ngs->fwdflat_wordlist);
ckd_free(ngs->expand_word_list);
bitvec_free(ngs->expand_word_flag);
n_words = ps_search_n_words(ngs);
ngs->fwdflat_wordlist = ckd_calloc(n_words + 1, sizeof(*ngs->fwdflat_wordlist));
ngs->expand_word_flag = bitvec_alloc(n_words);
ngs->expand_word_list = ckd_calloc(n_words + 1, sizeof(*ngs->expand_word_list));
/* No tree-search; take care of the expansion list and single phone words. */
if (!ngs->fwdtree) {
/* Free single-phone words. */
ngram_fwdflat_free_1ph(ngs);
/* Reallocate word_chan. */
ckd_free(ngs->word_chan);
ngs->word_chan = ckd_calloc(dict_size(ps_search_dict(ngs)),
sizeof(*ngs->word_chan));
/* Rebuild full expansion list from LM words. */
ngram_fwdflat_expand_all(ngs);
/* Allocate single phone words. */
ngram_fwdflat_allocate_1ph(ngs);
}
/* Otherwise there is nothing to do since the wordlist is
* generated anew every utterance. */
return 0;
}
lmset_t *
lmset_read_lm(const char *lmfile, dict_t * dict, const char *lmname,
float64 lw, float64 wip, float64 uw, const char *lmdumpdir,
logmath_t *logmath)
{
lmset_t *lms;
lms = (lmset_t *) ckd_calloc(1, sizeof(lmset_t));
lms->n_lm = 1;
lms->n_alloc_lm = 1;
/* Only allocate one single LM. This assumes no class definition would be defined.
*/
lms->lmarray = (lm_t **) ckd_calloc(1, sizeof(lm_t *));
/*
No need to check whether lmname exists here.
*/
if ((lms->lmarray[0] =
lm_read_advance(lmfile, lmname, lw, wip, uw, dict_size(dict),
NULL, 1, logmath, FALSE, FALSE)) == NULL)
E_FATAL
("lm_read_advance(%s, %e, %e, %e %d [Arbitrary Fmt], Weighted Apply) failed\n",
lmfile, lw, wip, uw, dict_size(dict));
if (dict != NULL) {
assert(lms->lmarray[0]);
if ((lms->lmarray[0]->dict2lmwid =
wid_dict_lm_map(dict, lms->lmarray[0], lw)) == NULL)
E_FATAL
("Dict/LM word-id mapping failed for LM index %d, named %s\n",
0, lmset_idx_to_name(lms, 0));
}
else {
E_FATAL
("Dict is specified to be NULL (dict_init is not called before lmset_read_lm?), dict2lmwid is not built inside lmset_read_lm\n");
}
return lms;
}
void
ld_read_lm(live_decoder_t * _decoder,
const char *lmpath, const char *lmname)
{
srch_t *s;
lm_t *lm;
int32 ndict;
s = (srch_t *) _decoder->kb.srch;
ndict = dict_size(_decoder->kb.kbcore->dict);
lm = lm_read_advance(lmpath, lmname, cmd_ln_float32("-lw"), cmd_ln_float32("-wip"), cmd_ln_float32("-uw"), ndict, NULL, 1 /* Weight apply */
);
s->srch_add_lm(s, lm, lmname);
}
/*
* Add silence and noise filler words to the FSG.
* Return the number of transitions added..
*/
static int32
word_fsg_add_filler(word_fsg_t * fsg, float32 silprob, float32 fillprob, logmath_t * logmath)
{
dict_t *dict;
int32 src;
int32 wid, silwid, n_word;
int32 n_trans;
int32 logsilp, logfillp;
E_INFO("Adding filler words to FSG\n");
assert(fsg);
dict = fsg->dict;
silwid = dict_silwid(dict);
n_word = dict_size(dict);
logsilp = (int32) (logs3(logmath, silprob) * fsg->lw);
logfillp = (int32) (logs3(logmath, fillprob) * fsg->lw);
/*
* Add silence and filler word self-loop transitions to each state.
* NOTE: No check to see if these words already exist in FSG!
*/
n_trans = 0;
if (silwid >= 0) {
for (src = 0; src < fsg->n_state; src++) {
word_fsg_trans_add(fsg, src, src, logsilp, silwid);
n_trans++;
if (fillprob > 0.0) {
/* Add other filler (noise) words */
for (wid = silwid + 1; wid < n_word; wid++) {
word_fsg_trans_add(fsg, src, src, logfillp, wid);
n_trans++;
}
}
}
}
return n_trans;
}
void
dict2pid_dump(FILE * fp, dict2pid_t * d2p)
{
int32 w, p, pronlen;
int32 i, j, b, l, r;
bin_mdef_t *mdef = d2p->mdef;
dict_t *dict = d2p->dict;
fprintf(fp, "# INTERNAL (wd comssid ssid ssid ... ssid comssid)\n");
for (w = 0; w < dict_size(dict); w++) {
fprintf(fp, "%30s ", dict_wordstr(dict, w));
pronlen = dict_pronlen(dict, w);
for (p = 0; p < pronlen; p++)
fprintf(fp, " %5d", dict2pid_internal(d2p, w, p));
fprintf(fp, "\n");
}
fprintf(fp, "#\n");
fprintf(fp, "# LDIPH_LC (b r l ssid)\n");
for (b = 0; b < bin_mdef_n_ciphone(mdef); b++) {
for (r = 0; r < bin_mdef_n_ciphone(mdef); r++) {
for (l = 0; l < bin_mdef_n_ciphone(mdef); l++) {
if (IS_S3SSID(d2p->ldiph_lc[b][r][l]))
fprintf(fp, "%6s %6s %6s %5d\n", bin_mdef_ciphone_str(mdef, (s3cipid_t) b), bin_mdef_ciphone_str(mdef, (s3cipid_t) r), bin_mdef_ciphone_str(mdef, (s3cipid_t) l), d2p->ldiph_lc[b][r][l]); /* RAH, ldiph_lc is returning an int32, %d expects an int16 */
}
}
}
fprintf(fp, "#\n");
fprintf(fp, "# SSEQ %d (senid senid ...)\n", mdef->n_sseq);
for (i = 0; i < mdef->n_sseq; i++) {
fprintf(fp, "%5d ", i);
for (j = 0; j < bin_mdef_n_emit_state(mdef); j++)
fprintf(fp, " %5d", mdef->sseq[i][j]);
fprintf(fp, "\n");
}
fprintf(fp, "#\n");
fprintf(fp, "# END\n");
fflush(fp);
}
void in_received_handler(
DictionaryIterator *received, void *context) {
// incoming message received
APP_LOG(APP_LOG_LEVEL_DEBUG, "In Received");
uint32_t size = dict_size(received);
uint8_t buffer[size];
Tuple *tuple = dict_read_begin_from_buffer(received, buffer, size);
char buf[500];
while(tuple) {
char tmp[500];
tuple = dict_read_next(received);
snprintf(tmp, 500, "%s\ntype: %d data:%s", buf, tuple->type, tuple->value->cstring);
snprintf(buf, 500, "%s", tmp);
}
tuple = dict_read_first(received);
while (tuple) {
char name[256];
char value[256];
for(int i = 0; i < tuple->length; i++) {
name[i] = tuple->value->cstring[i];
}
tuple = dict_read_next(received);
if(tuple) {
for(int i = 0; i < tuple->length; i++) {
value[i] = tuple->value->cstring[i];
}
put_entry(name, value);
} else {
break;
}
}
}
/**
* Parses messages received from phone and fowards them to correct class for processing
* @params: (see Pebble API for details)
*/
static void inbox_received_callback(DictionaryIterator *iterator, void *context) {
LOG("Message received!");
LOG("Size: %i", (int)dict_size(iterator));
Tuple *data_type = dict_find(iterator, MESSAGE_TYPE_INDEX);
//Parse message
if (data_type) {
char * type = data_type->value->cstring;
LOG("Message received with type: %s", type);
char* header = dict_find(iterator, MESSAGE_HEADER_INDEX)->value->cstring;
//Put all data into a linked list and send to handlers.
LinkedRoot* data = linked_list_create_root();
for (int i= MESSAGE_DATA_ROOT; i < MESSAGE_DATA_ROOT + MESSAGE_DATA_MAX; i++) {
Tuple* single_data = dict_find(iterator, i);
if (single_data){
linked_list_append(data, single_data->value->cstring);
}
else break;
}
if (strcmp(type,"READY") == 0) {
jsReady = true;
return;
};
if (strcmp(type,"MOVES") == 0) {
workout_parse_message(header, data);
}
if (strcmp(type,"WORKOUTS") == 0) {
storage_store_workouts(header, data);
vibes_short_pulse();
}
}
}
void
s3_decode_read_lm(s3_decode_t * _decode,
const char *lmpath, const char *lmname)
{
srch_t *s;
lm_t *lm;
int32 ndict;
s = (srch_t *) _decode->kb.srch;
ndict = dict_size(_decode->kb.kbcore->dict);
lm = lm_read_advance(lmpath, lmname,
cmd_ln_float32_r(kbcore_config(_decode->kbcore), "-lw"),
cmd_ln_float32_r(kbcore_config(_decode->kbcore), "-wip"),
cmd_ln_float32_r(kbcore_config(_decode->kbcore), "-uw"),
ndict, NULL, 1, /* Weight apply */
kbcore_logmath(s->kbc),
cmd_ln_boolean_r(kbcore_config(_decode->kbcore), "-ugonly"),
cmd_ln_boolean_r(kbcore_config(_decode->kbcore), "-bgonly")
);
s->funcs->add_lm(s, lm, lmname);
}
void
ps_search_base_reinit(ps_search_t *search, dict_t *dict,
dict2pid_t *d2p)
{
dict_free(search->dict);
dict2pid_free(search->d2p);
/* FIXME: _retain() should just return NULL if passed NULL. */
if (dict) {
search->dict = dict_retain(dict);
search->start_wid = dict_startwid(dict);
search->finish_wid = dict_finishwid(dict);
search->silence_wid = dict_silwid(dict);
search->n_words = dict_size(dict);
}
else {
search->dict = NULL;
search->start_wid = search->finish_wid = search->silence_wid = -1;
search->n_words = 0;
}
if (d2p)
search->d2p = dict2pid_retain(d2p);
else
search->d2p = NULL;
}
int
main(int argc, char *argv[])
{
dict_t *dict;
dict2pid_t *d2p;
bin_mdef_t *mdef;
glextree_t *tree;
hmm_context_t *ctx;
logmath_t *lmath;
tmat_t *tmat;
int i;
TEST_ASSERT(mdef = bin_mdef_read(NULL, MODELDIR "/hmm/en_US/hub4wsj_sc_8k/mdef"));
TEST_ASSERT(dict = dict_init(cmd_ln_init(NULL, NULL, FALSE,
"-dict", DATADIR "/turtle.dic",
"-dictcase", "no", NULL),
mdef));
TEST_ASSERT(d2p = dict2pid_build(mdef, dict));
lmath = logmath_init(1.0001, 0, TRUE);
tmat = tmat_init(MODELDIR "/hmm/en_US/hub4wsj_sc_8k/transition_matrices",
lmath, 1e-5, TRUE);
ctx = hmm_context_init(bin_mdef_n_emit_state(mdef), tmat->tp, NULL, mdef->sseq);
TEST_ASSERT(tree = glextree_build(ctx, dict, d2p, NULL, NULL));
/* Check that a path exists for all dictionary words. */
for (i = 0; i < dict_size(dict); ++i)
TEST_ASSERT(glextree_has_word(tree, i));
dict_free(dict);
dict2pid_free(d2p);
bin_mdef_free(mdef);
tmat_free(tmat);
hmm_context_free(ctx);
glextree_free(tree);
return 0;
}
static void in_received_handler(DictionaryIterator *received, void *context)
{
APP_LOG(APP_LOG_LEVEL_DEBUG, "in_received_handler received.size: %ld", dict_size(received));
// Log Time & Elapsed Time
time_t now = time(NULL);
time_t elapsedTime = now - lastUpdateTime;
lastUpdateTime = time(NULL);
struct tm *clock_time = localtime(&now);
char error_time_text[] = "00:00:00";
strftime(error_time_text, sizeof(error_time_text), "%T", clock_time);
receivedMessagesCounter++;
Tuple *tuple = dict_read_first(received);
while (tuple)
{
switch (tuple->key) {
case MESSAGE_KEY:
if (strcmp (text_layer_buffer, tuple->value->cstring) != 0)
{
strcpy (text_layer_buffer, tuple->value->cstring);
text_layer_set_text(text_layer, text_layer_buffer);
}
if (strcmp (text_layer_buffer, "hello") == 0)
textCounter = 0;
if (strcmp (text_layer_buffer, "hallo") == 0)
textCounter = 1;
if (strcmp (text_layer_buffer, "hola") == 0)
textCounter = 2;
if (strcmp (text_layer_buffer, "hej") == 0)
textCounter = 3;
if (strcmp (text_layer_buffer, "bonjour") == 0)
textCounter = 4;
if (strcmp (text_layer_buffer, "ciao") == 0)
textCounter = 5;
if (strcmp (text_layer_buffer, "salve") == 0)
textCounter = 6;
if (strcmp (text_layer_buffer, "ola") == 0)
textCounter = 7;
if (strcmp (text_layer_buffer, "chaoa") == 0)
textCounter = 8;
if (strcmp (text_layer_buffer, "kaixo") == 0)
textCounter = 9;
if (receivedMessagesCounter % 10000 == 0)
APP_LOG(APP_LOG_LEVEL_DEBUG, "[%s] App Message in_received_handler ELAPSED TIME: %ld message: %s", error_time_text, elapsedTime, tuple->value->cstring);
APP_LOG(APP_LOG_LEVEL_DEBUG, " in_received_handler tuple->key: MESSAGE_KEY");
break;
case DATA_KEY:
if (data_buffer != NULL)
free(data_buffer);
data_buffer = malloc(tuple->length);
memcpy(data_buffer, tuple->value->data, tuple->length);
APP_LOG(APP_LOG_LEVEL_DEBUG, " in_received_handler tuple->key: DATA_KEY");
break;
case SNIFF_KEY:
APP_LOG(APP_LOG_LEVEL_DEBUG, "[%s] App Message in_received_handler ELAPSED TIME: %ld SNIFF_KEY RECEIVED", error_time_text, elapsedTime);
if (tuple->value->uint8)
{
// Set Reduced Sniff Interval for Faster Response Time
app_comm_set_sniff_interval(SNIFF_INTERVAL_REDUCED);
char message_text[] = "Set SNIFF_INTERVAL_REDUCED";
// Send App Message Out
DictionaryIterator *iterator;
if (app_message_outbox_begin(&iterator) != APP_MSG_OK)
{
return;
}
if (dict_write_cstring(iterator, MESSAGE_KEY, message_text) != DICT_OK)
{
return;
}
app_message_outbox_send();
}
else
{
// Set Normal Sniff Interval
app_comm_set_sniff_interval(SNIFF_INTERVAL_NORMAL);
char message_text[] = "Set SNIFF_INTERVAL_NORMAL";
// Send App Message Out
DictionaryIterator *iterator;
if (app_message_outbox_begin(&iterator) != APP_MSG_OK)
{
return;
}
if (dict_write_cstring(iterator, MESSAGE_KEY, message_text) != DICT_OK)
{
return;
}
app_message_outbox_send();
}
APP_LOG(APP_LOG_LEVEL_DEBUG, " in_received_handler tuple->key: SNIFF_KEY");
break;
default:
break;
}
tuple = dict_read_next(received);
}
// Handle Tuples
/*
Tuple *message_tuple = dict_find(received, MESSAGE_KEY);
Tuple *data_tuple = dict_find(received, DATA_KEY);
Tuple *sniff_tuple = dict_find(received, SNIFF_KEY);
if (message_tuple)
{
// At high data rates, there is no guarantee the memory storing the message_tuple will not get overwritten by a new, incoming app message dictionary.
// Seting the text layer to the message_tuple->value->cstring can cause unexpected results.
// Copying the cstring value to a buffer and then setting the text layer text to the buffer appears to eliminate this issue.
// This code can produce unexpected results:
////
if (strcmp (text_layer_buffer, message_tuple->value->cstring) != 0)
{
strcpy (text_layer_buffer, message_tuple->value->cstring);
text_layer_set_text(text_layer, message_tuple->value->cstring);
}
////
// This code works as expected:
if (strcmp (text_layer_buffer, message_tuple->value->cstring) != 0)
{
strcpy (text_layer_buffer, message_tuple->value->cstring);
text_layer_set_text(text_layer, text_layer_buffer);
}
if (strcmp (text_layer_buffer, "hello") == 0)
{
textCounter = 0;
}
if (strcmp (text_layer_buffer, "hallo") == 0)
{
textCounter = 1;
}
if (strcmp (text_layer_buffer, "hola") == 0)
{
textCounter = 2;
}
if (strcmp (text_layer_buffer, "hej") == 0)
{
textCounter = 3;
}
if (strcmp (text_layer_buffer, "bonjour") == 0)
{
textCounter = 4;
}
if (strcmp (text_layer_buffer, "ciao") == 0)
{
textCounter = 5;
}
if (strcmp (text_layer_buffer, "salve") == 0)
{
textCounter = 6;
}
if (strcmp (text_layer_buffer, "ola") == 0)
{
textCounter = 7;
}
if (strcmp (text_layer_buffer, "chaoa") == 0)
{
textCounter = 8;
}
if (strcmp (text_layer_buffer, "kaixo") == 0)
{
textCounter = 9;
}
if (receivedMessagesCounter % 10000 == 0)
APP_LOG(APP_LOG_LEVEL_DEBUG, "[%s] App Message in_received_handler ELAPSED TIME: %ld message: %s", error_time_text, elapsedTime, message_tuple->value->cstring);
}
else
{
APP_LOG(APP_LOG_LEVEL_DEBUG, "[%s] App Message in_received_handler ELAPSED TIME: %ld NO MESSAGE TUPLE", error_time_text, elapsedTime);
}
if (data_tuple)
{
if (data_buffer != NULL)
{
free(data_buffer);
}
data_buffer = malloc(data_tuple->length);
memcpy(data_buffer, data_tuple->value->data, data_tuple->length);
}
if (sniff_tuple)
{
APP_LOG(APP_LOG_LEVEL_DEBUG, "[%s] App Message in_received_handler ELAPSED TIME: %ld SNIFF_KEY RECEIVED", error_time_text, elapsedTime);
if (sniff_tuple->value->uint8)
{
// Set Reduced Sniff Interval for Faster Response Time
app_comm_set_sniff_interval(SNIFF_INTERVAL_REDUCED);
char message_text[] = "Set SNIFF_INTERVAL_REDUCED";
// Send App Message Out
DictionaryIterator *iterator;
if (app_message_outbox_begin(&iterator) != APP_MSG_OK)
{
return;
}
if (dict_write_cstring(iterator, MESSAGE_KEY, message_text) != DICT_OK)
{
return;
}
app_message_outbox_send();
}
else
{
// Set Normal Sniff Interval
app_comm_set_sniff_interval(SNIFF_INTERVAL_NORMAL);
char message_text[] = "Set SNIFF_INTERVAL_NORMAL";
// Send App Message Out
DictionaryIterator *iterator;
if (app_message_outbox_begin(&iterator) != APP_MSG_OK)
{
return;
}
if (dict_write_cstring(iterator, MESSAGE_KEY, message_text) != DICT_OK)
{
return;
}
app_message_outbox_send();
}
}
*/
}
/*ARCHAN, to allow backward compatibility -lm, -lmctlfn coexists. This makes the current implmentation more complicated than necessary. */
void kb_init (kb_t *kb)
{
kbcore_t *kbcore;
mdef_t *mdef;
dict_t *dict;
dict2pid_t *d2p;
lm_t *lm;
lmset_t *lmset;
s3cipid_t sil, ci;
s3wid_t w;
int32 i, n, n_lc;
wordprob_t *wp;
s3cipid_t *lc;
bitvec_t lc_active;
char *str;
int32 cisencnt;
int32 j;
/* Initialize the kb structure to zero, just in case */
memset(kb, 0, sizeof(*kb));
kb->kbcore = NULL;
kb->kbcore = kbcore_init (cmd_ln_float32 ("-logbase"),
cmd_ln_str("-feat"),
cmd_ln_str("-cmn"),
cmd_ln_str("-varnorm"),
cmd_ln_str("-agc"),
cmd_ln_str("-mdef"),
cmd_ln_str("-dict"),
cmd_ln_str("-fdict"),
"", /* Hack!! Hardwired constant
for -compsep argument */
cmd_ln_str("-lm"),
cmd_ln_str("-lmctlfn"),
cmd_ln_str("-lmdumpdir"),
cmd_ln_str("-fillpen"),
cmd_ln_str("-senmgau"),
cmd_ln_float32("-silprob"),
cmd_ln_float32("-fillprob"),
cmd_ln_float32("-lw"),
cmd_ln_float32("-wip"),
cmd_ln_float32("-uw"),
cmd_ln_str("-mean"),
cmd_ln_str("-var"),
cmd_ln_float32("-varfloor"),
cmd_ln_str("-mixw"),
cmd_ln_float32("-mixwfloor"),
cmd_ln_str("-subvq"),
cmd_ln_str("-gs"),
cmd_ln_str("-tmat"),
cmd_ln_float32("-tmatfloor"));
if(kb->kbcore==NULL){
E_FATAL("Initialization of kb failed\n");
}
kbcore = kb->kbcore;
mdef = kbcore_mdef(kbcore);
dict = kbcore_dict(kbcore);
lm = kbcore_lm(kbcore);
lmset=kbcore_lmset(kbcore);
d2p = kbcore_dict2pid(kbcore);
if (NOT_S3WID(dict_startwid(dict)) || NOT_S3WID(dict_finishwid(dict)))
E_FATAL("%s or %s not in dictionary\n", S3_START_WORD, S3_FINISH_WORD);
if(lmset){
for(i=0;i<kbcore_nlm(kbcore);i++){
if (NOT_S3LMWID(lm_startwid(lmset[i].lm)) || NOT_S3LMWID(lm_finishwid(lmset[i].lm)))
E_FATAL("%s or %s not in LM %s\n", S3_START_WORD, S3_FINISH_WORD,lmset[i].name);
}
}else if(lm){
if (NOT_S3LMWID(lm_startwid(lm)) || NOT_S3LMWID(lm_finishwid(lm)))
E_FATAL("%s or %s not in LM\n", S3_START_WORD, S3_FINISH_WORD);
}
/* Check that HMM topology restrictions are not violated */
if (tmat_chk_1skip (kbcore->tmat) < 0)
E_FATAL("Tmat contains arcs skipping more than 1 state\n");
/*
* Unlink <s> and </s> between dictionary and LM, to prevent their
* recognition. They are merely dummy words (anchors) at the beginning
* and end of each utterance.
*/
if(lmset){
for(i=0;i<kbcore_nlm(kbcore);i++){
lm_lmwid2dictwid(lmset[i].lm, lm_startwid(lmset[i].lm)) = BAD_S3WID;
lm_lmwid2dictwid(lmset[i].lm, lm_finishwid(lmset[i].lm)) = BAD_S3WID;
for (w = dict_startwid(dict); IS_S3WID(w); w = dict_nextalt(dict, w))
lmset[i].lm->dict2lmwid[w] = BAD_S3LMWID;
for (w = dict_finishwid(dict); IS_S3WID(w); w = dict_nextalt(dict, w))
lmset[i].lm->dict2lmwid[w] = BAD_S3LMWID;
}
}else if(lm){ /* No LM is set at this point*/
lm_lmwid2dictwid(lm, lm_startwid(lm)) = BAD_S3WID;
lm_lmwid2dictwid(lm, lm_finishwid(lm)) = BAD_S3WID;
for (w = dict_startwid(dict); IS_S3WID(w); w = dict_nextalt(dict, w))
kbcore->dict2lmwid[w] = BAD_S3LMWID;
for (w = dict_finishwid(dict); IS_S3WID(w); w = dict_nextalt(dict, w))
kbcore->dict2lmwid[w] = BAD_S3LMWID;
}
sil = mdef_silphone (kbcore_mdef (kbcore));
if (NOT_S3CIPID(sil))
E_FATAL("Silence phone '%s' not in mdef\n", S3_SILENCE_CIPHONE);
kb->sen_active = (int32 *) ckd_calloc (mdef_n_sen(mdef), sizeof(int32));
kb->rec_sen_active = (int32 *) ckd_calloc (mdef_n_sen(mdef), sizeof(int32));
kb->ssid_active = (int32 *) ckd_calloc (mdef_n_sseq(mdef), sizeof(int32));
kb->comssid_active = (int32 *) ckd_calloc (dict2pid_n_comsseq(d2p), sizeof(int32));
/* Build set of all possible left contexts */
lc = (s3cipid_t *) ckd_calloc (mdef_n_ciphone(mdef) + 1, sizeof(s3cipid_t));
lc_active = bitvec_alloc (mdef_n_ciphone (mdef));
for (w = 0; w < dict_size (dict); w++) {
ci = dict_pron (dict, w, dict_pronlen(dict, w) - 1);
if (! mdef_is_fillerphone (mdef, (int)ci))
bitvec_set (lc_active, ci);
}
ci = mdef_silphone(mdef);
bitvec_set (lc_active, ci);
for (ci = 0, n_lc = 0; ci < mdef_n_ciphone(mdef); ci++) {
if (bitvec_is_set (lc_active, ci))
lc[n_lc++] = ci;
}
lc[n_lc] = BAD_S3CIPID;
E_INFO("Building lextrees\n");
/* Get the number of lexical tree*/
kb->n_lextree = cmd_ln_int32 ("-Nlextree");
if (kb->n_lextree < 1) {
E_ERROR("No. of ugtrees specified: %d; will instantiate 1 ugtree\n",
kb->n_lextree);
kb->n_lextree = 1;
}
/* ARCHAN: This code was rearranged in s3.4 implementation of dynamic LM */
/* Build active word list */
wp = (wordprob_t *) ckd_calloc (dict_size(dict), sizeof(wordprob_t));
if(lmset){
kb->ugtreeMulti = (lextree_t **) ckd_calloc (kbcore_nlm(kbcore)*kb->n_lextree, sizeof(lextree_t *));
/* Just allocate pointers*/
kb->ugtree = (lextree_t **) ckd_calloc (kb->n_lextree, sizeof(lextree_t *));
for(i=0;i<kbcore_nlm(kbcore);i++){
E_INFO("Creating Unigram Table for lm %d name %s\n",i,lmset[i].name);
n=0;
for(j=0;j<dict_size(dict);j++){ /*try to be very careful again */
wp[j].wid=-1;
wp[j].prob=-1;
}
n = lm_ug_wordprob (lmset[i].lm, dict,MAX_NEG_INT32, wp);
E_INFO("Size of word table after unigram + words in class: %d.\n",n);
if (n < 1)
E_FATAL("%d active words in %s\n", n,lmset[i].name);
n = wid_wordprob2alt(dict,wp,n);
E_INFO("Size of word table after adding alternative prons: %d.\n",n);
if (cmd_ln_int32("-treeugprob") == 0) {
for (i = 0; i < n; i++)
wp[i].prob = -1; /* Flatten all initial probabilities */
}
for (j = 0; j < kb->n_lextree; j++) {
kb->ugtreeMulti[i*kb->n_lextree+j] = lextree_build (kbcore, wp, n, lc);
lextree_type (kb->ugtreeMulti[i*kb->n_lextree+j]) = 0;
E_INFO("Lextrees (%d) for lm %d name %s, %d nodes(ug)\n",
kb->n_lextree, i, lmset[i].name,lextree_n_node(kb->ugtreeMulti[i*kb->n_lextree+j]));
}
}
}else if (lm){
E_INFO("Creating Unigram Table\n");
n=0;
n = lm_ug_wordprob (lm, dict,MAX_NEG_INT32, wp);
E_INFO("Size of word table after unigram + words in class: %d\n",n);
if (n < 1)
E_FATAL("%d active words\n", n);
n = wid_wordprob2alt (dict, wp, n); /* Add alternative pronunciations */
/* Retain or remove unigram probs from lextree, depending on option */
if (cmd_ln_int32("-treeugprob") == 0) {
for (i = 0; i < n; i++)
wp[i].prob = -1; /* Flatten all initial probabilities */
}
/* Create the desired no. of unigram lextrees */
kb->ugtree = (lextree_t **) ckd_calloc (kb->n_lextree, sizeof(lextree_t *));
for (i = 0; i < kb->n_lextree; i++) {
kb->ugtree[i] = lextree_build (kbcore, wp, n, lc);
lextree_type (kb->ugtree[i]) = 0;
}
E_INFO("Lextrees(%d), %d nodes(ug)\n",
kb->n_lextree, lextree_n_node(kb->ugtree[0]));
}
/* Create filler lextrees */
/* ARCHAN : only one filler tree is supposed to be build even for dynamic LMs */
n = 0;
for (i = dict_filler_start(dict); i <= dict_filler_end(dict); i++) {
if (dict_filler_word(dict, i)) {
wp[n].wid = i;
wp[n].prob = fillpen (kbcore->fillpen, i);
n++;
}
}
kb->fillertree = (lextree_t **)ckd_calloc(kb->n_lextree,sizeof(lextree_t*));
for (i = 0; i < kb->n_lextree; i++) {
kb->fillertree[i] = lextree_build (kbcore, wp, n, NULL);
lextree_type (kb->fillertree[i]) = -1;
}
ckd_free ((void *) wp);
ckd_free ((void *) lc);
bitvec_free (lc_active);
E_INFO("Lextrees(%d), %d nodes(filler)\n",
kb->n_lextree,
lextree_n_node(kb->fillertree[0]));
if (cmd_ln_int32("-lextreedump")) {
if(lmset){
E_FATAL("Currently, doesn't support -lextreedump for multiple-LMs\n");
}
for (i = 0; i < kb->n_lextree; i++) {
fprintf (stderr, "UGTREE %d\n", i);
lextree_dump (kb->ugtree[i], dict, stderr);
}
for (i = 0; i < kb->n_lextree; i++) {
fprintf (stderr, "FILLERTREE %d\n", i);
lextree_dump (kb->fillertree[i], dict, stderr);
}
fflush (stderr);
}
kb->ascr = ascr_init (mgau_n_mgau(kbcore_mgau(kbcore)),
kbcore->dict2pid->n_comstate);
kb->beam = beam_init (cmd_ln_float64("-subvqbeam"),
cmd_ln_float64("-beam"),
cmd_ln_float64("-pbeam"),
cmd_ln_float64("-wbeam"));
E_INFO("Beam= %d, PBeam= %d, WBeam= %d, SVQBeam= %d\n",
kb->beam->hmm, kb->beam->ptrans, kb->beam->word, kb->beam->subvq);
/*Sections of optimization related parameters*/
kb->ds_ratio=cmd_ln_int32("-ds");
E_INFO("Down Sampling Ratio = %d\n",kb->ds_ratio);
kb->rec_bstcid=-1;
kb->skip_count=0;
kb->cond_ds=cmd_ln_int32("-cond_ds");
E_INFO("Conditional Down Sampling Parameter = %d\n",kb->cond_ds);
if(kb->cond_ds>0&&kb->kbcore->gs==NULL) E_FATAL("Conditional Down Sampling require the use of Gaussian Selection map\n");
kb->gs4gs=cmd_ln_int32("-gs4gs");
E_INFO("GS map would be used for Gaussian Selection? = %d\n",kb->gs4gs);
kb->svq4svq=cmd_ln_int32("-svq4svq");
E_INFO("SVQ would be used as Gaussian Score ?= %d\n",kb->svq4svq);
kb->ci_pbeam=-1*logs3(cmd_ln_float32("-ci_pbeam"));
E_INFO("CI phone beam to prune the number of parent CI phones in CI-base GMM Selection = %d\n",kb->ci_pbeam);
if(kb->ci_pbeam>10000000){
E_INFO("Virtually no CI phone beam is applied now. (ci_pbeam>1000000)\n");
}
kb->wend_beam=-1*logs3(cmd_ln_float32("-wend_beam"));
E_INFO("Word-end pruning beam: %d\n",kb->wend_beam);
kb->pl_window=cmd_ln_int32("-pl_window");
E_INFO("Phoneme look-ahead window size = %d\n",kb->pl_window);
kb->pl_window_start=0;
kb->pl_beam=logs3(cmd_ln_float32("-pl_beam"));
E_INFO("Phoneme look-ahead beam = %d\n",kb->pl_beam);
for(cisencnt=0;cisencnt==mdef->cd2cisen[cisencnt];cisencnt++)
;
kb->cache_ci_senscr=(int32**)ckd_calloc_2d(kb->pl_window,cisencnt,sizeof(int32));
kb->cache_best_list=(int32*)ckd_calloc(kb->pl_window,sizeof(int32));
kb->phn_heur_list=(int32*)ckd_calloc(mdef_n_ciphone (mdef),sizeof(int32));
if ((kb->feat = feat_array_alloc(kbcore_fcb(kbcore),S3_MAX_FRAMES)) == NULL)
E_FATAL("feat_array_alloc() failed\n");
kb->vithist = vithist_init(kbcore, kb->beam->word, cmd_ln_int32("-bghist"));
ptmr_init (&(kb->tm_sen));
ptmr_init (&(kb->tm_srch));
ptmr_init (&(kb->tm_ovrhd));
kb->tot_fr = 0;
kb->tot_sen_eval = 0.0;
kb->tot_gau_eval = 0.0;
kb->tot_hmm_eval = 0.0;
kb->tot_wd_exit = 0.0;
kb->hmm_hist_binsize = cmd_ln_int32("-hmmhistbinsize");
if(lmset)
n = ((kb->ugtreeMulti[0]->n_node) + (kb->fillertree[0]->n_node)) * kb->n_lextree;
else
n = ((kb->ugtree[0]->n_node) + (kb->fillertree[0]->n_node)) * kb->n_lextree;
n /= kb->hmm_hist_binsize;
kb->hmm_hist_bins = n+1;
kb->hmm_hist = (int32 *) ckd_calloc (n+1, sizeof(int32)); /* Really no need for +1 */
/* Open hypseg file if specified */
str = cmd_ln_str("-hypseg");
kb->matchsegfp = NULL;
if (str) {
#ifdef SPEC_CPU_WINDOWS
if ((kb->matchsegfp = fopen(str, "wt")) == NULL)
#else
if ((kb->matchsegfp = fopen(str, "w")) == NULL)
#endif
E_ERROR("fopen(%s,w) failed; use FWDXCT: from std logfile\n", str);
}
str = cmd_ln_str("-hyp");
kb->matchfp = NULL;
if (str) {
#ifdef SPEC_CPU_WINDOWS
if ((kb->matchfp = fopen(str, "wt")) == NULL)
#else
if ((kb->matchfp = fopen(str, "w")) == NULL)
#endif
E_ERROR("fopen(%s,w) failed; use FWDXCT: from std logfile\n", str);
}
}
dict2pid_t *
dict2pid_build(bin_mdef_t * mdef, dict_t * dict)
{
dict2pid_t *dict2pid;
s3ssid_t ***rdiph_rc;
bitvec_t *ldiph, *rdiph, *single;
int32 pronlen;
int32 b, l, r, w, p;
E_INFO("Building PID tables for dictionary\n");
assert(mdef);
assert(dict);
dict2pid = (dict2pid_t *) ckd_calloc(1, sizeof(dict2pid_t));
dict2pid->refcount = 1;
dict2pid->mdef = bin_mdef_retain(mdef);
dict2pid->dict = dict_retain(dict);
E_INFO("Allocating %d^3 * %d bytes (%d KiB) for word-initial triphones\n",
mdef->n_ciphone, sizeof(s3ssid_t),
mdef->n_ciphone * mdef->n_ciphone * mdef->n_ciphone * sizeof(s3ssid_t) / 1024);
dict2pid->ldiph_lc =
(s3ssid_t ***) ckd_calloc_3d(mdef->n_ciphone, mdef->n_ciphone,
mdef->n_ciphone, sizeof(s3ssid_t));
/* Only used internally to generate rssid */
rdiph_rc =
(s3ssid_t ***) ckd_calloc_3d(mdef->n_ciphone, mdef->n_ciphone,
mdef->n_ciphone, sizeof(s3ssid_t));
dict2pid->lrdiph_rc = (s3ssid_t ***) ckd_calloc_3d(mdef->n_ciphone,
mdef->n_ciphone,
mdef->n_ciphone,
sizeof
(s3ssid_t));
/* Actually could use memset for this, if BAD_S3SSID is guaranteed
* to be 65535... */
for (b = 0; b < mdef->n_ciphone; ++b) {
for (r = 0; r < mdef->n_ciphone; ++r) {
for (l = 0; l < mdef->n_ciphone; ++l) {
dict2pid->ldiph_lc[b][r][l] = BAD_S3SSID;
dict2pid->lrdiph_rc[b][l][r] = BAD_S3SSID;
rdiph_rc[b][l][r] = BAD_S3SSID;
}
}
}
/* Track which diphones / ciphones have been seen. */
ldiph = bitvec_alloc(mdef->n_ciphone * mdef->n_ciphone);
rdiph = bitvec_alloc(mdef->n_ciphone * mdef->n_ciphone);
single = bitvec_alloc(mdef->n_ciphone);
for (w = 0; w < dict_size(dict2pid->dict); w++) {
pronlen = dict_pronlen(dict, w);
if (pronlen >= 2) {
b = dict_first_phone(dict, w);
r = dict_second_phone(dict, w);
/* Populate ldiph_lc */
if (bitvec_is_clear(ldiph, b * mdef->n_ciphone + r)) {
/* Mark this diphone as done */
bitvec_set(ldiph, b * mdef->n_ciphone + r);
/* Record all possible ssids for b(?,r) */
for (l = 0; l < bin_mdef_n_ciphone(mdef); l++) {
p = bin_mdef_phone_id_nearest(mdef, (s3cipid_t) b,
(s3cipid_t) l, (s3cipid_t) r,
WORD_POSN_BEGIN);
dict2pid->ldiph_lc[b][r][l] = bin_mdef_pid2ssid(mdef, p);
}
}
/* Populate rdiph_rc */
l = dict_second_last_phone(dict, w);
b = dict_last_phone(dict, w);
if (bitvec_is_clear(rdiph, b * mdef->n_ciphone + l)) {
/* Mark this diphone as done */
bitvec_set(rdiph, b * mdef->n_ciphone + l);
for (r = 0; r < bin_mdef_n_ciphone(mdef); r++) {
p = bin_mdef_phone_id_nearest(mdef, (s3cipid_t) b,
(s3cipid_t) l, (s3cipid_t) r,
WORD_POSN_END);
rdiph_rc[b][l][r] = bin_mdef_pid2ssid(mdef, p);
}
}
}
else if (pronlen == 1) {
b = dict_pron(dict, w, 0);
E_DEBUG(1,("Building tables for single phone word %s phone %d = %s\n",
dict_wordstr(dict, w), b, bin_mdef_ciphone_str(mdef, b)));
/* Populate lrdiph_rc (and also ldiph_lc, rdiph_rc if needed) */
if (bitvec_is_clear(single, b)) {
populate_lrdiph(dict2pid, rdiph_rc, b);
bitvec_set(single, b);
}
}
}
bitvec_free(ldiph);
bitvec_free(rdiph);
bitvec_free(single);
/* Try to compress rdiph_rc into rdiph_rc_compressed */
compress_right_context_tree(dict2pid, rdiph_rc);
compress_left_right_context_tree(dict2pid);
ckd_free_3d(rdiph_rc);
dict2pid_report(dict2pid);
return dict2pid;
}
main (int32 argc, char *argv[])
{
kb_t kb;
kbcore_t *kbcore;
bitvec_t active;
int32 w;
cmd_ln_parse (arglist, argc, argv);
unlimit();
kbcore = kbcore_init (cmd_ln_float32("-logbase"),
cmd_ln_str("-feat"),
cmd_ln_str("-mdef"),
cmd_ln_str("-dict"),
cmd_ln_str("-fdict"),
cmd_ln_str("-compsep"),
cmd_ln_str("-lm"),
cmd_ln_str("-fillpen"),
cmd_ln_float32("-silprob"),
cmd_ln_float32("-fillprob"),
cmd_ln_float32("-lw"),
cmd_ln_float32("-wip"),
cmd_ln_str("-mean"),
cmd_ln_str("-var"),
cmd_ln_float32("-varfloor"),
cmd_ln_str("-senmgau"),
cmd_ln_str("-mixw"),
cmd_ln_float32("-mixwfloor"),
cmd_ln_str("-tmat"),
cmd_ln_float32("-tmatfloor"));
/* Here's the perfect candidate for inheritance */
kb.mdef = kbcore->mdef;
kb.dict = kbcore->dict;
kb.lm = kbcore->lm;
kb.fillpen = kbcore->fillpen;
kb.tmat = kbcore->tmat;
kb.dict2lmwid = kbcore->dict2lmwid;
if ((kb.am = acoustic_init (kbcore->fcb, kbcore->gau, kbcore->sen, cmd_ln_float32("-mgaubeam"),
S3_MAX_FRAMES)) == NULL) {
E_FATAL("Acoustic models initialization failed\n");
}
kb.beam = logs3 (cmd_ln_float64("-beam"));
kb.wordbeam = logs3 (cmd_ln_float64("-wordbeam"));
kb.wordmax = cmd_ln_int32("-wordmax");
/* Mark the active words and build lextree */
active = bitvec_alloc (dict_size (kb.dict));
bitvec_clear_all (active, dict_size(kb.dict));
for (w = 0; w < dict_size(kb.dict); w++) {
if (IS_LMWID(kb.dict2lmwid[w]) || dict_filler_word (kb.dict, w))
bitvec_set (active, w);
}
kb.lextree_root = lextree_build (kb.dict, kb.mdef, active, cmd_ln_int32("-flatdepth"));
kb.vithist = (glist_t *) ckd_calloc (S3_MAX_FRAMES+2, sizeof(glist_t));
kb.vithist++; /* Allow for dummy frame -1 for start word */
kb.lextree_active = NULL;
kb.wd_last_sf = (int32 *) ckd_calloc (dict_size(kb.dict), sizeof(int32));
kb.tm = (ptmr_t *) ckd_calloc (1, sizeof(ptmr_t));
kb.tm_search = (ptmr_t *) ckd_calloc (1, sizeof(ptmr_t));
ctl_process (cmd_ln_str("-ctl"), cmd_ln_int32("-ctloffset"), cmd_ln_int32("-ctlcount"),
decode_utt, &kb);
exit(0);
}
int main(int argc, char *argv[])
{
/* setup */
char *path = "./data/ct/";
char *path2 = "/home/gyc/Sources/linux.doc/kernel/";
vector_char str;
vector_char_init(&str,100);
VECTOR(str)[0] = '\0';
SetupLexer();
dic_setup();
struct dictionary *dict = new_dictionary(10000);
graph_t lkn;
vector_int edges;
catch_function_call_dir(path, dict, &edges);
printf("capacity=%d,size=%d\n",dict_capacity(dict), dict_size(dict));
new_graph(&lkn, &edges, 0, GRAPH_DIRECTED);
struct dictionary *filedict = new_dictionary(4);
vector_funcP flist;
vector_funcP_init_(&flist, dict_size(dict));
get_function_filename(path2, dict, filedict, &flist);
printf("filedict: capacity=%d,size=%d\n",dict_capacity(filedict), dict_size(filedict));
/* reciprocal */
printf("reciprocal = %f \n", graph_reciprocal(&lkn));
vector_double res;
vector_double_init(&res, 0);
graph_betweenness(&lkn, &res, graph_vss_all(), GRAPH_DIRECTED);
printf("betweenness directed:"); print_vector_double(&(res),stdout);
vector_double_destroy(&res);
/* degree */
graph_degree(&lkn, &edges, graph_vss_all(), GRAPH_OUT, GRAPH_NO_LOOPS);
printf(">>>out, no loops");
int min, max, sum;
double ave;
graph_degree_minmax_avesum(&lkn, &min, &max, &ave, &sum, GRAPH_OUT, GRAPH_NO_LOOPS);
printf("minout=%d\nmaxout=%d\nsumout=%d\naveout=%f\n",min,max,sum,ave);
graph_degree_minmax_avesum(&lkn, &min, &max, &ave, &sum, GRAPH_IN, GRAPH_NO_LOOPS);
printf("minin=%d\nmaxin=%d\nsumin=%d\navein=%f\n",min,max,sum,ave);
/* fast community */
graph_reverse(&lkn);
vector_int v1;
vector_int_init(&v1,0);
int ncom = 0;
double modularity = graph_community_fastgreedy(&lkn, &v1, &ncom);
printf("modularity = %f,ncom = %d\n",modularity,ncom);
FILE *f = fopen("funccom.fc.xlsx","w");
fprintf(f, "comID\tname\n");
for (int i = 0; i < dict_size(dict);i++) {
fprintf(f, "%d\t", VECTOR(v1)[i]);
struct rb_node* e = dict_ele(dict, i);
dic_traceback_string(e, &str);
fprintf(f, "%s\n",VECTOR(str));
}
fclose(f);
f = fopen("comID.fc.xlsx","w");
output_com_filename(&flist, &v1, graph_vertices_count(&lkn), ncom, filedict, f);
fclose(f);
//print_vector_int(&v1, stdout);
print_communities(&lkn, &v1, "community.fc.xlsx", "comedge.fc.xlsx");
vector_funcP_destroy(&flist);
vector_int_destroy(&v1);
vector_char_destroy(&str);
vector_int_destroy(&edges);
graph_destroy(&lkn);
return 0;
}
int megahal_keywords(brain_t brain, const list_t *words, dict_t **keywords) {
dict_t *keywords_p;
uint_fast32_t i;
uint32_t size;
int ret;
WARN_IF(words == NULL);
WARN_IF(keywords == NULL);
*keywords = dict_alloc();
if (*keywords == NULL) return -ENOMEM;
keywords_p = *keywords;
ret = list_size(words, &size);
if (ret) return ret;
for (i = 0; i < size; i++) {
word_t word;
ret = list_get(words, i, &word);
if (ret) return ret;
ret = db_map_get(brain, MAP_SWAP, word, &word);
if (ret != OK && ret != -ENOTFOUND) return ret;
ret = db_list_contains(brain, LIST_BAN, word);
if (ret == OK) continue;
if (ret != -ENOTFOUND) return ret;
ret = db_list_contains(brain, LIST_AUX, word);
if (ret == OK) continue;
if (ret != -ENOTFOUND) return ret;
ret = db_model_contains(brain, word);
if (ret == -ENOTFOUND) continue;
if (ret != OK) return ret;
ret = add_keyword(keywords_p, word);
if (ret) return ret;
}
ret = dict_size(keywords_p, &size);
if (ret) return ret;
if (size == 0)
return OK;
ret = list_size(words, &size);
if (ret) return ret;
for (i = 0; i < size; i++) {
word_t word;
ret = list_get(words, i, &word);
if (ret) return ret;
ret = db_map_get(brain, MAP_SWAP, word, &word);
if (ret != OK && ret != -ENOTFOUND) return ret;
ret = db_list_contains(brain, LIST_AUX, word);
if (ret == -ENOTFOUND) continue;
if (ret != OK) return ret;
ret = db_model_contains(brain, word);
if (ret == -ENOTFOUND) continue;
if (ret != OK) return ret;
ret = add_keyword(keywords_p, word);
if (ret) return ret;
}
return OK;
}
/*
* Reads function-like macro definition.
*/
static void read_funclike_define(CppContext *ctx, String *name) {
Dict *param = make_string_dict();
bool is_varg = read_funclike_define_args(ctx, param);
List *body = read_funclike_define_body(ctx, param);
store_macro(ctx, name, make_func_macro(body, dict_size(param), is_varg));
}
/* Update kb w/ new dictionary and new LM.
* assumes: single-LM kbcore (before & after)
* requires: updating kbcore
* Lucian Galescu, 08/11/2005
*/
void kb_update_lm(kb_t *kb, char *dictfile, char *lmfile)
{
kbcore_t *kbcore;
mdef_t *mdef;
dict_t *dict;
dict2pid_t *d2p;
lm_t *lm;
s3cipid_t ci;
s3wid_t w;
int32 i, n, n_lc;
wordprob_t *wp;
s3cipid_t *lc;
bitvec_t lc_active;
/*** clean up ***/
vithist_t *vithist = kb->vithist;
if (kb->fillertree)
ckd_free ((void *)kb->fillertree);
if (kb->hmm_hist)
ckd_free ((void *)kb->hmm_hist);
/* vithist */
if (vithist) {
ckd_free ((void *) vithist->entry);
ckd_free ((void *) vithist->frame_start);
ckd_free ((void *) vithist->bestscore);
ckd_free ((void *) vithist->bestvh);
ckd_free ((void *) vithist->lms2vh_root);
ckd_free ((void *) kb->vithist);
}
/*** re-initialize ***/
kb->kbcore = kbcore_update_lm(kb->kbcore,
dictfile,
cmd_ln_str("-fdict"),
"", /* Hack!! Hardwired constant for -compsep argument */
lmfile,
cmd_ln_str("-fillpen"),
cmd_ln_float32("-silprob"),
cmd_ln_float32("-fillprob"),
cmd_ln_float32("-lw"),
cmd_ln_float32("-wip"),
cmd_ln_float32("-uw"));
if(kb->kbcore==NULL){
E_FATAL("Updating kbcore failed\n");
}
kbcore = kb->kbcore;
mdef = kbcore_mdef(kbcore);
dict = kbcore_dict(kbcore);
lm = kbcore_lm(kbcore);
d2p = kbcore_dict2pid(kbcore);
if (NOT_S3WID(dict_startwid(dict)) || NOT_S3WID(dict_finishwid(dict)))
E_FATAL("%s or %s not in dictionary\n", S3_START_WORD, S3_FINISH_WORD);
if(lm){
if (NOT_S3LMWID(lm_startwid(lm)) || NOT_S3LMWID(lm_finishwid(lm)))
E_FATAL("%s or %s not in LM\n", S3_START_WORD, S3_FINISH_WORD);
}
/*
* Unlink <s> and </s> between dictionary and LM, to prevent their
* recognition. They are merely dummy words (anchors) at the beginning
* and end of each utterance.
*/
if(lm){
lm_lmwid2dictwid(lm, lm_startwid(lm)) = BAD_S3WID;
lm_lmwid2dictwid(lm, lm_finishwid(lm)) = BAD_S3WID;
for (w = dict_startwid(dict); IS_S3WID(w); w = dict_nextalt(dict, w))
kbcore->dict2lmwid[w] = BAD_S3LMWID;
for (w = dict_finishwid(dict); IS_S3WID(w); w = dict_nextalt(dict, w))
kbcore->dict2lmwid[w] = BAD_S3LMWID;
}
/* Build set of all possible left contexts */
lc = (s3cipid_t *) ckd_calloc (mdef_n_ciphone(mdef) + 1, sizeof(s3cipid_t));
lc_active = bitvec_alloc (mdef_n_ciphone (mdef));
for (w = 0; w < dict_size (dict); w++) {
ci = dict_pron (dict, w, dict_pronlen(dict, w) - 1);
if (! mdef_is_fillerphone (mdef, (int)ci))
bitvec_set (lc_active, ci);
}
ci = mdef_silphone(mdef);
bitvec_set (lc_active, ci);
for (ci = 0, n_lc = 0; ci < mdef_n_ciphone(mdef); ci++) {
if (bitvec_is_set (lc_active, ci))
lc[n_lc++] = ci;
}
lc[n_lc] = BAD_S3CIPID;
E_INFO("Building lextrees\n");
/* Get the number of lexical tree*/
kb->n_lextree = cmd_ln_int32 ("-Nlextree");
if (kb->n_lextree < 1) {
E_ERROR("No. of ugtrees specified: %d; will instantiate 1 ugtree\n",
kb->n_lextree);
kb->n_lextree = 1;
}
/* ARCHAN: This code was rearranged in s3.4 implementation of dynamic LM */
/* Build active word list */
wp = (wordprob_t *) ckd_calloc (dict_size(dict), sizeof(wordprob_t));
if (lm) {
E_INFO("Creating Unigram Table\n");
n=0;
n = lm_ug_wordprob (lm, dict, MAX_NEG_INT32, wp);
E_INFO("Size of word table after unigram + words in class: %d\n",n);
if (n < 1)
E_FATAL("%d active words\n", n);
n = wid_wordprob2alt (dict, wp, n); /* Add alternative pronunciations */
/* Retain or remove unigram probs from lextree, depending on option */
if (cmd_ln_int32("-treeugprob") == 0) {
for (i = 0; i < n; i++)
wp[i].prob = -1; /* Flatten all initial probabilities */
}
/* Create the desired no. of unigram lextrees */
kb->ugtree = (lextree_t **) ckd_calloc (kb->n_lextree, sizeof(lextree_t *));
for (i = 0; i < kb->n_lextree; i++) {
kb->ugtree[i] = lextree_build (kbcore, wp, n, lc);
lextree_type (kb->ugtree[i]) = 0;
}
E_INFO("Lextrees(%d), %d nodes(ug)\n",
kb->n_lextree, lextree_n_node(kb->ugtree[0]));
}
/* Create filler lextrees */
/* ARCHAN : only one filler tree is supposed to be build even for dynamic LMs */
n = 0;
for (i = dict_filler_start(dict); i <= dict_filler_end(dict); i++) {
if (dict_filler_word(dict, i)) {
wp[n].wid = i;
wp[n].prob = fillpen (kbcore->fillpen, i);
n++;
}
}
kb->fillertree = (lextree_t **)ckd_calloc(kb->n_lextree,sizeof(lextree_t*));
for (i = 0; i < kb->n_lextree; i++) {
kb->fillertree[i] = lextree_build (kbcore, wp, n, NULL);
lextree_type (kb->fillertree[i]) = -1;
}
ckd_free ((void *) wp);
ckd_free ((void *) lc);
bitvec_free (lc_active);
E_INFO("Lextrees(%d), %d nodes(filler)\n",
kb->n_lextree,
lextree_n_node(kb->fillertree[0]));
if (cmd_ln_int32("-lextreedump")) {
for (i = 0; i < kb->n_lextree; i++) {
fprintf (stderr, "UGTREE %d\n", i);
lextree_dump (kb->ugtree[i], dict, stderr);
}
for (i = 0; i < kb->n_lextree; i++) {
fprintf (stderr, "FILLERTREE %d\n", i);
lextree_dump (kb->fillertree[i], dict, stderr);
}
fflush (stderr);
}
kb->ascr = ascr_init (mgau_n_mgau(kbcore_mgau(kbcore)),
kbcore->dict2pid->n_comstate);
kb->vithist = vithist_init(kbcore, kb->beam->word, cmd_ln_int32("-bghist"));
kb->hmm_hist_binsize = cmd_ln_int32("-hmmhistbinsize");
n = ((kb->ugtree[0]->n_node) + (kb->fillertree[0]->n_node)) * kb->n_lextree;
n /= kb->hmm_hist_binsize;
kb->hmm_hist_bins = n+1;
kb->hmm_hist = (int32 *) ckd_calloc (n+1, sizeof(int32)); /* Really no need for +1 */
}
// App Message API
static void in_received_handler(DictionaryIterator *iter, void *context){
if (debugMode)
APP_LOG(APP_LOG_LEVEL_INFO, "Message received!");
// Get the first pair
Tuple *t = dict_read_first(iter);
// Long lived buffers
static char roadName_buffer[60];
static char roadCode_buffer[6];
static char busService_buffer[5];
static int loadC, loadN;
static char arrC_data_buffer[10];
static char arrN_data_buffer[10];
static bool pgLoad = false, pgMaxLoad = false;
static int8_t wabC, wabN;
// Process all pairs present
while(t != NULL) {
// Process this pair's key
switch (t->key) {
case MESSAGE_DATA_EVENT:
APP_LOG(APP_LOG_LEVEL_INFO, "Data received for Dictionary %hu", t->value->int16);
APP_LOG(APP_LOG_LEVEL_INFO, "Dictionary Size: %lu", dict_size(iter));
break;
case MESSAGE_ROAD_NAME:
snprintf(roadName_buffer, sizeof(roadName_buffer), "%s", t->value->cstring);
text_layer_set_text(textlayer_busstop_name, roadName_buffer);
break;
case MESSAGE_ROAD_CODE:
snprintf(roadCode_buffer, sizeof(roadCode_buffer), "%s", t->value->cstring);
text_layer_set_text(textlayer_busstop_code, roadCode_buffer);
break;
case MESSAGE_BUS_SERVICE:
snprintf(busService_buffer, sizeof(busService_buffer), "%s", t->value->cstring);
text_layer_set_text(textlayer_bus_no, busService_buffer);
break;
case ESTIMATE_ARR_CURRENT_DATA:
snprintf(arrC_data_buffer, sizeof(arrC_data_buffer), "%s", t->value->cstring);
text_layer_set_text(textlayer_arrive_now, arrC_data_buffer);
break;
case ESTIMATE_ARR_NEXT_DATA:
snprintf(arrN_data_buffer, sizeof(arrN_data_buffer), "%s", t->value->cstring);
text_layer_set_text(textlayer_arrive_next, arrN_data_buffer);
break;
case MESSAGE_CURRENT_FAV:
current = t->value->int16;
pgLoad = true;
break;
case MESSAGE_MAX_FAV:
max = t->value->int16;
pgMaxLoad = true;
break;
case ESTIMATE_LOAD_CURRENT_DATA:
loadC = t->value->int16;
updateLoad(loadC, 1);
break;
case ESTIMATE_LOAD_NEXT_DATA:
loadN = t->value->int16;
updateLoad(loadN, 2);
break;
//s_res_bus_wheelchair_accessible
//bitmap_wab_now
//bitmap_wab_next
case MESSAGE_WAB_CURRENT:
wabC = t->value->int8;
if (debugMode)
APP_LOG(APP_LOG_LEVEL_INFO, "Current Has WAB: %i", wabC);
if (wabC == 1)
bitmap_layer_set_bitmap(bitmap_wab_now, bus_wheelchair_accessible);
break;
case MESSAGE_WAB_NEXT:
wabN = t->value->int8;
if (debugMode)
APP_LOG(APP_LOG_LEVEL_INFO, "Next Has WAB: %i", wabN);
if (wabN == 1)
bitmap_layer_set_bitmap(bitmap_wab_next, bus_wheelchair_accessible);
break;
//No Favourites? Tell them too
case ERROR_NO_DATA:
text_layer_set_text(textlayer_debug, "No Favourites");
break;
}
//Handle paging
static char paging_buffer[10];
if (pgLoad && pgMaxLoad){
snprintf(paging_buffer, sizeof(paging_buffer), "%d/%d", current, max);
text_layer_set_text(textlayer_pages, paging_buffer);
}
// Get next pair, if any
t = dict_read_next(iter);
}
}
void kb_init (kb_t *kb)
{
kbcore_t *kbcore;
mdef_t *mdef;
dict_t *dict;
dict2pid_t *d2p;
lm_t *lm;
s3cipid_t sil, ci;
s3wid_t w;
int32 i, n, n_lc;
wordprob_t *wp;
s3cipid_t *lc;
bitvec_t lc_active;
char *str;
/* Initialize the kb structure to zero, just in case */
memset(kb, 0, sizeof(*kb));
kb->kbcore = kbcore_init (cmd_ln_float32 ("-logbase"),
"1s_c_d_dd", /* Hack!! Hardwired constant
for -feat argument */
cmd_ln_str("-cmn"),
cmd_ln_str("-varnorm"),
cmd_ln_str("-agc"),
cmd_ln_str("-mdef"),
cmd_ln_str("-dict"),
cmd_ln_str("-fdict"),
"", /* Hack!! Hardwired constant
for -compsep argument */
cmd_ln_str("-lm"),
cmd_ln_str("-fillpen"),
cmd_ln_float32("-silprob"),
cmd_ln_float32("-fillprob"),
cmd_ln_float32("-lw"),
cmd_ln_float32("-wip"),
cmd_ln_float32("-uw"),
cmd_ln_str("-mean"),
cmd_ln_str("-var"),
cmd_ln_float32("-varfloor"),
cmd_ln_str("-mixw"),
cmd_ln_float32("-mixwfloor"),
cmd_ln_str("-subvq"),
cmd_ln_str("-tmat"),
cmd_ln_float32("-tmatfloor"));
kbcore = kb->kbcore;
mdef = kbcore_mdef(kbcore);
dict = kbcore_dict(kbcore);
lm = kbcore_lm(kbcore);
d2p = kbcore_dict2pid(kbcore);
if (NOT_S3WID(dict_startwid(dict)) || NOT_S3WID(dict_finishwid(dict)))
E_FATAL("%s or %s not in dictionary\n", S3_START_WORD, S3_FINISH_WORD);
if (NOT_S3LMWID(lm_startwid(lm)) || NOT_S3LMWID(lm_finishwid(lm)))
E_FATAL("%s or %s not in LM\n", S3_START_WORD, S3_FINISH_WORD);
/* Check that HMM topology restrictions are not violated */
if (tmat_chk_1skip (kbcore->tmat) < 0)
E_FATAL("Tmat contains arcs skipping more than 1 state\n");
/*
* Unlink <s> and </s> between dictionary and LM, to prevent their
* recognition. They are merely dummy words (anchors) at the beginning
* and end of each utterance.
*/
lm_lmwid2dictwid(lm, lm_startwid(lm)) = BAD_S3WID;
lm_lmwid2dictwid(lm, lm_finishwid(lm)) = BAD_S3WID;
for (w = dict_startwid(dict); IS_S3WID(w); w = dict_nextalt(dict, w))
kbcore->dict2lmwid[w] = BAD_S3LMWID;
for (w = dict_finishwid(dict); IS_S3WID(w); w = dict_nextalt(dict, w))
kbcore->dict2lmwid[w] = BAD_S3LMWID;
sil = mdef_silphone (kbcore_mdef (kbcore));
if (NOT_S3CIPID(sil))
E_FATAL("Silence phone '%s' not in mdef\n", S3_SILENCE_CIPHONE);
E_INFO("Building lextrees\n");
kb->sen_active = (int32 *) ckd_calloc (mdef_n_sen(mdef), sizeof(int32));
kb->ssid_active = (int32 *) ckd_calloc (mdef_n_sseq(mdef), sizeof(int32));
kb->comssid_active = (int32 *) ckd_calloc (dict2pid_n_comsseq(d2p),
sizeof(int32));
/* Build active word list */
wp = (wordprob_t *) ckd_calloc (dict_size(dict), sizeof(wordprob_t));
n = lm_ug_wordprob (lm, MAX_NEG_INT32, wp);
if (n < 1)
E_FATAL("%d active words\n", n);
n = wid_wordprob2alt (dict, wp, n); /* Add alternative pronunciations */
/* Retain or remove unigram probs from lextree, depending on option */
if (cmd_ln_int32("-treeugprob") == 0) {
for (i = 0; i < n; i++)
wp[i].prob = -1; /* Flatten all initial probabilities */
}
/* Build set of all possible left contexts */
lc = (s3cipid_t *) ckd_calloc (mdef_n_ciphone(mdef) + 1, sizeof(s3cipid_t));
lc_active = bitvec_alloc (mdef_n_ciphone (mdef));
for (w = 0; w < dict_size (dict); w++) {
ci = dict_pron (dict, w, dict_pronlen(dict, w) - 1);
if (! mdef_is_fillerphone (mdef, (int)ci))
bitvec_set (lc_active, ci);
}
ci = mdef_silphone(mdef);
bitvec_set (lc_active, ci);
for (ci = 0, n_lc = 0; ci < mdef_n_ciphone(mdef); ci++) {
if (bitvec_is_set (lc_active, ci))
lc[n_lc++] = ci;
}
lc[n_lc] = BAD_S3CIPID;
/* Create the desired no. of unigram lextrees */
kb->n_lextree = cmd_ln_int32 ("-Nlextree");
if (kb->n_lextree < 1) {
E_ERROR("No. of ugtrees specified: %d; will instantiate 1 ugtree\n",
kb->n_lextree);
kb->n_lextree = 1;
}
kb->ugtree = (lextree_t **) ckd_calloc (kb->n_lextree, sizeof(lextree_t *));
for (i = 0; i < kb->n_lextree; i++) {
kb->ugtree[i] = lextree_build (kbcore, wp, n, lc);
lextree_type (kb->ugtree[i]) = 0;
}
bitvec_free (lc_active);
ckd_free ((void *) lc);
/* Create filler lextrees */
n = 0;
for (i = dict_filler_start(dict); i <= dict_filler_end(dict); i++) {
if (dict_filler_word(dict, i)) {
wp[n].wid = i;
wp[n].prob = fillpen (kbcore->fillpen, i);
n++;
}
}
kb->fillertree = (lextree_t **)ckd_calloc(kb->n_lextree,sizeof(lextree_t*));
for (i = 0; i < kb->n_lextree; i++) {
kb->fillertree[i] = lextree_build (kbcore, wp, n, NULL);
lextree_type (kb->fillertree[i]) = -1;
}
ckd_free ((void *) wp);
E_INFO("Lextrees(%d), %d nodes(ug), %d nodes(filler)\n",
kb->n_lextree, lextree_n_node(kb->ugtree[0]),
lextree_n_node(kb->fillertree[0]));
if (cmd_ln_int32("-lextreedump")) {
for (i = 0; i < kb->n_lextree; i++) {
fprintf (stderr, "UGTREE %d\n", i);
lextree_dump (kb->ugtree[i], dict, stderr);
}
for (i = 0; i < kb->n_lextree; i++) {
fprintf (stderr, "FILLERTREE %d\n", i);
lextree_dump (kb->fillertree[i], dict, stderr);
}
fflush (stderr);
}
kb->ascr = ascr_init (mgau_n_mgau(kbcore_mgau(kbcore)),
kbcore->dict2pid->n_comstate);
kb->beam = beam_init (cmd_ln_float64("-subvqbeam"),
cmd_ln_float64("-beam"),
cmd_ln_float64("-pbeam"),
cmd_ln_float64("-wbeam"));
E_INFO("Beam= %d, PBeam= %d, WBeam= %d, SVQBeam= %d\n",
kb->beam->hmm, kb->beam->ptrans, kb->beam->word, kb->beam->subvq);
if ((kb->feat = feat_array_alloc(kbcore_fcb(kbcore),S3_MAX_FRAMES)) == NULL)
E_FATAL("feat_array_alloc() failed\n");
kb->vithist = vithist_init(kbcore, kb->beam->word, cmd_ln_int32("-bghist"));
ptmr_init (&(kb->tm_sen));
ptmr_init (&(kb->tm_srch));
kb->tot_fr = 0;
kb->tot_sen_eval = 0.0;
kb->tot_gau_eval = 0.0;
kb->tot_hmm_eval = 0.0;
kb->tot_wd_exit = 0.0;
kb->hmm_hist_binsize = cmd_ln_int32("-hmmhistbinsize");
n = ((kb->ugtree[0]->n_node) + (kb->fillertree[0]->n_node)) * kb->n_lextree;
n /= kb->hmm_hist_binsize;
kb->hmm_hist_bins = n+1;
kb->hmm_hist = (int32 *) ckd_calloc (n+1, sizeof(int32)); /* Really no need for +1 */
/* Open hypseg file if specified */
str = cmd_ln_str("-hypseg");
kb->matchsegfp = NULL;
if (str) {
#ifdef WIN32
if ((kb->matchsegfp = fopen(str, "wt")) == NULL)
#else
if ((kb->matchsegfp = fopen(str, "w")) == NULL)
#endif
E_ERROR("fopen(%s,w) failed; use FWDXCT: from std logfile\n", str);
}
}
lmset_t *
lmset_read_ctl(const char *ctlfile,
dict_t * dict,
float64 lw, float64 wip, float64 uw,
const char *lmdumpdir, logmath_t *logmath)
{
FILE *ctlfp;
FILE *tmp;
char lmfile[4096], lmname[4096], str[4096];
lmclass_set_t *lmclass_set;
lmclass_t **lmclass, *cl;
int32 n_lmclass, n_lmclass_used;
int32 i;
lm_t *lm;
lmset_t *lms = NULL;
tmp = NULL;
E_INFO("Reading LM control file '%s'\n", ctlfile);
if ((ctlfp = fopen(ctlfile, "r")) == NULL) {
E_ERROR_SYSTEM("Failed to open LM control file");
return NULL;
}
lmclass_set = lmclass_newset();
lms = (lmset_t *) ckd_calloc(1, sizeof(lmset_t));
lms->n_lm = 0;
lms->n_alloc_lm = 0;
if (fscanf(ctlfp, "%s", str) == 1) {
if (strcmp(str, "{") == 0) {
/* Load LMclass files */
while ((fscanf(ctlfp, "%s", str) == 1)
&& (strcmp(str, "}") != 0))
lmclass_set = lmclass_loadfile(lmclass_set, str, logmath);
if (strcmp(str, "}") != 0)
E_FATAL("Unexpected EOF(%s)\n", ctlfile);
if (fscanf(ctlfp, "%s", str) != 1)
str[0] = '\0';
}
}
else
str[0] = '\0';
/* Fill in dictionary word id information for each LMclass word */
for (cl = lmclass_firstclass(lmclass_set);
lmclass_isclass(cl); cl = lmclass_nextclass(lmclass_set, cl)) {
/*
For every words in the class, set the dictwid correctly
The following piece of code replace s2's kb_init_lmclass_dictwid (cl);
doesn't do any checking even the id is a bad dict id.
This only sets the information in the lmclass_set, but not
lm-2-dict or dict-2-lm map. In Sphinx 3, they are done in
wid_dict_lm_map in wid.c.
*/
lmclass_word_t *w;
int32 wid;
for (w = lmclass_firstword(cl); lmclass_isword(w);
w = lmclass_nextword(cl, w)) {
wid = dict_wordid(dict, lmclass_getword(w));
#if 0
E_INFO("In class %s, Word %s, wid %d\n", cl->name,
lmclass_getword(w), wid);
#endif
lmclass_set_dictwid(w, wid);
}
}
/* At this point if str[0] != '\0', we have an LM filename */
n_lmclass = lmclass_get_nclass(lmclass_set);
lmclass = (lmclass_t **) ckd_calloc(n_lmclass, sizeof(lmclass_t *));
E_INFO("Number of LM class specified %d in file %s\n", n_lmclass,
ctlfile);
/* Read in one LM at a time */
while (str[0] != '\0') {
strcpy(lmfile, str);
if (fscanf(ctlfp, "%s", lmname) != 1)
E_FATAL("LMname missing after LMFileName '%s'\n", lmfile);
n_lmclass_used = 0;
if (fscanf(ctlfp, "%s", str) == 1) {
if (strcmp(str, "{") == 0) {
while ((fscanf(ctlfp, "%s", str) == 1) &&
(strcmp(str, "}") != 0)) {
if (n_lmclass_used >= n_lmclass) {
E_FATAL("Too many LM classes specified for '%s'\n",
lmfile);
}
lmclass[n_lmclass_used] =
lmclass_get_lmclass(lmclass_set, str);
if (!(lmclass_isclass(lmclass[n_lmclass_used])))
E_FATAL("LM class '%s' not found\n", str);
n_lmclass_used++;
}
if (strcmp(str, "}") != 0)
E_FATAL("Unexpected EOF(%s)\n", ctlfile);
if (fscanf(ctlfp, "%s", str) != 1)
str[0] = '\0';
}
}
else
str[0] = '\0';
lm = (lm_t *) lm_read_advance(lmfile, lmname, lw, wip, uw,
dict_size(dict), NULL, 1, logmath, FALSE, FALSE);
if (n_lmclass_used > 0) {
E_INFO("Did I enter here?\n");
lm_build_lmclass_info(lm, lw, uw, wip, n_lmclass_used,
lmclass);
}
if (lms->n_lm == lms->n_alloc_lm) {
lms->lmarray =
(lm_t **) ckd_realloc(lms->lmarray,
(lms->n_alloc_lm +
LM_ALLOC_BLOCK) * sizeof(lm_t *));
lms->n_alloc_lm += LM_ALLOC_BLOCK;
}
lms->lmarray[lms->n_lm] = lm;
lms->n_lm += 1;
E_INFO("%d %d\n", lms->n_alloc_lm, lms->n_lm);
}
assert(lms);
assert(lms->lmarray);
E_INFO("No. of LM set allocated %d, no. of LM %d \n", lms->n_alloc_lm,
lms->n_lm);
if (dict != NULL) {
for (i = 0; i < lms->n_lm; i++) {
assert(lms->lmarray[i]);
assert(dict);
if ((lms->lmarray[i]->dict2lmwid =
wid_dict_lm_map(dict, lms->lmarray[i], lw)) == NULL)
E_FATAL
("Dict/LM word-id mapping failed for LM index %d, named %s\n",
i, lmset_idx_to_name(lms, i));
}
}
else {
E_FATAL
("Dict is specified to be NULL (dict_init is not called before lmset_read_lm?), dict2lmwid is not built inside lmset_read_lm\n");
}
ckd_free(lmclass_set);
ckd_free(lmclass);
fclose(ctlfp);
return lms;
}
/* RAH 4.16.01 This code has several leaks that must be fixed */
dict2pid_t *dict2pid_build (mdef_t *mdef, dict_t *dict)
{
dict2pid_t *dict2pid;
s3ssid_t *internal, **ldiph, **rdiph, *single;
int32 pronlen;
hash_table_t *hs, *hp;
glist_t g;
gnode_t *gn;
s3senid_t *sen;
hash_entry_t *he;
int32 *cslen;
int32 i, j, b, l, r, w, n, p;
E_INFO("Building PID tables for dictionary\n");
dict2pid = (dict2pid_t *) ckd_calloc (1, sizeof(dict2pid_t));
dict2pid->internal = (s3ssid_t **) ckd_calloc (dict_size(dict), sizeof(s3ssid_t *));
dict2pid->ldiph_lc = (s3ssid_t ***) ckd_calloc_3d (mdef->n_ciphone,
mdef->n_ciphone,
mdef->n_ciphone,
sizeof(s3ssid_t));
dict2pid->single_lc = (s3ssid_t **) ckd_calloc_2d (mdef->n_ciphone,
mdef->n_ciphone,
sizeof(s3ssid_t));
dict2pid->n_comstate = 0;
dict2pid->n_comsseq = 0;
hs = hash_new (mdef->n_ciphone * mdef->n_ciphone * mdef->n_emit_state, HASH_CASE_YES);
hp = hash_new (mdef->n_ciphone * mdef->n_ciphone, HASH_CASE_YES);
for (w = 0, n = 0; w < dict_size(dict); w++) {
pronlen = dict_pronlen(dict, w);
if (pronlen < 0)
E_FATAL("Pronunciation-length(%s)= %d\n", dict_wordstr(dict, w), pronlen);
n += pronlen;
}
internal = (s3ssid_t *) ckd_calloc (n, sizeof(s3ssid_t));
/* Temporary */
ldiph = (s3ssid_t **) ckd_calloc_2d (mdef->n_ciphone, mdef->n_ciphone, sizeof(s3ssid_t));
rdiph = (s3ssid_t **) ckd_calloc_2d (mdef->n_ciphone, mdef->n_ciphone, sizeof(s3ssid_t));
single = (s3ssid_t *) ckd_calloc (mdef->n_ciphone, sizeof(s3ssid_t));
for (b = 0; b < mdef->n_ciphone; b++) {
for (l = 0; l < mdef->n_ciphone; l++) {
for (r = 0; r < mdef->n_ciphone; r++)
dict2pid->ldiph_lc[b][r][l] = BAD_S3SSID;
dict2pid->single_lc[b][l] = BAD_S3SSID;
ldiph[b][l] = BAD_S3SSID;
rdiph[b][l] = BAD_S3SSID;
}
single[b] = BAD_S3SSID;
}
for (w = 0; w < dict_size(dict); w++) {
dict2pid->internal[w] = internal;
pronlen = dict_pronlen(dict,w);
if (pronlen >= 2) {
b = dict_pron(dict, w, 0);
r = dict_pron(dict, w, 1);
if (NOT_S3SSID(ldiph[b][r])) {
g = ldiph_comsseq(mdef, b, r);
ldiph[b][r] = ssidlist2comsseq (g, mdef, dict2pid, hs, hp);
glist_free (g);
for (l = 0; l < mdef_n_ciphone(mdef); l++) {
p = mdef_phone_id_nearest (mdef, (s3cipid_t)b, (s3cipid_t)l, (s3cipid_t)r, WORD_POSN_BEGIN);
dict2pid->ldiph_lc[b][r][l] = mdef_pid2ssid(mdef, p);
}
}
internal[0] = ldiph[b][r];
for (i = 1; i < pronlen-1; i++) {
l = b;
b = r;
r = dict_pron(dict, w, i+1);
p = mdef_phone_id_nearest(mdef, (s3cipid_t)b, (s3cipid_t)l, (s3cipid_t)r, WORD_POSN_INTERNAL);
internal[i] = mdef_pid2ssid(mdef, p);
}
l = b;
b = r;
if (NOT_S3SSID(rdiph[b][l])) {
g = rdiph_comsseq(mdef, b, l);
rdiph[b][l] = ssidlist2comsseq (g, mdef, dict2pid, hs, hp);
glist_free (g);
}
internal[pronlen-1] = rdiph[b][l];
} else if (pronlen == 1) {
b = dict_pron(dict, w, 0);
if (NOT_S3SSID(single[b])) {
g = single_comsseq(mdef, b);
single[b] = ssidlist2comsseq (g, mdef, dict2pid, hs, hp);
glist_free (g);
for (l = 0; l < mdef_n_ciphone(mdef); l++) {
g = single_lc_comsseq(mdef, b, l);
dict2pid->single_lc[b][l] = ssidlist2comsseq (g, mdef, dict2pid, hs, hp);
glist_free (g);
}
}
internal[0] = single[b];
}
internal += pronlen;
}
ckd_free_2d ((void **) ldiph);
ckd_free_2d ((void **) rdiph);
ckd_free ((void *) single);
/* Allocate space for composite state table */
cslen = (int32 *) ckd_calloc (dict2pid->n_comstate, sizeof(int32));
g = hash_tolist(hs, &n);
assert (n == dict2pid->n_comstate);
n = 0;
for (gn = g; gn; gn = gnode_next(gn)) {
he = (hash_entry_t *) gnode_ptr (gn);
sen = (s3senid_t *) hash_entry_key(he);
for (i = 0; IS_S3SENID(sen[i]); i++);
cslen[hash_entry_val(he)] = i+1; /* +1 for terminating sentinel */
n += (i+1);
}
dict2pid->comstate = (s3senid_t **) ckd_calloc (dict2pid->n_comstate, sizeof(s3senid_t *));
sen = (s3senid_t *) ckd_calloc (n, sizeof(s3senid_t));
for (i = 0; i < dict2pid->n_comstate; i++) {
dict2pid->comstate[i] = sen;
sen += cslen[i];
}
/* Build composite state table from hash table hs */
for (gn = g; gn; gn = gnode_next(gn)) {
he = (hash_entry_t *) gnode_ptr (gn);
sen = (s3senid_t *) hash_entry_key(he);
i = hash_entry_val(he);
for (j = 0; j < cslen[i]; j++)
dict2pid->comstate[i][j] = sen[j];
assert (sen[j-1] == BAD_S3SENID);
ckd_free ((void *)sen);
}
ckd_free (cslen);
glist_free (g);
hash_free (hs);
/* Allocate space for composite sseq table */
dict2pid->comsseq = (s3senid_t **) ckd_calloc (dict2pid->n_comsseq, sizeof(s3senid_t *));
g = hash_tolist (hp, &n);
assert (n == dict2pid->n_comsseq);
/* Build composite sseq table */
for (gn = g; gn; gn = gnode_next(gn)) {
he = (hash_entry_t *) gnode_ptr (gn);
i = hash_entry_val(he);
dict2pid->comsseq[i] = (s3senid_t *) hash_entry_key(he);
}
glist_free (g);
hash_free (hp);
/* Weight for each composite state */
dict2pid->comwt = (int32 *) ckd_calloc (dict2pid->n_comstate, sizeof(int32));
for (i = 0; i < dict2pid->n_comstate; i++) {
sen = dict2pid->comstate[i];
for (j = 0; IS_S3SENID(sen[j]); j++);
#if 0
/* if comstate i has N states, its weight= (1/N^2) (Major Hack!!) */
dict2pid->comwt[i] = - (logs3 ((float64)j) << 1);
#else
/* if comstate i has N states, its weight= 1/N */
dict2pid->comwt[i] = - logs3 ((float64)j);
#endif
}
E_INFO("%d composite states; %d composite sseq\n",
dict2pid->n_comstate, dict2pid->n_comsseq);
return dict2pid;
}