int main(int argc, char *argv[]) { logmath_t *lmath; ngram_model_t *model; /* Initialize a logmath object to pass to ngram_read */ lmath = logmath_init(1.0001, 0, 0); /* Read a language model */ model = ngram_model_read(NULL, LMDIR "/100.arpa.DMP", NGRAM_BIN, lmath); TEST_ASSERT(model); ngram_model_casefold(model, NGRAM_UPPER); TEST_EQUAL(0, strcmp("</s>", ngram_word(model, 5))); TEST_EQUAL(0, strcmp("BE", ngram_word(model, 42))); TEST_EQUAL(0, strcmp("FLOORED", ngram_word(model, 130))); TEST_EQUAL(0, strcmp("ZERO", ngram_word(model, 398))); TEST_EQUAL(0, strcmp("~", ngram_word(model, 399))); ngram_model_casefold(model, NGRAM_LOWER); TEST_EQUAL(0, strcmp("</s>", ngram_word(model, 5))); TEST_EQUAL(0, strcmp("be", ngram_word(model, 42))); TEST_EQUAL(0, strcmp("floored", ngram_word(model, 130))); TEST_EQUAL(0, strcmp("zero", ngram_word(model, 398))); TEST_EQUAL(0, strcmp("~", ngram_word(model, 399))); ngram_model_free(model); logmath_free(lmath); return 0; }
void vithist_dump(vithist_t * vh, int32 frm, ngram_model_t *lm, s3dict_t *dict, FILE * fp) { int32 i, j; vithist_entry_t *ve; int32 sf, ef; if (frm >= 0) { sf = frm; ef = frm; fprintf(fp, "VITHIST frame %d #entries %d\n", frm, vh->frame_start[sf + 1] - vh->frame_start[sf]); } else { sf = 0; ef = vh->n_frm - 1; fprintf(fp, "VITHIST #frames %d #entries %d\n", vh->n_frm, vh->n_entry); } fprintf(fp, "\t%7s %5s %5s %11s %9s %8s %7s %4s Word (LM-state)\n", "Seq/Val", "SFrm", "EFrm", "PathScr", "SegAScr", "SegLScr", "Pred", "Type"); for (i = sf; i <= ef; i++) { fprintf(fp, "%5d BS: %11d BV: %8d\n", i, vh->bestscore[i], vh->bestvh[i]); for (j = vh->frame_start[i]; j < vh->frame_start[i + 1]; j++) { int32 lwid; ve = vithist_id2entry(vh, j); fprintf(fp, "\t%c%6d %5d %5d %11d %9d %8d %7d %4d %s", (ve->valid ? ' ' : '*'), j, ve->sf, ve->ef, ve->path.score, ve->ascr, ve->lscr, ve->path.pred, ve->type, s3dict_wordstr(dict, ve->wid)); fprintf(fp, " (%s", ngram_word(lm, ve->lmstate.lm3g.lwid[0])); lwid = ve->lmstate.lm3g.lwid[1]; fprintf(fp, ", %s", ngram_word(lm, lwid)); fprintf(fp, ")\n"); } if (j == vh->frame_start[i]) fprintf(fp, "\n"); } fprintf(fp, "END_VITHIST\n"); fflush(fp); }
char *unwind_phoneme(ngram_model_t *model, tree_element_t *tree_element) { int32 i, j, size = 0; char* phoneme; const char* unigram_phoneme; tree_element_t *element = tree_element; while (element) { unigram_phoneme = strstr(ngram_word(model, element->wid), "}") + 1; if (strcmp(unigram_phoneme, "_") != 0) { size += strlen(unigram_phoneme) + 1; } element = element->parent; } phoneme = ckd_malloc(size); phoneme[size - 1] = '\0'; i = size - 2; element = tree_element; while (element) { unigram_phoneme = strstr(ngram_word(model, element->wid), "}") + 1; if (strcmp(unigram_phoneme, "_") != 0) { i -= strlen(unigram_phoneme); j = i + 1; while (*unigram_phoneme) { phoneme[j] = *unigram_phoneme == '|' ? ' ' : *unigram_phoneme; j++; unigram_phoneme++; } if (i >= 0) { phoneme[i] = ' '; } i--; } element = element->parent; } return phoneme; }
static int test_lm_vals(ngram_model_t *model) { int32 n_used; TEST_ASSERT(model); TEST_EQUAL(ngram_wid(model, "<UNK>"), 0); TEST_EQUAL(strcmp(ngram_word(model, 0), "<UNK>"), 0); TEST_EQUAL(ngram_wid(model, "absolute"), 13); TEST_EQUAL(strcmp(ngram_word(model, 13), "absolute"), 0); /* Test unigrams. */ TEST_EQUAL(ngram_score(model, "<UNK>", NULL), -75346); TEST_EQUAL(ngram_bg_score(model, ngram_wid(model, "<UNK>"), NGRAM_INVALID_WID, &n_used), -75346); TEST_EQUAL(n_used, 1); TEST_EQUAL(ngram_score(model, "sphinxtrain", NULL), -64208); TEST_EQUAL(ngram_bg_score(model, ngram_wid(model, "sphinxtrain"), NGRAM_INVALID_WID, &n_used), -64208); TEST_EQUAL(n_used, 1); /* Test bigrams. */ TEST_EQUAL(ngram_score(model, "huggins", "david", NULL), -831); /* Test trigrams. */ TEST_EQUAL_LOG(ngram_score(model, "daines", "huggins", "david", NULL), -9450); return 0; }
struct winner_t dict_get_winner_wid(ngram_model_t *model, const char * word_grapheme, glist_t history_list, const int32 total_unigrams, int word_offset) { int32 current_prob = -2147483647; struct winner_t winner; int32 i = 0, j = 0; int nused; int32 ngram_order = ngram_model_get_size(model); int32 *history = ckd_calloc((size_t)ngram_order+1, sizeof(int32)); gnode_t *gn; const char *vocab; const char *sub; int32 prob; unigram_t unigram; for (gn = history_list; gn; gn = gnode_next(gn)) { history[ngram_order-j] = gnode_int32(gn); j++; if (j >= ngram_order) break; } for (i = 0; i < total_unigrams; i++) { vocab = ngram_word(model, i); unigram = dict_split_unigram(vocab); sub = word_grapheme + word_offset; if (dict_starts_with(unigram.word, sub)){ prob = ngram_ng_prob(model, i, history, j, &nused); if (current_prob < prob) { current_prob = prob; winner.winner_wid = i; winner.length_match = strlen(unigram.word); winner.len_phoneme = strlen(unigram.phone); } } if (unigram.word) ckd_free(unigram.word); if (unigram.phone) ckd_free(unigram.phone); } if (history) ckd_free(history); return winner; }
char *g2p(ngram_model_t *model, char *grapheme, uint32 level_count_limit) { int32 i, j, n, wid, fit_count; array_heap_t **tree_table; const char* unigram_text; char* phoneme; int32 *history_buffer; int32 start_wid, end_wid; const uint32 total_unigrams = *ngram_model_get_counts(model); n = strlen(grapheme); tree_table = ckd_calloc(n + 1, sizeof(array_heap_t *)); for (i = 0; i < n; i++) { tree_table[i] = array_heap_new(level_count_limit); } tree_table[n] = array_heap_new(1); history_buffer = ckd_calloc(n + 1, sizeof(int32)); start_wid = ngram_wid(model, "<s>"); end_wid = ngram_wid(model, "</s>"); for (i = 0; i < n; i++) { for (wid = 0; wid < total_unigrams; wid++) { unigram_text = ngram_word(model, wid); fit_count = graphemes_fit_count(grapheme, i, unigram_text); if (fit_count != 0) { try_add_tree_elements(model, wid, i == 0 ? NULL : tree_table[i - 1], tree_table[i + fit_count - 1], history_buffer, start_wid); } } } try_add_tree_elements(model, end_wid, tree_table[n - 1], tree_table[n], history_buffer, start_wid); phoneme = (tree_table[n]->size == 0) ? NULL : unwind_phoneme(model, ((tree_element_t*) array_heap_element(tree_table[n], 0))->parent); for (i = 0; i <= n; i++) { for (j = 0; j < tree_table[i]->size; j++) { ckd_free(array_heap_element(tree_table[i], j)); } array_heap_free(tree_table[i]); } ckd_free(tree_table); ckd_free(history_buffer); return phoneme; }
static int calc_entropy(ngram_model_t *lm, char **words, int32 n, int32 *out_n_ccs, int32 *out_n_oovs, int32 *out_lm_score) { int32 *wids; int32 startwid; int32 i, ch, nccs, noovs, unk; if (n == 0) return 0; unk = ngram_unknown_wid(lm); /* Reverse this array into an array of word IDs. */ wids = ckd_calloc(n, sizeof(*wids)); for (i = 0; i < n; ++i) wids[n-i-1] = ngram_wid(lm, words[i]); /* Skip <s> as it's a context cue (HACK, this should be configurable). */ startwid = ngram_wid(lm, "<s>"); /* Now evaluate the list of words in reverse using the * remainder of the array as the history. */ ch = noovs = nccs = 0; for (i = 0; i < n; ++i) { int32 n_used; int32 prob; /* Skip <s> as it's a context cue (HACK, this should be configurable). */ if (wids[i] == startwid) { ++nccs; continue; } /* Skip and count OOVs. */ if (wids[i] == NGRAM_INVALID_WID || wids[i] == unk) { ++noovs; continue; } /* Sum up information for each N-gram */ prob = ngram_ng_score(lm, wids[i], wids + i + 1, n - i - 1, &n_used); if (verbose) { int m; printf("log P(%s|", ngram_word(lm, wids[i])); m = i + ngram_model_get_size(lm) - 1; if (m >= n) m = n - 1; while (m > i) { printf("%s ", ngram_word(lm, wids[m--])); } printf(") = %d\n", prob); } ch -= prob; } if (out_n_ccs) *out_n_ccs = nccs; if (out_n_oovs) *out_n_oovs = noovs; /* Calculate cross-entropy CH = - 1/N sum log P(W|H) */ n -= (nccs + noovs); if (n <= 0) return 0; if (out_lm_score) *out_lm_score = -ch; return ch / n; }
char * dict_g2p(char const *word_grapheme, ngram_model_t *ngram_g2p_model) { char *final_phone = NULL; int totalh = 0; size_t increment = 1; int word_offset = 0; int j; size_t grapheme_len = 0, final_phoneme_len = 0; glist_t history_list = NULL; gnode_t *gn; int first = 0; const int32 *total_unigrams; struct winner_t winner; const char *word; unigram_t unigram; total_unigrams = ngram_model_get_counts(ngram_g2p_model); int32 wid_sentence = ngram_wid(ngram_g2p_model,"<s>"); // start with sentence history_list = glist_add_int32(history_list, wid_sentence); grapheme_len = strlen(word_grapheme); for (j = 0; j < grapheme_len; j += increment) { winner = dict_get_winner_wid(ngram_g2p_model, word_grapheme, history_list, *total_unigrams, word_offset); increment = winner.length_match; if (increment == 0) { E_ERROR("Error trying to find matching phoneme (%s) Exiting.. \n" , word_grapheme); return NULL; } history_list = glist_add_int32(history_list, winner.winner_wid); totalh = j + 1; word_offset += winner.length_match; final_phoneme_len += winner.len_phoneme; } history_list = glist_reverse(history_list); final_phone = ckd_calloc(1, final_phoneme_len * 2); for (gn = history_list; gn; gn = gnode_next(gn)) { if (!first) { first = 1; continue; } word = ngram_word(ngram_g2p_model, gnode_int32(gn)); if (!word) continue; unigram = dict_split_unigram(word); if (strcmp(unigram.phone, "_") == 0) { if (unigram.word) ckd_free(unigram.word); if (unigram.phone) ckd_free(unigram.phone); continue; } strcat(final_phone, unigram.phone); strcat(final_phone, " "); if (unigram.word) ckd_free(unigram.word); if (unigram.phone) ckd_free(unigram.phone); } if (history_list) glist_free(history_list); return final_phone; }
void run_tests(logmath_t *lmath, ngram_model_t *model) { int32 rv, i; TEST_ASSERT(model); TEST_EQUAL(ngram_wid(model, "scylla"), 285); TEST_EQUAL(strcmp(ngram_word(model, 285), "scylla"), 0); rv = ngram_model_read_classdef(model, LMDIR "/100.probdef"); TEST_EQUAL(rv, 0); /* Verify that class word IDs remain the same. */ TEST_EQUAL(ngram_wid(model, "scylla"), 285); TEST_EQUAL(strcmp(ngram_word(model, 285), "scylla"), 0); /* Verify in-class word IDs. */ TEST_EQUAL(ngram_wid(model, "scylla:scylla"), 0x80000000 | 400); /* Verify in-class and out-class unigram scores. */ TEST_EQUAL_LOG(ngram_score(model, "scylla:scylla", NULL), logmath_log10_to_log(lmath, -2.7884) + logmath_log(lmath, 0.4)); TEST_EQUAL_LOG(ngram_score(model, "scooby:scylla", NULL), logmath_log10_to_log(lmath, -2.7884) + logmath_log(lmath, 0.1)); TEST_EQUAL_LOG(ngram_score(model, "scylla", NULL), logmath_log10_to_log(lmath, -2.7884)); TEST_EQUAL_LOG(ngram_score(model, "oh:zero", NULL), logmath_log10_to_log(lmath, -1.9038) + logmath_log(lmath, 0.7)); TEST_EQUAL_LOG(ngram_score(model, "zero", NULL), logmath_log10_to_log(lmath, -1.9038)); /* Verify class bigram scores. */ TEST_EQUAL_LOG(ngram_score(model, "scylla", "on", NULL), logmath_log10_to_log(lmath, -1.2642)); TEST_EQUAL_LOG(ngram_score(model, "scylla:scylla", "on", NULL), logmath_log10_to_log(lmath, -1.2642) + logmath_log(lmath, 0.4)); TEST_EQUAL_LOG(ngram_score(model, "apparently", "scylla", NULL), logmath_log10_to_log(lmath, -0.5172)); TEST_EQUAL_LOG(ngram_score(model, "apparently", "karybdis:scylla", NULL), logmath_log10_to_log(lmath, -0.5172)); TEST_EQUAL_LOG(ngram_score(model, "apparently", "scooby:scylla", NULL), logmath_log10_to_log(lmath, -0.5172)); /* Verify class trigram scores. */ TEST_EQUAL_LOG(ngram_score(model, "zero", "be", "will", NULL), logmath_log10_to_log(lmath, -0.5725)); TEST_EQUAL_LOG(ngram_score(model, "oh:zero", "be", "will", NULL), logmath_log10_to_log(lmath, -0.5725) + logmath_log(lmath, 0.7)); TEST_EQUAL_LOG(ngram_score(model, "should", "variance", "zero", NULL), logmath_log10_to_log(lmath, -0.9404)); TEST_EQUAL_LOG(ngram_score(model, "should", "variance", "zero:zero", NULL), logmath_log10_to_log(lmath, -0.9404)); /* Add words to classes. */ rv = ngram_model_add_class_word(model, "scylla", "scrappy:scylla", 1.0); TEST_ASSERT(rv >= 0); TEST_EQUAL(ngram_wid(model, "scrappy:scylla"), 0x80000196); TEST_EQUAL_LOG(ngram_score(model, "scrappy:scylla", NULL), logmath_log10_to_log(lmath, -2.7884) + logmath_log(lmath, 0.2)); printf("scrappy:scylla %08x %d %f\n", ngram_wid(model, "scrappy:scylla"), ngram_score(model, "scrappy:scylla", NULL), logmath_exp(lmath, ngram_score(model, "scrappy:scylla", NULL))); /* Add a lot of words to a class. */ for (i = 0; i < 129; ++i) { char word[32]; sprintf(word, "%d:scylla", i); rv = ngram_model_add_class_word(model, "scylla", word, 1.0); printf("%s %08x %d %f\n", word, ngram_wid(model, word), ngram_score(model, word, NULL), logmath_exp(lmath, ngram_score(model, word, NULL))); TEST_ASSERT(rv >= 0); TEST_EQUAL(ngram_wid(model, word), 0x80000197 + i); } /* Add a new class. */ { const char *words[] = { "blatz:foobie", "hurf:foobie" }; float32 weights[] = { 0.6, 0.4 }; int32 foobie_prob; rv = ngram_model_add_class(model, "[foobie]", 1.0, words, weights, 2); TEST_ASSERT(rv >= 0); foobie_prob = ngram_score(model, "[foobie]", NULL); TEST_EQUAL_LOG(ngram_score(model, "blatz:foobie", NULL), foobie_prob + logmath_log(lmath, 0.6)); TEST_EQUAL_LOG(ngram_score(model, "hurf:foobie", NULL), foobie_prob + logmath_log(lmath, 0.4)); } }