static int
write_lattice(ps_decoder_t *ps, char const *latdir, char const *uttid)
{
ps_lattice_t *lat;
logmath_t *lmath;
cmd_ln_t *config;
char *outfile;
int32 beam;
if ((lat = ps_get_lattice(ps)) == NULL) {
E_ERROR("Failed to obtain word lattice for utterance %s\n", uttid);
return -1;
}
config = ps_get_config(ps);
outfile = string_join(latdir, "/", uttid,
cmd_ln_str_r(config, "-outlatext"), NULL);
/* Prune lattice. */
lmath = ps_get_logmath(ps);
beam = logmath_log(lmath, cmd_ln_float64_r(config, "-outlatbeam"));
ps_lattice_posterior_prune(lat, beam);
if (0 == strcmp("htk", cmd_ln_str_r(config, "-outlatfmt"))) {
if (ps_lattice_write_htk(lat, outfile) < 0) {
E_ERROR("Failed to write lattice to %s\n", outfile);
return -1;
}
}
else {
if (ps_lattice_write(lat, outfile) < 0) {
E_ERROR("Failed to write lattice to %s\n", outfile);
return -1;
}
}
return 0;
}
ps_nbest_t *
ps_nbest(ps_decoder_t *ps, int sf, int ef,
char const *ctx1, char const *ctx2)
{
ps_lattice_t *dag;
ngram_model_t *lmset;
ps_astar_t *nbest;
float32 lwf;
int32 w1, w2;
if (ps->search == NULL)
return NULL;
if ((dag = ps_get_lattice(ps)) == NULL)
return NULL;
/* FIXME: This is all quite specific to N-Gram search. Either we
* should make N-best a method for each search module or it needs
* to be abstracted to work for N-Gram and FSG. */
if (0 != strcmp(ps_search_name(ps->search), PS_SEARCH_NGRAM)) {
lmset = NULL;
lwf = 1.0f;
} else {
lmset = ((ngram_search_t *)ps->search)->lmset;
lwf = ((ngram_search_t *)ps->search)->bestpath_fwdtree_lw_ratio;
}
w1 = ctx1 ? dict_wordid(ps_search_dict(ps->search), ctx1) : -1;
w2 = ctx2 ? dict_wordid(ps_search_dict(ps->search), ctx2) : -1;
nbest = ps_astar_start(dag, lmset, lwf, sf, ef, w1, w2);
return (ps_nbest_t *)nbest;
}
ps_nbest_t *
ps_nbest(ps_decoder_t *ps)
{
ps_lattice_t *dag;
ngram_model_t *lmset;
ps_astar_t *nbest;
float32 lwf;
if (ps->search == NULL)
return NULL;
if ((dag = ps_get_lattice(ps)) == NULL)
return NULL;
/* FIXME: This is all quite specific to N-Gram search. Either we
* should make N-best a method for each search module or it needs
* to be abstracted to work for N-Gram and FSG. */
if (0 != strcmp(ps_search_type(ps->search), PS_SEARCH_TYPE_NGRAM)) {
lmset = NULL;
lwf = 1.0f;
} else {
lmset = ((ngram_search_t *)ps->search)->lmset;
lwf = ((ngram_search_t *)ps->search)->bestpath_fwdtree_lw_ratio;
}
nbest = ps_astar_start(dag, lmset, lwf, 0, -1, -1, -1);
nbest = ps_nbest_next(nbest);
return (ps_nbest_t *)nbest;
}
static void fsg_processor(context_t *ctx,
srs_srec_utterance_t *utt,
srs_srec_candidate_t *cands,
srs_srec_candidate_t **sorted)
{
filter_buf_t *filtbuf;
decoder_set_t *decset;
decoder_t *dec;
logmath_t *lmath;
const char *uttid;
int32_t score;
double prob;
srs_srec_candidate_t *cand;
srs_srec_token_t *tkn;
ps_lattice_t *dag;
ps_latlink_t *lnk;
ps_latnode_t *nod;
const char *token;
int32_t frlen;
int32_t start, end;
int16 fef, lef;
if (!ctx || !(filtbuf = ctx->filtbuf) ||
!(decset = ctx->decset) || !(dec = decset->curdec))
return;
frlen = filtbuf->frlen;
lmath = ps_get_logmath(dec->ps);
ps_get_hyp(dec->ps, &score, &uttid);
prob = logmath_exp(lmath, score);
cand = cands;
cand->score = 1.0;
cand->ntoken = 0;
tkn = NULL;
if ((dag = ps_get_lattice(dec->ps))) {
if ((lnk = ps_lattice_traverse_edges(dag, NULL, NULL))) {
ps_latlink_nodes(lnk, &nod);
if (nod && (token = ps_latnode_word(dag, nod)) && *token != '<') {
tkn = cand->tokens + cand->ntoken++;
tkn->token = tknbase(token);
tkn->start = ps_latnode_times(nod, &fef, &lef) * frlen;
tkn->end = ((fef + lef) / 2) * frlen;
}
goto handle_destination_node;
while ((lnk = ps_lattice_traverse_next(dag, NULL))) {
handle_destination_node:
nod = ps_latlink_nodes(lnk, NULL);
if (nod && (token = ps_latnode_word(dag,nod)) && *token != '<')
{
start = ps_latnode_times(nod, &fef, &lef) * frlen;
end = fef * frlen;
if (tkn && start < (int32_t)tkn->end)
break; /* just take one candidate */
if (!tkn || !tkneq(token, tkn->token)) {
tkn = cand->tokens + cand->ntoken++;
tkn->token = tknbase(token);
tkn->start = start;
tkn->end = end + frlen;
}
}
}
}
}
sorted[0] = cands;
sorted[1] = NULL;
utt->id = uttid;
utt->score = prob < 0.00001 ? 0.00001 : prob;
//utt->length = dag ? ps_lattice_n_frames(dag) * frlen : 0;
utt->length = filtbuf->len;
utt->ncand = 1;
utt->cands = sorted;
}
int
main(int argc, char *argv[])
{
ps_decoder_t *ps;
cmd_ln_t *config;
acmod_t *acmod;
fsg_search_t *fsgs;
ps_lattice_t *dag;
ps_seg_t *seg;
int32 score;
TEST_ASSERT(config =
cmd_ln_init(NULL, ps_args(), TRUE,
"-hmm", DATADIR "/tidigits/hmm",
"-fsg", DATADIR "/tidigits/lm/tidigits.fsg",
"-dict", DATADIR "/tidigits/lm/tidigits.dic",
"-bestpath", "no",
"-input_endian", "little",
"-samprate", "16000", NULL));
TEST_ASSERT(ps = ps_init(config));
fsgs = (fsg_search_t *)ps->search;
acmod = ps->acmod;
setbuf(stdout, NULL);
{
FILE *rawfh;
int16 buf[2048];
size_t nread;
int16 const *bptr;
char const *hyp;
int nfr;
TEST_ASSERT(rawfh = fopen(DATADIR "/numbers.raw", "rb"));
TEST_EQUAL(0, acmod_start_utt(acmod));
fsg_search_start(ps_search_base(fsgs));
while (!feof(rawfh)) {
nread = fread(buf, sizeof(*buf), 2048, rawfh);
bptr = buf;
while ((nfr = acmod_process_raw(acmod, &bptr, &nread, FALSE)) > 0) {
while (acmod->n_feat_frame > 0) {
fsg_search_step(ps_search_base(fsgs),
acmod->output_frame);
acmod_advance(acmod);
}
}
}
fsg_search_finish(ps_search_base(fsgs));
hyp = fsg_search_hyp(ps_search_base(fsgs), &score, NULL);
printf("FSG: %s (%d)\n", hyp, score);
TEST_ASSERT(acmod_end_utt(acmod) >= 0);
fclose(rawfh);
}
for (seg = ps_seg_iter(ps); seg;
seg = ps_seg_next(seg)) {
char const *word;
int sf, ef;
int32 post, lscr, ascr, lback;
word = ps_seg_word(seg);
ps_seg_frames(seg, &sf, &ef);
post = ps_seg_prob(seg, &ascr, &lscr, &lback);
printf("%s (%d:%d) P(w|o) = %f ascr = %d lscr = %d lback = %d\n", word, sf, ef,
logmath_exp(ps_get_logmath(ps), post), ascr, lscr, lback);
}
/* Now get the DAG and play with it. */
dag = ps_get_lattice(ps);
ps_lattice_write(dag, "test_fsg3.lat");
printf("BESTPATH: %s\n",
ps_lattice_hyp(dag, ps_lattice_bestpath(dag, NULL, 1.0, 15.0)));
ps_lattice_posterior(dag, NULL, 15.0);
ps_free(ps);
cmd_ln_free_r(config);
return 0;
}
static gboolean
gst_pocketsphinx_event(GstPad *pad, GstEvent *event)
{
GstPocketSphinx *ps;
ps = GST_POCKETSPHINX(GST_OBJECT_PARENT(pad));
/* Pick out VAD events. */
switch (event->type) {
case GST_EVENT_NEWSEGMENT:
/* Initialize the decoder once the audio starts, if it's not
* there yet. */
if (ps->ps == NULL) {
ps->ps = ps_init(ps->config);
if (ps->ps == NULL) {
GST_ELEMENT_ERROR(GST_ELEMENT(ps), LIBRARY, INIT,
("Failed to initialize PocketSphinx"),
("Failed to initialize PocketSphinx"));
return FALSE;
}
}
return gst_pad_event_default(pad, event);
case GST_EVENT_VADER_START:
ps->listening = TRUE;
ps_start_utt(ps->ps, NULL);
/* Forward this event. */
return gst_pad_event_default(pad, event);
case GST_EVENT_EOS:
case GST_EVENT_VADER_STOP: {
GstBuffer *buffer;
int32 score;
char const *hyp;
char const *uttid;
hyp = NULL;
if (ps->listening) {
ps->listening = FALSE;
ps_end_utt(ps->ps);
hyp = ps_get_hyp(ps->ps, &score, &uttid);
/* Dump the lattice if requested. */
if (ps->latdir) {
char *latfile = string_join(ps->latdir, "/", uttid, ".lat", NULL);
ps_lattice_t *dag;
if ((dag = ps_get_lattice(ps->ps)))
ps_lattice_write(dag, latfile);
ckd_free(latfile);
}
}
if (hyp) {
/* Emit a signal for applications. */
g_signal_emit(ps, gst_pocketsphinx_signals[SIGNAL_RESULT],
0, hyp, uttid);
/* Forward this result in a buffer. */
buffer = gst_buffer_new_and_alloc(strlen(hyp) + 2);
strcpy((char *)GST_BUFFER_DATA(buffer), hyp);
GST_BUFFER_DATA(buffer)[strlen(hyp)] = '\n';
GST_BUFFER_DATA(buffer)[strlen(hyp)+1] = '\0';
GST_BUFFER_TIMESTAMP(buffer) = GST_EVENT_TIMESTAMP(event);
gst_buffer_set_caps(buffer, GST_PAD_CAPS(ps->srcpad));
gst_pad_push(ps->srcpad, buffer);
}
/* Forward this event. */
return gst_pad_event_default(pad, event);
}
default:
/* Don't bother with other events. */
return gst_pad_event_default(pad, event);
}
}
static void
gst_pocketsphinx_get_property(GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec)
{
GstPocketSphinx *ps = GST_POCKETSPHINX(object);
switch (prop_id) {
case PROP_DECODER:
g_value_set_boxed(value, ps->ps);
break;
case PROP_CONFIGURED:
g_value_set_boolean(value, ps->ps != NULL);
break;
case PROP_HMM_DIR:
g_value_set_string(value, cmd_ln_str_r(ps->config, "-hmm"));
break;
case PROP_LM_FILE:
g_value_set_string(value, cmd_ln_str_r(ps->config, "-lm"));
break;
case PROP_LMCTL_FILE:
g_value_set_string(value, cmd_ln_str_r(ps->config, "-lmctl"));
break;
case PROP_LM_NAME:
g_value_set_string(value, cmd_ln_str_r(ps->config, "-lmname"));
break;
case PROP_DICT_FILE:
g_value_set_string(value, cmd_ln_str_r(ps->config, "-dict"));
break;
case PROP_MLLR_FILE:
g_value_set_string(value, cmd_ln_str_r(ps->config, "-mllr"));
break;
case PROP_FSG_FILE:
g_value_set_string(value, cmd_ln_str_r(ps->config, "-fsg"));
break;
case PROP_FWDFLAT:
g_value_set_boolean(value, cmd_ln_boolean_r(ps->config, "-fwdflat"));
break;
case PROP_BESTPATH:
g_value_set_boolean(value, cmd_ln_boolean_r(ps->config, "-bestpath"));
break;
case PROP_LATDIR:
g_value_set_string(value, ps->latdir);
break;
case PROP_LATTICE: {
ps_lattice_t *dag;
if (ps->ps && (dag = ps_get_lattice(ps->ps)))
g_value_set_boxed(value, dag);
else
g_value_set_boxed(value, NULL);
break;
}
case PROP_MAXHMMPF:
g_value_set_int(value, cmd_ln_int32_r(ps->config, "-maxhmmpf"));
break;
case PROP_MAXWPF:
g_value_set_int(value, cmd_ln_int32_r(ps->config, "-maxwpf"));
break;
case PROP_BEAM:
g_value_set_double(value, cmd_ln_float_r(ps->config, "-beam"));
break;
case PROP_PBEAM:
g_value_set_double(value, cmd_ln_float_r(ps->config, "-pbeam"));
break;
case PROP_WBEAM:
g_value_set_double(value, cmd_ln_float_r(ps->config, "-wbeam"));
break;
case PROP_DSRATIO:
g_value_set_int(value, cmd_ln_int32_r(ps->config, "-ds"));
break;
case PROP_NBEST_SIZE:
g_value_set_int(value, ps->n_best_size);
break;
case PROP_NBEST: {
int i = 0, out_score = 0;
GValueArray *arr;
if (!ps->ps) {
break;
}
arr = g_value_array_new(1);
ps_nbest_t *ps_nbest_list = ps_nbest(ps->ps, 0, -1, NULL, NULL);
if (ps_nbest_list) {
ps_nbest_list = ps_nbest_next(ps_nbest_list);
while ((i < ps->n_best_size) && (ps_nbest_list != NULL)) {
GValue value1 = { 0 };
g_value_init (&value1, G_TYPE_STRING);
const char* hyp = ps_nbest_hyp(ps_nbest_list, &out_score);
g_value_set_string(&value1, hyp);
g_value_array_append(arr, &value1);
ps_nbest_list = ps_nbest_next(ps_nbest_list);
i++;
}
if (ps_nbest_list) {
ps_nbest_free(ps_nbest_list);
}
}
g_value_set_boxed (value, arr);
break;
}
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
break;
}
}
int
main(int argc, char *argv[])
{
ps_decoder_t *ps;
cmd_ln_t *config;
acmod_t *acmod;
fsg_search_t *fsgs;
jsgf_t *jsgf;
jsgf_rule_t *rule;
fsg_model_t *fsg;
ps_seg_t *seg;
ps_lattice_t *dag;
FILE *rawfh;
char const *hyp, *uttid;
int32 score, prob;
clock_t c;
int i;
TEST_ASSERT(config =
cmd_ln_init(NULL, ps_args(), TRUE,
"-hmm", MODELDIR "/hmm/en_US/hub4wsj_sc_8k",
"-dict", MODELDIR "/lm/en/turtle.dic",
"-input_endian", "little",
"-samprate", "16000", NULL));
TEST_ASSERT(ps = ps_init(config));
jsgf = jsgf_parse_file(DATADIR "/goforward.gram", NULL);
TEST_ASSERT(jsgf);
rule = jsgf_get_rule(jsgf, "<goforward.move2>");
TEST_ASSERT(rule);
fsg = jsgf_build_fsg(jsgf, rule, ps->lmath, 7.5);
TEST_ASSERT(fsg);
fsg_model_write(fsg, stdout);
ps_set_fsg(ps, "<goforward.move2>", fsg);
ps_set_search(ps, "<goforward.move2>");
acmod = ps->acmod;
fsgs = (fsg_search_t *) fsg_search_init(fsg, config, acmod, ps->dict, ps->d2p);
setbuf(stdout, NULL);
c = clock();
for (i = 0; i < 5; ++i) {
int16 buf[2048];
size_t nread;
int16 const *bptr;
int nfr;
int is_final;
TEST_ASSERT(rawfh = fopen(DATADIR "/goforward.raw", "rb"));
TEST_EQUAL(0, acmod_start_utt(acmod));
fsg_search_start(ps_search_base(fsgs));
is_final = FALSE;
while (!feof(rawfh)) {
nread = fread(buf, sizeof(*buf), 2048, rawfh);
bptr = buf;
while ((nfr = acmod_process_raw(acmod, &bptr, &nread, FALSE)) > 0) {
while (acmod->n_feat_frame > 0) {
fsg_search_step(ps_search_base(fsgs),
acmod->output_frame);
acmod_advance(acmod);
}
}
hyp = fsg_search_hyp(ps_search_base(fsgs), &score, &is_final);
printf("FSG: %s (%d) frame %d final %s\n", hyp, score, acmod->output_frame, is_final ? "FINAL" : "");
TEST_EQUAL (is_final, (acmod->output_frame > 170));
}
fsg_search_finish(ps_search_base(fsgs));
hyp = fsg_search_hyp(ps_search_base(fsgs), &score, NULL);
printf("FSG: %s (%d)\n", hyp, score);
TEST_ASSERT(acmod_end_utt(acmod) >= 0);
fclose(rawfh);
}
TEST_EQUAL(0, strcmp("go forward ten meters",
fsg_search_hyp(ps_search_base(fsgs), &score, NULL)));
ps->search = (ps_search_t *)fsgs;
for (seg = ps_seg_iter(ps, &score); seg;
seg = ps_seg_next(seg)) {
char const *word;
int sf, ef;
word = ps_seg_word(seg);
ps_seg_frames(seg, &sf, &ef);
printf("%s %d %d\n", word, sf, ef);
}
c = clock() - c;
printf("5 * fsg search in %.2f sec\n", (double)c / CLOCKS_PER_SEC);
dag = ps_get_lattice(ps);
ps_lattice_write(dag, "test_jsgf.lat");
jsgf_grammar_free(jsgf);
fsg_search_free(ps_search_base(fsgs));
ps_free(ps);
cmd_ln_free_r(config);
TEST_ASSERT(config =
cmd_ln_init(NULL, ps_args(), TRUE,
"-hmm", MODELDIR "/hmm/en_US/hub4wsj_sc_8k",
"-dict", MODELDIR "/lm/en/turtle.dic",
"-jsgf", DATADIR "/goforward.gram",
"-input_endian", "little",
"-samprate", "16000", NULL));
TEST_ASSERT(ps = ps_init(config));
TEST_ASSERT(rawfh = fopen(DATADIR "/goforward.raw", "rb"));
ps_decode_raw(ps, rawfh, "goforward", -1);
hyp = ps_get_hyp(ps, &score, &uttid);
prob = ps_get_prob(ps, &uttid);
printf("%s: %s (%d, %d)\n", uttid, hyp, score, prob);
TEST_EQUAL(0, strcmp("go forward ten meters", hyp));
ps_free(ps);
fclose(rawfh);
cmd_ln_free_r(config);
TEST_ASSERT(config =
cmd_ln_init(NULL, ps_args(), TRUE,
"-hmm", MODELDIR "/hmm/en_US/hub4wsj_sc_8k",
"-dict", MODELDIR "/lm/en/turtle.dic",
"-jsgf", DATADIR "/goforward.gram",
"-toprule", "goforward.move2",
"-input_endian", "little",
"-samprate", "16000", NULL));
TEST_ASSERT(ps = ps_init(config));
TEST_ASSERT(rawfh = fopen(DATADIR "/goforward.raw", "rb"));
ps_decode_raw(ps, rawfh, "goforward", -1);
hyp = ps_get_hyp(ps, &score, &uttid);
prob = ps_get_prob(ps, &uttid);
printf("%s: %s (%d, %d)\n", uttid, hyp, score, prob);
TEST_EQUAL(0, strcmp("go forward ten meters", hyp));
ps_free(ps);
cmd_ln_free_r(config);
fclose(rawfh);
TEST_ASSERT(config =
cmd_ln_init(NULL, ps_args(), TRUE,
"-hmm", MODELDIR "/hmm/en_US/hub4wsj_sc_8k",
"-dict", MODELDIR "/lm/en/turtle.dic",
"-jsgf", DATADIR "/defective.gram",
NULL));
TEST_ASSERT(NULL == ps_init(config));
cmd_ln_free_r(config);
return 0;
}
int main(int argc, char** argv)
{
int i;
int rv;
FILE* fh;
char const *hyp, *uttid;
int32 score;
ps_decoder_t *ps;
cmd_ln_t *config;
ps_lattice_t* dag;
config = cmd_ln_init(NULL, ps_args(), TRUE,
"-hmm", "./hmm/zh_broadcastnews_ptm256_8000",
"-lm", "./lm/syllables.lm.DMP",
"-dict", "./lm/syllables_sorted.dic",
NULL);
if (config == NULL)
return 1;
ps = ps_init(config);
if (ps == NULL)
return 1;
fh = fopen(argv[1], "rb");
if (fh == NULL) {
perror("Failed to open audio file.");
return 1;
}
rv = ps_decode_raw(ps, fh, "test", -1);
if (rv < 0)
return 1;
hyp = ps_get_hyp(ps, &score, &uttid);
if (hyp == NULL)
return 1;
printf("Recognized: %s\n", hyp);
inverted_index_t* index = inverted_index_init("./syllable.lst");
if (index == NULL) {
exit(1);
}
dag = ps_get_lattice(ps);
if (dag == NULL) {
perror("No lattice");
return 1;
}
/*
printf("# Total number of words: %d\n", index->n_word);
for(i = 0; i < index->n_word; i++) {
printf("%3d: %s\n", i+1, index->word_list[i]);
}*/
float32 ascale = cmd_ln_float32_r(config, "-ascale");
printf("ascale: %f\n", ascale);
//printf("%d: %s\n", inverted_index_get_wid(index, "ba"), "ba");
//printf("%d: %s\n", inverted_index_get_wid(index, "bia"), "bia");
inverted_index_addhits(index, "test", dag, 1.0/ascale);
inverted_index_write(index, "./index");
inverted_index_free(index);
index = inverted_index_read("./index");
inverted_index_write(index, "./index2");
char* query[] = {"jin", "tian", "jie", "mu"};
result_list_t* rl;
inverted_index_search(index, ps_get_lmset(ps), 1.0/ascale, query, 4, &rl);
inverted_index_free(index);
return 0;
}
