Ejemplo n.º 1
0
 int32
 ngram_model_init(ngram_model_t *base,
                  ngram_funcs_t *funcs,
                  logmath_t *lmath,
                  int32 n, int32 n_unigram)
 {
     base->refcount = 1;
     base->funcs = funcs;
     base->n = n;
     /* If this was previously initialized... */
    if (base->n_counts == NULL)
        base->n_counts = ckd_calloc(3, sizeof(*base->n_counts));
    /* Don't reset weights if logmath object hasn't changed. */
    if (base->lmath != lmath) {
        /* Set default values for weights. */
        base->lw = 1.0;
        base->log_wip = 0; /* i.e. 1.0 */
        base->log_uw = 0;  /* i.e. 1.0 */
        base->log_uniform = logmath_log(lmath, 1.0 / n_unigram);
        base->log_uniform_weight = logmath_get_zero(lmath);
        base->log_zero = logmath_get_zero(lmath);
        base->lmath = lmath;
    }
    /* Allocate or reallocate space for word strings. */
    if (base->word_str) {
        /* Free all previous word strings if they were allocated. */
        if (base->writable) {
            int32 i;
            for (i = 0; i < base->n_words; ++i) {
                ckd_free(base->word_str[i]);
                base->word_str[i] = NULL;
            }
        }
        base->word_str = ckd_realloc(base->word_str, n_unigram * sizeof(char *));
    }
    else
        base->word_str = ckd_calloc(n_unigram, sizeof(char *));
    /* NOTE: They are no longer case-insensitive since we are allowing
     * other encodings for word strings.  Beware. */
    if (base->wid)
        hash_table_empty(base->wid);
    else
        base->wid = hash_table_new(n_unigram, FALSE);
    base->n_counts[0] = base->n_1g_alloc = base->n_words = n_unigram;

    return 0;
}
Ejemplo n.º 2
0
void
acmod_grow_feat_buf(acmod_t *acmod, int nfr)
{
    mfcc_t ***new_feat_buf;

    new_feat_buf = feat_array_alloc(acmod->fcb, nfr);
    if (acmod->n_feat_frame || acmod->grow_feat) {
        memcpy(new_feat_buf[0][0], acmod->feat_buf[0][0],
               (acmod->n_feat_alloc
                * feat_dimension(acmod->fcb)
                * sizeof(***acmod->feat_buf)));
    }
    feat_array_free(acmod->feat_buf);
    acmod->framepos = ckd_realloc(acmod->framepos,
                                  nfr * sizeof(*acmod->framepos));
    acmod->feat_buf = new_feat_buf;
    acmod->n_feat_alloc = nfr;
}
Ejemplo n.º 3
0
/**
 * Add a word to the word string and ID mapping.
 */
int32
ngram_add_word_internal(ngram_model_t * model,
                        const char *word, int32 classid)
{

    /* Check for hash collisions. */
    int32 wid;
    if (hash_table_lookup_int32(model->wid, word, &wid) == 0) {
        E_WARN("Omit duplicate word '%s'\n", word);
        return wid;
    }

    /* Take the next available word ID */
    wid = model->n_words;
    if (classid >= 0) {
        wid = NGRAM_CLASSWID(wid, classid);
    }

    /* Reallocate word_str if necessary. */
    if (model->n_words >= model->n_1g_alloc) {
        model->n_1g_alloc += UG_ALLOC_STEP;
        model->word_str = ckd_realloc(model->word_str,
                                      sizeof(*model->word_str) *
                                      model->n_1g_alloc);
    }
    /* Add the word string in the appropriate manner. */
    /* Class words are always dynamically allocated. */
    model->word_str[model->n_words] = ckd_salloc(word);
    /* Now enter it into the hash table. */
    if (hash_table_enter_int32
        (model->wid, model->word_str[model->n_words], wid) != wid) {
        E_ERROR
            ("Hash insertion failed for word %s => %p (should not happen)\n",
             model->word_str[model->n_words], (void *) (long) (wid));
    }
    /* Increment number of words. */
    ++model->n_words;
    return wid;
}
Ejemplo n.º 4
0
static size_t
strnappend(char **dest, size_t *dest_allocation, 
       const char *source, size_t n)
{
    size_t source_len, required_allocation;

    if (dest == NULL || dest_allocation == NULL)
        return -1;
    if (*dest == NULL && *dest_allocation != 0)
        return -1;
    if (source == NULL)
        return *dest_allocation;

    source_len = strlen(source);
    if (n && n < source_len)
        source_len = n;

    required_allocation = (*dest ? strlen(*dest) : 0) + source_len + 1;
    if (*dest_allocation < required_allocation) {
        if (*dest_allocation == 0) {
            *dest = ckd_calloc(required_allocation * 2, 1);
        } else {
            *dest = ckd_realloc(*dest, required_allocation * 2);
        }
        *dest_allocation = required_allocation * 2;
    } 
 #import "OpenEarsStaticAnalysisToggle.h"
#ifdef STATICANALYZEDEPENDENCIES
#define __clang_analyzer__ 1
#endif
#if !defined(__clang_analyzer__) || defined(STATICANALYZEDEPENDENCIES)
#undef __clang_analyzer__
    strncat(*dest, source, source_len);
#endif
    return *dest_allocation;
}
Ejemplo n.º 5
0
int
fsg_model_word_add(fsg_model_t *fsg, char const *word)
{
    int wid;

    /* Search for an existing word matching this. */
    wid = fsg_model_word_id(fsg, word);
    /* If not found, add this to the vocab. */
    if (wid == -1) {
        wid = fsg->n_word;
        if (fsg->n_word == fsg->n_word_alloc) {
            fsg->n_word_alloc += 10;
            fsg->vocab = ckd_realloc(fsg->vocab,
                                     fsg->n_word_alloc * sizeof(*fsg->vocab));
            if (fsg->silwords)
                fsg->silwords = bitvec_realloc(fsg->silwords, fsg->n_word_alloc);
            if (fsg->altwords)
                fsg->altwords = bitvec_realloc(fsg->altwords, fsg->n_word_alloc);
        }
        ++fsg->n_word;
        fsg->vocab[wid] = ckd_salloc(word);
    }
    return wid;
}
Ejemplo n.º 6
0
lmset_t *
lmset_read_ctl(const char *ctlfile,
               dict_t * dict,
               float64 lw, float64 wip, float64 uw,
               const char *lmdumpdir, logmath_t *logmath)
{
    FILE *ctlfp;
    FILE *tmp;
    char lmfile[4096], lmname[4096], str[4096];

    lmclass_set_t *lmclass_set;
    lmclass_t **lmclass, *cl;
    int32 n_lmclass, n_lmclass_used;
    int32 i;
    lm_t *lm;
    lmset_t *lms = NULL;
    tmp = NULL;

    E_INFO("Reading LM control file '%s'\n", ctlfile);
    if ((ctlfp = fopen(ctlfile, "r")) == NULL) {
	    E_ERROR_SYSTEM("Failed to open LM control file");
	    return NULL;
    }

    lmclass_set = lmclass_newset();

    lms = (lmset_t *) ckd_calloc(1, sizeof(lmset_t));
    lms->n_lm = 0;
    lms->n_alloc_lm = 0;

    if (fscanf(ctlfp, "%s", str) == 1) {
        if (strcmp(str, "{") == 0) {
            /* Load LMclass files */
            while ((fscanf(ctlfp, "%s", str) == 1)
                   && (strcmp(str, "}") != 0))
                lmclass_set = lmclass_loadfile(lmclass_set, str, logmath);

            if (strcmp(str, "}") != 0)
                E_FATAL("Unexpected EOF(%s)\n", ctlfile);

            if (fscanf(ctlfp, "%s", str) != 1)
                str[0] = '\0';
        }
    }
    else
        str[0] = '\0';

    /* Fill in dictionary word id information for each LMclass word */
    for (cl = lmclass_firstclass(lmclass_set);
         lmclass_isclass(cl); cl = lmclass_nextclass(lmclass_set, cl)) {

        /*
           For every words in the class, set the dictwid correctly 
           The following piece of code replace s2's kb_init_lmclass_dictwid (cl);
           doesn't do any checking even the id is a bad dict id. 
           This only sets the information in the lmclass_set, but not 
           lm-2-dict or dict-2-lm map.  In Sphinx 3, they are done in 
           wid_dict_lm_map in wid.c.
         */

        lmclass_word_t *w;
        int32 wid;
        for (w = lmclass_firstword(cl); lmclass_isword(w);
             w = lmclass_nextword(cl, w)) {
            wid = dict_wordid(dict, lmclass_getword(w));
#if 0
            E_INFO("In class %s, Word %s, wid %d\n", cl->name,
                   lmclass_getword(w), wid);
#endif
            lmclass_set_dictwid(w, wid);
        }
    }

    /* At this point if str[0] != '\0', we have an LM filename */

    n_lmclass = lmclass_get_nclass(lmclass_set);
    lmclass = (lmclass_t **) ckd_calloc(n_lmclass, sizeof(lmclass_t *));

    E_INFO("Number of LM class specified %d in file %s\n", n_lmclass,
           ctlfile);

    /* Read in one LM at a time */
    while (str[0] != '\0') {
        strcpy(lmfile, str);
        if (fscanf(ctlfp, "%s", lmname) != 1)
            E_FATAL("LMname missing after LMFileName '%s'\n", lmfile);

        n_lmclass_used = 0;

        if (fscanf(ctlfp, "%s", str) == 1) {
            if (strcmp(str, "{") == 0) {
                while ((fscanf(ctlfp, "%s", str) == 1) &&
                       (strcmp(str, "}") != 0)) {
                    if (n_lmclass_used >= n_lmclass) {
                        E_FATAL("Too many LM classes specified for '%s'\n",
                                lmfile);
                    }

                    lmclass[n_lmclass_used] =
                        lmclass_get_lmclass(lmclass_set, str);
                    if (!(lmclass_isclass(lmclass[n_lmclass_used])))
                        E_FATAL("LM class '%s' not found\n", str);
                    n_lmclass_used++;
                }
                if (strcmp(str, "}") != 0)
                    E_FATAL("Unexpected EOF(%s)\n", ctlfile);
                if (fscanf(ctlfp, "%s", str) != 1)
                    str[0] = '\0';
            }
        }
        else
            str[0] = '\0';

        lm = (lm_t *) lm_read_advance(lmfile, lmname, lw, wip, uw,
                                      dict_size(dict), NULL, 1, logmath, FALSE, FALSE);


        if (n_lmclass_used > 0) {
            E_INFO("Did I enter here?\n");
            lm_build_lmclass_info(lm, lw, uw, wip, n_lmclass_used,
                                  lmclass);
        }

        if (lms->n_lm == lms->n_alloc_lm) {
            lms->lmarray =
                (lm_t **) ckd_realloc(lms->lmarray,
                                      (lms->n_alloc_lm +
                                       LM_ALLOC_BLOCK) * sizeof(lm_t *));
            lms->n_alloc_lm += LM_ALLOC_BLOCK;
        }

        lms->lmarray[lms->n_lm] = lm;
        lms->n_lm += 1;
        E_INFO("%d %d\n", lms->n_alloc_lm, lms->n_lm);
    }

    assert(lms);
    assert(lms->lmarray);
    E_INFO("No. of LM set allocated %d, no. of LM %d \n", lms->n_alloc_lm,
           lms->n_lm);


    if (dict != NULL) {
        for (i = 0; i < lms->n_lm; i++) {
            assert(lms->lmarray[i]);
            assert(dict);
            if ((lms->lmarray[i]->dict2lmwid =
                 wid_dict_lm_map(dict, lms->lmarray[i], lw)) == NULL)
                E_FATAL
                    ("Dict/LM word-id mapping failed for LM index %d, named %s\n",
                     i, lmset_idx_to_name(lms, i));
        }
    }
    else {
        E_FATAL
            ("Dict is specified to be NULL (dict_init is not called before lmset_read_lm?), dict2lmwid is not built inside lmset_read_lm\n");
    }


    ckd_free(lmclass_set);
    ckd_free(lmclass);
    fclose(ctlfp);
    return lms;
}
Ejemplo n.º 7
0
int32
feat_s2mfc2feat(feat_t * fcb, const char *file, const char *dir, const char *cepext,
                int32 sf, int32 ef, mfcc_t *** feat, int32 maxfr)
{
    char *path;
    char *ps = "/";
    int32 win, nfr;
    int32 file_length, cepext_length, path_length = 0;
    mfcc_t **mfc;

    if (fcb->cepsize <= 0) {
        E_ERROR("Bad cepsize: %d\n", fcb->cepsize);
        return -1;
    }

    if (cepext == 0)
        cepext = "";

    /*
     * Create mfc filename, combining file, dir and extension if
     * necessary
     */

    /*
     * First we decide about the path. If dir is defined, then use
     * it. Otherwise assume the filename already contains the path.
     */
    if (dir == 0) {
        dir = "";
        ps = "";
        /*
         * This is not true but some 3rd party apps
         * may parse the output explicitly checking for this line
         */
        E_INFO("At directory . (current directory)\n");
    }
    else {
        E_INFO("At directory %s\n", dir);
        /*
         * Do not forget the path separator!
         */
        path_length += strlen(dir) + 1;
    }

    /*
     * Include cepext, if it's not already part of the filename.
     */
    file_length = strlen(file);
    cepext_length = strlen(cepext);
    if ((file_length > cepext_length)
        && (strcmp(file + file_length - cepext_length, cepext) == 0)) {
        cepext = "";
        cepext_length = 0;
    }

    /*
     * Do not forget the '\0'
     */
    path_length += file_length + cepext_length + 1;
    path = (char*) ckd_calloc(path_length, sizeof(char));

#ifdef HAVE_SNPRINTF
    /*
     * Paranoia is our best friend...
     */
    while ((file_length = snprintf(path, path_length, "%s%s%s%s", dir, ps, file, cepext)) > path_length) {
        path_length = file_length;
        path = (char*) ckd_realloc(path, path_length * sizeof(char));
    }
#else
#ifndef POCKETSPHINX_NET
    sprintf(path, "%s%s%s%s", dir, ps, file, cepext);
#else
	strcpy(path,dir);
	strcat(path,ps);
	strcat(path,file);
	strcat(path,cepext);
#endif
#endif

    win = feat_window_size(fcb);
    /* Pad maxfr with win, so we read enough raw feature data to
     * calculate the requisite number of dynamic features. */
    if (maxfr >= 0)
        maxfr += win * 2;

    if (feat != 0) {
        /* Read mfc file including window or padding if necessary. */
        nfr = feat_s2mfc_read_norm_pad(fcb, path, win, sf, ef, &mfc, maxfr, fcb->cepsize);
        ckd_free(path);
        if (nfr < 0) {
            ckd_free_2d((void **) mfc);
            return -1;
        }

        /* Actually compute the features */
        feat_compute_utt(fcb, mfc, nfr, win, feat);
        
        ckd_free_2d((void **) mfc);
    }
    else {
        /* Just calculate the number of frames we would need. */
        nfr = feat_s2mfc_read_norm_pad(fcb, path, win, sf, ef, 0, maxfr, fcb->cepsize);
        ckd_free(path);
        if (nfr < 0)
            return nfr;
    }


    return (nfr - win * 2);
}
Ejemplo n.º 8
0
cmd_ln_t *
cmd_ln_parse_file_r(cmd_ln_t *inout_cmdln, const arg_t * defn, const char *filename, int32 strict)
{
    FILE *file;
    int argc;
    int argv_size;
    char *str;
    int arg_max_length = 512;
    int len = 0;
    int quoting, ch;
    char **f_argv;
    int rv = 0;
    const char separator[] = " \t\r\n";

    if ((file = fopen(filename, "r")) == NULL) {
        E_ERROR("Cannot open configuration file %s for reading\n",
                filename);
        return NULL;
    }

    ch = fgetc(file);
    /* Skip to the next interesting character */
    for (; ch != EOF && strchr(separator, ch); ch = fgetc(file)) ;

    if (ch == EOF) {
        fclose(file);
        return NULL;
    }

    /*
     * Initialize default argv, argc, and argv_size.
     */
    argv_size = 10;
    argc = 0;
    f_argv = ckd_calloc(argv_size, sizeof(char *));
    /* Silently make room for \0 */
    str = ckd_calloc(arg_max_length + 1, sizeof(char));
    quoting = 0;

    do {
        /* Handle arguments that are commented out */
        if (len == 0 && argc % 2 == 0) {
            while (ch == '#') {
                /* Skip everything until newline */
                for (ch = fgetc(file); ch != EOF && ch != '\n'; ch = fgetc(file)) ;
                /* Skip to the next interesting character */
                for (ch = fgetc(file); ch != EOF && strchr(separator, ch); ch = fgetc(file)) ;
            }

            /* Check if we are at the last line (without anything interesting in it) */
            if (ch == EOF)
                break;
        }

        /* Handle quoted arguments */
        if (ch == '"' || ch == '\'') {
            if (quoting == ch) /* End a quoted section with the same type */
                quoting = 0;
            else if (quoting) {
                E_ERROR("Nesting quotations is not supported!\n");
                rv = 1;
                break;
            }
            else
                quoting = ch; /* Start a quoted section */
        }
        else if (ch == EOF || (!quoting && strchr(separator, ch))) {
            /* Reallocate argv so it is big enough to contain all the arguments */
            if (argc >= argv_size) {
                char **tmp_argv;
                if (!(tmp_argv =
                       ckd_realloc(f_argv, argv_size * 2 * sizeof(char *)))) {
                    rv = 1;
                    break;
                }
                f_argv = tmp_argv;
                argv_size *= 2;
            }
            /* Add the string to the list of arguments */
            f_argv[argc] = ckd_salloc(str);
            len = 0;
            str[0] = '\0';
            argc++;

            if (quoting)
                E_WARN("Unclosed quotation, having EOF close it...\n");

            /* Skip to the next interesting character */
            for (; ch != EOF && strchr(separator, ch); ch = fgetc(file)) ;

            if (ch == EOF)
                break;

            /* We already have the next character */
            continue;
        }
        else {
            if (len >= arg_max_length) {
 #import "OpenEarsStaticAnalysisToggle.h"
#ifdef STATICANALYZEDEPENDENCIES
#define __clang_analyzer__ 1
#endif
#if !defined(__clang_analyzer__) || defined(STATICANALYZEDEPENDENCIES)
#undef __clang_analyzer__
                
                /* Make room for more chars (including the \0 !) */
                char *tmp_str = str;
                if ((tmp_str = ckd_realloc(str, (1 + arg_max_length * 2) * sizeof(char))) == NULL) {
                    rv = 1;
                    break;
                }
                str = tmp_str;
                arg_max_length *= 2;
#endif                
            }
            /* Add the char to the argument string */
            str[len++] = ch;
            /* Always null terminate */
            str[len] = '\0';
        }

        ch = fgetc(file);
    } while (1);

    fclose(file);

    ckd_free(str);

    if (rv) {
        for (ch = 0; ch < argc; ++ch)
            ckd_free(f_argv[ch]);
        ckd_free(f_argv);
        return NULL;
    }

    return parse_options(inout_cmdln, defn, argc, f_argv, strict);
}
Ejemplo n.º 9
0
int
ps_decoder_test(cmd_ln_t *config, char const *sname, char const *expected)
{
    ps_decoder_t *ps;
    mfcc_t **cepbuf;
    FILE *rawfh;
    int16 *buf;
    int16 const *bptr;
    size_t nread;
    size_t nsamps;
    int32 nfr, i, score, prob;
    char const *hyp;
    char const *uttid;
    double n_speech, n_cpu, n_wall;
    ps_seg_t *seg;

    TEST_ASSERT(ps = ps_init(config));
    /* Test it first with pocketsphinx_decode_raw() */
    TEST_ASSERT(rawfh = fopen(DATADIR "/goforward.raw", "rb"));
    ps_decode_raw(ps, rawfh, "goforward", -1);
    hyp = ps_get_hyp(ps, &score, &uttid);
    prob = ps_get_prob(ps, &uttid);
    printf("%s (%s): %s (%d, %d)\n", sname, uttid, hyp, score, prob);
    TEST_EQUAL(0, strcmp(hyp, expected));
    TEST_ASSERT(prob <= 0);
    ps_get_utt_time(ps, &n_speech, &n_cpu, &n_wall);
    printf("%.2f seconds speech, %.2f seconds CPU, %.2f seconds wall\n",
           n_speech, n_cpu, n_wall);
    printf("%.2f xRT (CPU), %.2f xRT (elapsed)\n",
           n_cpu / n_speech, n_wall / n_speech);

    /* Test it with ps_process_raw() */
    clearerr(rawfh);
    fseek(rawfh, 0, SEEK_END);
    nsamps = ftell(rawfh) / sizeof(*buf);
    fseek(rawfh, 0, SEEK_SET);
    TEST_EQUAL(0, ps_start_utt(ps, NULL));
    nsamps = 2048;
    buf = ckd_calloc(nsamps, sizeof(*buf));
    while (!feof(rawfh)) {
        nread = fread(buf, sizeof(*buf), nsamps, rawfh);
        ps_process_raw(ps, buf, nread, FALSE, FALSE);
    }
    TEST_EQUAL(0, ps_end_utt(ps));
    hyp = ps_get_hyp(ps, &score, &uttid);
    prob = ps_get_prob(ps, &uttid);
    printf("%s (%s): %s (%d, %d)\n", sname, uttid, hyp, score, prob);
    TEST_EQUAL(0, strcmp(uttid, "000000000"));
    TEST_EQUAL(0, strcmp(hyp, expected));
    ps_get_utt_time(ps, &n_speech, &n_cpu, &n_wall);
    printf("%.2f seconds speech, %.2f seconds CPU, %.2f seconds wall\n",
           n_speech, n_cpu, n_wall);
    printf("%.2f xRT (CPU), %.2f xRT (elapsed)\n",
           n_cpu / n_speech, n_wall / n_speech);

    /* Now read the whole file and produce an MFCC buffer. */
    clearerr(rawfh);
    fseek(rawfh, 0, SEEK_END);
    nsamps = ftell(rawfh) / sizeof(*buf);
    fseek(rawfh, 0, SEEK_SET);
    bptr = buf = ckd_realloc(buf, nsamps * sizeof(*buf));
    TEST_EQUAL(nsamps, fread(buf, sizeof(*buf), nsamps, rawfh));
    fe_process_frames(ps->acmod->fe, &bptr, &nsamps, NULL, &nfr);
    cepbuf = ckd_calloc_2d(nfr + 1,
                   fe_get_output_size(ps->acmod->fe),
                   sizeof(**cepbuf));
    fe_start_utt(ps->acmod->fe);
    fe_process_frames(ps->acmod->fe, &bptr, &nsamps, cepbuf, &nfr);
    fe_end_utt(ps->acmod->fe, cepbuf[nfr], &i);

    /* Decode it with process_cep() */
    TEST_EQUAL(0, ps_start_utt(ps, NULL));
    for (i = 0; i < nfr; ++i) {
        ps_process_cep(ps, cepbuf + i, 1, FALSE, FALSE);
    }
    TEST_EQUAL(0, ps_end_utt(ps));
    hyp = ps_get_hyp(ps, &score, &uttid);
    prob = ps_get_prob(ps, &uttid);
    printf("%s (%s): %s (%d, %d)\n", sname, uttid, hyp, score, prob);
    TEST_EQUAL(0, strcmp(uttid, "000000001"));
    TEST_EQUAL(0, strcmp(hyp, expected));
    TEST_ASSERT(prob <= 0);
    for (seg = ps_seg_iter(ps, &score); seg;
         seg = ps_seg_next(seg)) {
        char const *word;
        int sf, ef;
        int32 post, lscr, ascr, lback;

        word = ps_seg_word(seg);
        ps_seg_frames(seg, &sf, &ef);
        post = ps_seg_prob(seg, &ascr, &lscr, &lback);
        printf("%s (%d:%d) P(w|o) = %f ascr = %d lscr = %d lback = %d\n", word, sf, ef,
               logmath_exp(ps_get_logmath(ps), post), ascr, lscr, lback);
        TEST_ASSERT(post <= 2); // Due to numerical errors with float it sometimes could go out of 0
    }

    ps_get_utt_time(ps, &n_speech, &n_cpu, &n_wall);
    printf("%.2f seconds speech, %.2f seconds CPU, %.2f seconds wall\n",
           n_speech, n_cpu, n_wall);
    printf("%.2f xRT (CPU), %.2f xRT (elapsed)\n",
           n_cpu / n_speech, n_wall / n_speech);
    ps_get_all_time(ps, &n_speech, &n_cpu, &n_wall);
    printf("TOTAL: %.2f seconds speech, %.2f seconds CPU, %.2f seconds wall\n",
           n_speech, n_cpu, n_wall);
    printf("TOTAL: %.2f xRT (CPU), %.2f xRT (elapsed)\n",
           n_cpu / n_speech, n_wall / n_speech);

    fclose(rawfh);
    ps_free(ps);
    cmd_ln_free_r(config);
    ckd_free_2d(cepbuf);
    ckd_free(buf);

    return 0;
}
Ejemplo n.º 10
0
int
main(int argc, char *argv[])
{
    acmod_t *acmod;
    logmath_t *lmath;
    cmd_ln_t *config;
    FILE *rawfh;
    int16 *buf;
    int16 const *bptr;
    mfcc_t **cepbuf, **cptr;
    size_t nread, nsamps;
    int nfr;
    int frame_counter;
    int bestsen1[270];

    lmath = logmath_init(1.0001, 0, 0);
    config = cmd_ln_init(NULL, ps_args(), TRUE,
                 "-featparams", MODELDIR "/hmm/en_US/hub4wsj_sc_8k/feat.params",
                 "-mdef", MODELDIR "/hmm/en_US/hub4wsj_sc_8k/mdef",
                 "-mean", MODELDIR "/hmm/en_US/hub4wsj_sc_8k/means",
                 "-var", MODELDIR "/hmm/en_US/hub4wsj_sc_8k/variances",
                 "-tmat", MODELDIR "/hmm/en_US/hub4wsj_sc_8k/transition_matrices",
                 "-sendump", MODELDIR "/hmm/en_US/hub4wsj_sc_8k/sendump",
                 "-compallsen", "true",
                 "-cmn", "prior",
                 "-tmatfloor", "0.0001",
                 "-mixwfloor", "0.001",
                 "-varfloor", "0.0001",
                 "-mmap", "no",
                 "-topn", "4",
                 "-ds", "1",
                 "-input_endian", "little",
                 "-samprate", "16000", NULL);
    TEST_ASSERT(config);
    TEST_ASSERT(acmod = acmod_init(config, lmath, NULL, NULL));
    cmn_prior_set(acmod->fcb->cmn_struct, prior);

    nsamps = 2048;
    frame_counter = 0;
    buf = ckd_calloc(nsamps, sizeof(*buf));
    TEST_ASSERT(rawfh = fopen(DATADIR "/goforward.raw", "rb"));
    TEST_EQUAL(0, acmod_start_utt(acmod));
    E_INFO("Incremental(2048):\n");
    while (!feof(rawfh)) {
        nread = fread(buf, sizeof(*buf), nsamps, rawfh);
        bptr = buf;
        while ((nfr = acmod_process_raw(acmod, &bptr, &nread, FALSE)) > 0 || nread > 0) {
            int16 const *senscr;
            int16 best_score;
            int frame_idx = -1, best_senid;
            while (acmod->n_feat_frame > 0) {
                senscr = acmod_score(acmod, &frame_idx);
                acmod_advance(acmod);
                best_score = acmod_best_score(acmod, &best_senid);
                E_INFO("Frame %d best senone %d score %d\n",
                       frame_idx, best_senid, best_score);
                TEST_EQUAL(frame_counter, frame_idx);
                if (frame_counter < 190)
                    bestsen1[frame_counter] = best_score;
                ++frame_counter;
                frame_idx = -1;
            }
        }
    }
    TEST_EQUAL(0, acmod_end_utt(acmod));
    nread = 0;
    {
        int16 const *senscr;
        int16 best_score;
        int frame_idx = -1, best_senid;
        while (acmod->n_feat_frame > 0) {
            senscr = acmod_score(acmod, &frame_idx);
            acmod_advance(acmod);
            best_score = acmod_best_score(acmod, &best_senid);
            E_INFO("Frame %d best senone %d score %d\n",
                   frame_idx, best_senid, best_score);
            if (frame_counter < 190)
                bestsen1[frame_counter] = best_score;
            TEST_EQUAL(frame_counter, frame_idx);
            ++frame_counter;
            frame_idx = -1;
        }
    }

    /* Now try to process the whole thing at once. */
    E_INFO("Whole utterance:\n");
    cmn_prior_set(acmod->fcb->cmn_struct, prior);
    nsamps = ftell(rawfh) / sizeof(*buf);
    clearerr(rawfh);
    fseek(rawfh, 0, SEEK_SET);
    buf = ckd_realloc(buf, nsamps * sizeof(*buf));
    TEST_EQUAL(nsamps, fread(buf, sizeof(*buf), nsamps, rawfh));
    bptr = buf;
    TEST_EQUAL(0, acmod_start_utt(acmod));
    acmod_process_raw(acmod, &bptr, &nsamps, TRUE);
    TEST_EQUAL(0, acmod_end_utt(acmod));
    {
        int16 const *senscr;
        int16 best_score;
        int frame_idx = -1, best_senid;
        frame_counter = 0;
        while (acmod->n_feat_frame > 0) {
            senscr = acmod_score(acmod, &frame_idx);
            acmod_advance(acmod);
            best_score = acmod_best_score(acmod, &best_senid);
            E_INFO("Frame %d best senone %d score %d\n",
               frame_idx, best_senid, best_score);
            if (frame_counter < 190)
                TEST_EQUAL_LOG(best_score, bestsen1[frame_counter]);
            TEST_EQUAL(frame_counter, frame_idx);
            ++frame_counter;
            frame_idx = -1;
        }
    }

    /* Now process MFCCs and make sure we get the same results. */
    cepbuf = ckd_calloc_2d(frame_counter,
                   fe_get_output_size(acmod->fe),
                   sizeof(**cepbuf));
    fe_start_utt(acmod->fe);
    nsamps = ftell(rawfh) / sizeof(*buf);
    bptr = buf;
    nfr = frame_counter;
    fe_process_frames(acmod->fe, &bptr, &nsamps, cepbuf, &nfr);
    fe_end_utt(acmod->fe, cepbuf[frame_counter-1], &nfr);

    E_INFO("Incremental(MFCC):\n");
    cmn_prior_set(acmod->fcb->cmn_struct, prior);
    TEST_EQUAL(0, acmod_start_utt(acmod));
    cptr = cepbuf;
    nfr = frame_counter;
    frame_counter = 0;
    while ((acmod_process_cep(acmod, &cptr, &nfr, FALSE)) > 0) {
        int16 const *senscr;
        int16 best_score;
        int frame_idx = -1, best_senid;
        while (acmod->n_feat_frame > 0) {
            senscr = acmod_score(acmod, &frame_idx);
            acmod_advance(acmod);
            best_score = acmod_best_score(acmod, &best_senid);
            E_INFO("Frame %d best senone %d score %d\n",
                   frame_idx, best_senid, best_score);
            TEST_EQUAL(frame_counter, frame_idx);
            if (frame_counter < 190)
                TEST_EQUAL_LOG(best_score, bestsen1[frame_counter]);
            ++frame_counter;
            frame_idx = -1;
        }
    }
    TEST_EQUAL(0, acmod_end_utt(acmod));
    nfr = 0;
    acmod_process_cep(acmod, &cptr, &nfr, FALSE);
    {
        int16 const *senscr;
        int16 best_score;
        int frame_idx = -1, best_senid;
        while (acmod->n_feat_frame > 0) {
            senscr = acmod_score(acmod, &frame_idx);
            acmod_advance(acmod);
            best_score = acmod_best_score(acmod, &best_senid);
            E_INFO("Frame %d best senone %d score %d\n",
                   frame_idx, best_senid, best_score);
            TEST_EQUAL(frame_counter, frame_idx);
            if (frame_counter < 190)
                TEST_EQUAL_LOG(best_score, bestsen1[frame_counter]);
            ++frame_counter;
            frame_idx = -1;
        }
    }

    /* Note that we have to process the whole thing again because
     * !#@$@ s2mfc2feat modifies its argument (not for long) */
    fe_start_utt(acmod->fe);
    nsamps = ftell(rawfh) / sizeof(*buf);
    bptr = buf;
    nfr = frame_counter;
    fe_process_frames(acmod->fe, &bptr, &nsamps, cepbuf, &nfr);
    fe_end_utt(acmod->fe, cepbuf[frame_counter-1], &nfr);

    E_INFO("Whole utterance (MFCC):\n");
    cmn_prior_set(acmod->fcb->cmn_struct, prior);
    TEST_EQUAL(0, acmod_start_utt(acmod));
    cptr = cepbuf;
    nfr = frame_counter;
    acmod_process_cep(acmod, &cptr, &nfr, TRUE);
    TEST_EQUAL(0, acmod_end_utt(acmod));
    {
        int16 const *senscr;
        int16 best_score;
        int frame_idx = -1, best_senid;
        frame_counter = 0;
        while (acmod->n_feat_frame > 0) {
            senscr = acmod_score(acmod, &frame_idx);
            acmod_advance(acmod);
            best_score = acmod_best_score(acmod, &best_senid);
            E_INFO("Frame %d best senone %d score %d\n",
                   frame_idx, best_senid, best_score);
            if (frame_counter < 190)
                TEST_EQUAL_LOG(best_score, bestsen1[frame_counter]);
            TEST_EQUAL(frame_counter, frame_idx);
            ++frame_counter;
            frame_idx = -1;
        }
    }

    E_INFO("Rewound (MFCC):\n");
    TEST_EQUAL(0, acmod_rewind(acmod));
    {
        int16 const *senscr;
        int16 best_score;
        int frame_idx = -1, best_senid;
        frame_counter = 0;
        while (acmod->n_feat_frame > 0) {
            senscr = acmod_score(acmod, &frame_idx);
            acmod_advance(acmod);
            best_score = acmod_best_score(acmod, &best_senid);
            E_INFO("Frame %d best senone %d score %d\n",
                   frame_idx, best_senid, best_score);
            if (frame_counter < 190)
                TEST_EQUAL_LOG(best_score, bestsen1[frame_counter]);
            TEST_EQUAL(frame_counter, frame_idx);
            ++frame_counter;
            frame_idx = -1;
        }
    }

    /* Clean up, go home. */
    ckd_free_2d(cepbuf);
    fclose(rawfh);
    ckd_free(buf);
    acmod_free(acmod);
    logmath_free(lmath);
    cmd_ln_free_r(config);
    return 0;
}
Ejemplo n.º 11
0
s3wid_t
dict_add_word(dict_t * d, char const *word, s3cipid_t const * p, int32 np)
{
    int32 len;
    dictword_t *wordp;
    s3wid_t newwid;
    char *wword;

    if (d->n_word >= d->max_words) {
        E_INFO("Reallocating to %d KiB for word entries\n",
               (d->max_words + S3DICT_INC_SZ) * sizeof(dictword_t) / 1024);
        d->word =
            (dictword_t *) ckd_realloc(d->word,
                                       (d->max_words +
                                        S3DICT_INC_SZ) * sizeof(dictword_t));
        d->max_words = d->max_words + S3DICT_INC_SZ;
    }

    wordp = d->word + d->n_word;
    wordp->word = (char *) ckd_salloc(word);    /* Freed in dict_free */

    /* Associate word string with d->n_word in hash table */
    if (hash_table_enter_int32(d->ht, wordp->word, d->n_word) != d->n_word) {
        ckd_free(wordp->word);
        wordp->word = NULL;
        return BAD_S3WID;
    }

    /* Fill in word entry, and set defaults */
    if (p && (np > 0)) {
        wordp->ciphone = (s3cipid_t *) ckd_malloc(np * sizeof(s3cipid_t));      /* Freed in dict_free */
        memcpy(wordp->ciphone, p, np * sizeof(s3cipid_t));
        wordp->pronlen = np;
    }
    else {
        wordp->ciphone = NULL;
        wordp->pronlen = 0;
    }
    wordp->alt = BAD_S3WID;
    wordp->basewid = d->n_word;

    /* Determine base/alt wids */
    wword = ckd_salloc(word);
    if ((len = dict_word2basestr(wword)) > 0) {
	int32 w;

        /* Truncated to a baseword string; find its ID */
        if (hash_table_lookup_int32(d->ht, wword, &w) < 0) {
            E_ERROR("Missing base word for: %s\n", word);
            ckd_free(wword);
            ckd_free(wordp->word);
            wordp->word = NULL;
            return BAD_S3WID;
        }

        /* Link into alt list */
        wordp->basewid = w;
        wordp->alt = d->word[w].alt;
        d->word[w].alt = d->n_word;
    }
    ckd_free(wword);

    newwid = d->n_word++;

    return newwid;
}
Ejemplo n.º 12
0
static int32
dict_read(FILE * fp, dict_t * d)
{
    lineiter_t *li;
    char **wptr;
    s3cipid_t *p;
    int32 lineno, nwd;
    s3wid_t w;
    int32 i, maxwd;
    size_t stralloc, phnalloc;

    maxwd = 512;
    p = (s3cipid_t *) ckd_calloc(maxwd + 4, sizeof(*p));
    wptr = (char **) ckd_calloc(maxwd, sizeof(char *)); /* Freed below */

    lineno = 0;
    stralloc = phnalloc = 0;
    for (li = lineiter_start(fp); li; li = lineiter_next(li)) {
        lineno++;
        if (0 == strncmp(li->buf, "##", 2)
            || 0 == strncmp(li->buf, ";;", 2))
            continue;

        if ((nwd = str2words(li->buf, wptr, maxwd)) < 0) {
            /* Increase size of p, wptr. */
            nwd = str2words(li->buf, NULL, 0);
            assert(nwd > maxwd); /* why else would it fail? */
            maxwd = nwd;
            p = (s3cipid_t *) ckd_realloc(p, (maxwd + 4) * sizeof(*p));
            wptr = (char **) ckd_realloc(wptr, maxwd * sizeof(*wptr));
        }

        if (nwd == 0)           /* Empty line */
            continue;
        /* wptr[0] is the word-string and wptr[1..nwd-1] the pronunciation sequence */
        if (nwd == 1) {
            E_ERROR("Line %d: No pronunciation for word %s; ignored\n",
                    lineno, wptr[0]);
            continue;
        }


        /* Convert pronunciation string to CI-phone-ids */
        for (i = 1; i < nwd; i++) {
            p[i - 1] = dict_ciphone_id(d, wptr[i]);
            if (NOT_S3CIPID(p[i - 1])) {
                E_ERROR("Line %d: Bad ciphone: %s; word %s ignored\n",
                        lineno, wptr[i], wptr[0]);
                break;
            }
        }

        if (i == nwd) {         /* All CI-phones successfully converted to IDs */
            w = dict_add_word(d, wptr[0], p, nwd - 1);
            if (NOT_S3WID(w))
                E_ERROR
                    ("Line %d: dict_add_word (%s) failed (duplicate?); ignored\n",
                     lineno, wptr[0]);
            else {
                stralloc += strlen(d->word[w].word);
                phnalloc += d->word[w].pronlen * sizeof(s3cipid_t);
            }
        }
    }
    E_INFO("Allocated %d KiB for strings, %d KiB for phones\n",
           (int)stralloc / 1024, (int)phnalloc / 1024);
    ckd_free(p);
    ckd_free(wptr);

    return 0;
}
Ejemplo n.º 13
0
void kb_setlm(char* lmname,kb_t* kb)
{
  lmset_t* lms;
  kbcore_t* kbc=NULL;
  int i;
  int j;
  int n;
  /*  s3wid_t dictid;*/

  kbc=kb->kbcore;
  lms=kbc->lmset;

  E_INFO("Inside kb_setlm\n");
  kbc->lm=NULL;
  for(j=0;j<kb->n_lextree;j++){
    kb->ugtree[j]=NULL;
  }

  E_INFO("Inside kb_setlm\n");
  if(lms!=NULL || cmd_ln_str("-lmctlfn")){
    for(i=0;i<kbc->n_lm;i++){
      if(!strcmp(lmname,lms[i].name)){
	/* Point the current lm to a particular lm */
	kbc->lm=lms[i].lm;

	for(j=0;j<kb->n_lextree;j++){
	  kb->ugtree[j]=kb->ugtreeMulti[i*kb->n_lextree+j];
	}

	break;
      }
    }
    if(kbc->lm==NULL){
      E_ERROR("LM name %s cannot be found! Fall back to use base LM model\n",lmname);
      kbc->lm=lms[0].lm;
      for(j=0;j<kb->n_lextree;j++){
	kb->ugtree[j]=kb->ugtreeMulti[j];
      }
    }
  }

  E_INFO("Current LM name %s.\n",lms[i].name);
  /*  if((kb->vithist->lms2vh_root=
     (vh_lms2vh_t**)ckd_realloc(kb->vithist->lms2vh_root,
				lm_n_ug(kbc->lm)*sizeof(vh_lms2vh_t *)
				))==NULL) 
    {
      E_FATAL("failed to allocate memory for vithist\n");
      }*/


  n = ((kb->ugtree[0]->n_node) + (kb->fillertree[0]->n_node)) * kb->n_lextree;
  n /= kb->hmm_hist_binsize;
  kb->hmm_hist_bins = n+1;
  kb->hmm_hist = (int32 *) ckd_realloc (kb->hmm_hist,(n+1)*sizeof(int32));	/* Really no need for +1 */

  E_INFO("Current LM name %s\n",lms[i].name);
  E_INFO("LM ug size %d\n",kbc->lm->n_ug);
  E_INFO("LM bg size %d\n",kbc->lm->n_bg);
  E_INFO("LM tg size %d\n",kbc->lm->n_tg);
  E_INFO("HMM history bin size %d\n", n+1);

  for(j=0;j<kb->n_lextree;j++){
    E_INFO("Lextrees(%d), %d nodes(ug)\n",
	   kb->n_lextree, lextree_n_node(kb->ugtree[j]));
  }

  /*  for (n = 0; n < dict_size(kbcore_dict(kbc)); n++){
    E_INFO("Index %d, map %d word %s\n",n,kbc->lm->dict2lmwid[n],dict_wordstr(kbcore_dict(kbc),n));
    }*/

}
Ejemplo n.º 14
0
int
ngram_model_recode(ngram_model_t *model, const char *from, const char *to)
{
    iconv_t ic;
    char *outbuf;
    size_t maxlen;
    int i, writable;
    hash_table_t *new_wid;

    /* FIXME: Need to do a special case thing for the GB-HEX encoding
     * used in Sphinx3 Mandarin models. */
    if ((ic = iconv_open(to, from)) == (iconv_t)-1) {
        E_ERROR_SYSTEM("iconv_open() failed");
        return -1;
    }
    /* iconv(3) is a piece of crap and won't accept a NULL out buffer,
     * unlike wcstombs(3). So we have to either call it over and over
     * again until our buffer is big enough, or call it with a huge
     * buffer and then copy things back to the output.  We will use a
     * mix of these two approaches here.  We'll keep a single big
     * buffer around, and expand it as necessary.
     */
    maxlen = 0;
    for (i = 0; i < model->n_words; ++i) {
        if (strlen(model->word_str[i]) > maxlen)
            maxlen = strlen(model->word_str[i]);
    }
    /* Were word strings already allocated? */
    writable = model->writable;
    /* Either way, we are going to allocate some word strings. */
    model->writable = TRUE;
    /* Really should be big enough except for pathological cases. */
    maxlen = maxlen * sizeof(int) + 15;
    outbuf = ckd_calloc(maxlen, 1);
    /* And, don't forget, we need to rebuild the word to unigram ID
     * mapping. */
    new_wid = hash_table_new(model->n_words, FALSE);
    for (i = 0; i < model->n_words; ++i) {
        ICONV_CONST char *in;
        char *out;
        size_t inleft, outleft, result;

    start_conversion:
        in = (ICONV_CONST char *)model->word_str[i];
        /* Yes, this assumes that we don't have any NUL bytes. */
        inleft = strlen(in);
        out = outbuf;
        outleft = maxlen;

        while ((result = iconv(ic, &in, &inleft, &out, &outleft)) == (size_t)-1) {
            if (errno != E2BIG) {
                /* FIXME: if we already converted any words, then they
                 * are going to be in an inconsistent state. */
                E_ERROR_SYSTEM("iconv() failed");
                ckd_free(outbuf);
                hash_table_free(new_wid);
                return -1;
            }
            /* Reset the internal state of conversion. */
            iconv(ic, NULL, NULL, NULL, NULL);
            /* Make everything bigger. */
            maxlen *= 2;
            out = outbuf = ckd_realloc(outbuf, maxlen);
            /* Reset the input pointers. */
            in = (ICONV_CONST char *)model->word_str[i];
            inleft = strlen(in);
        }

        /* Now flush a shift-out sequence, if any. */
        if ((result = iconv(ic, NULL, NULL, &out, &outleft)) == (size_t)-1) {
            if (errno != E2BIG) {
                /* FIXME: if we already converted any words, then they
                 * are going to be in an inconsistent state. */
                E_ERROR_SYSTEM("iconv() failed (state reset sequence)");
                ckd_free(outbuf);
                hash_table_free(new_wid);
                return -1;
            }
            /* Reset the internal state of conversion. */
            iconv(ic, NULL, NULL, NULL, NULL);
            /* Make everything bigger. */
            maxlen *= 2;
            outbuf = ckd_realloc(outbuf, maxlen);
            /* Be very evil. */
            goto start_conversion;
        }

        result = maxlen - outleft;
        /* Okay, that was hard, now let's go shopping. */
        if (writable) {
            /* Grow or shrink the output string as necessary. */
            model->word_str[i] = ckd_realloc(model->word_str[i], result + 1);
            model->word_str[i][result] = '\0';
        }
        else {
            /* It actually was not allocated previously, so do that now. */
            model->word_str[i] = ckd_calloc(result + 1, 1);
        }
        /* Copy the new thing in. */
        memcpy(model->word_str[i], outbuf, result);

        /* Now update the hash table.  We might have terrible
         * collisions if a non-reversible conversion was requested.,
         * so warn about them. */
        if (hash_table_enter_int32(new_wid, model->word_str[i], i) != i) {
            E_WARN("Duplicate word in dictionary after conversion: %s\n",
                   model->word_str[i]);
        }
    }
    ckd_free(outbuf);
    iconv_close(ic);
    /* Swap out the hash table. */
    hash_table_free(model->wid);
    model->wid = new_wid;

    return 0;
}
Ejemplo n.º 15
0
s3wid_t
dict_add_word(dict_t * d, char *word, s3cipid_t * p, int32 np)
{
    int32 len;
    dictword_t *wordp;
    s3wid_t newwid;

    if (d->n_word >= d->max_words) {
        E_INFO
            ("Dictionary max size (%d) exceeded; reallocate another entries %d \n",
             d->max_words, DICT_INC_SZ);
        d->word =
            (dictword_t *) ckd_realloc(d->word,
                                       (d->max_words +
                                        DICT_INC_SZ) * sizeof(dictword_t));
        d->max_words = d->max_words + DICT_INC_SZ;

        return (BAD_S3WID);
    }

    wordp = d->word + d->n_word;
    wordp->word = (char *) ckd_salloc(word);    /* Freed in dict_free */

    /* Associate word string with d->n_word in hash table */
    if (hash_table_enter(d->ht, wordp->word, (void *)(long)d->n_word) != (void *)(long)d->n_word) {
        ckd_free(wordp->word);
        return (BAD_S3WID);
    }

    /* Fill in word entry, and set defaults */
    if (p && (np > 0)) {
        wordp->ciphone = (s3cipid_t *) ckd_malloc(np * sizeof(s3cipid_t));      /* Freed in dict_free */
        memcpy(wordp->ciphone, p, np * sizeof(s3cipid_t));
        wordp->pronlen = np;
    }
    else {
        wordp->ciphone = NULL;
        wordp->pronlen = 0;
    }
    wordp->alt = BAD_S3WID;
    wordp->basewid = d->n_word;
    wordp->n_comp = 0;
    wordp->comp = NULL;

    /* Determine base/alt wids */
    if ((len = dict_word2basestr(word)) > 0) {
	void *val;
	s3wid_t w;

        /* Truncated to a baseword string; find its ID */
        if (hash_table_lookup(d->ht, word, &val) < 0) {
            word[len] = '(';    /* Get back the original word */
            E_FATAL("Missing base word for: %s\n", word);
        }
        else
            word[len] = '(';    /* Get back the original word */

        /* Link into alt list */
	w = (s3wid_t)(long)val;
        wordp->basewid = w;
        wordp->alt = d->word[w].alt;
        d->word[w].alt = d->n_word;
    }

    newwid = d->n_word++;

    return (newwid);
}