示例#1
0
文件: lmset.c 项目: Ankit77/cmusphinx
lmset_t *
lmset_read_ctl(const char *ctlfile,
               dict_t * dict,
               float64 lw, float64 wip, float64 uw,
               const char *lmdumpdir, logmath_t *logmath)
{
    FILE *ctlfp;
    FILE *tmp;
    char lmfile[4096], lmname[4096], str[4096];

    lmclass_set_t *lmclass_set;
    lmclass_t **lmclass, *cl;
    int32 n_lmclass, n_lmclass_used;
    int32 i;
    lm_t *lm;
    lmset_t *lms = NULL;
    tmp = NULL;

    E_INFO("Reading LM control file '%s'\n", ctlfile);
    if ((ctlfp = fopen(ctlfile, "r")) == NULL) {
	    E_ERROR_SYSTEM("Failed to open LM control file");
	    return NULL;
    }

    lmclass_set = lmclass_newset();

    lms = (lmset_t *) ckd_calloc(1, sizeof(lmset_t));
    lms->n_lm = 0;
    lms->n_alloc_lm = 0;

    if (fscanf(ctlfp, "%s", str) == 1) {
        if (strcmp(str, "{") == 0) {
            /* Load LMclass files */
            while ((fscanf(ctlfp, "%s", str) == 1)
                   && (strcmp(str, "}") != 0))
                lmclass_set = lmclass_loadfile(lmclass_set, str, logmath);

            if (strcmp(str, "}") != 0)
                E_FATAL("Unexpected EOF(%s)\n", ctlfile);

            if (fscanf(ctlfp, "%s", str) != 1)
                str[0] = '\0';
        }
    }
    else
        str[0] = '\0';

    /* Fill in dictionary word id information for each LMclass word */
    for (cl = lmclass_firstclass(lmclass_set);
         lmclass_isclass(cl); cl = lmclass_nextclass(lmclass_set, cl)) {

        /*
           For every words in the class, set the dictwid correctly 
           The following piece of code replace s2's kb_init_lmclass_dictwid (cl);
           doesn't do any checking even the id is a bad dict id. 
           This only sets the information in the lmclass_set, but not 
           lm-2-dict or dict-2-lm map.  In Sphinx 3, they are done in 
           wid_dict_lm_map in wid.c.
         */

        lmclass_word_t *w;
        int32 wid;
        for (w = lmclass_firstword(cl); lmclass_isword(w);
             w = lmclass_nextword(cl, w)) {
            wid = dict_wordid(dict, lmclass_getword(w));
#if 0
            E_INFO("In class %s, Word %s, wid %d\n", cl->name,
                   lmclass_getword(w), wid);
#endif
            lmclass_set_dictwid(w, wid);
        }
    }

    /* At this point if str[0] != '\0', we have an LM filename */

    n_lmclass = lmclass_get_nclass(lmclass_set);
    lmclass = (lmclass_t **) ckd_calloc(n_lmclass, sizeof(lmclass_t *));

    E_INFO("Number of LM class specified %d in file %s\n", n_lmclass,
           ctlfile);

    /* Read in one LM at a time */
    while (str[0] != '\0') {
        strcpy(lmfile, str);
        if (fscanf(ctlfp, "%s", lmname) != 1)
            E_FATAL("LMname missing after LMFileName '%s'\n", lmfile);

        n_lmclass_used = 0;

        if (fscanf(ctlfp, "%s", str) == 1) {
            if (strcmp(str, "{") == 0) {
                while ((fscanf(ctlfp, "%s", str) == 1) &&
                       (strcmp(str, "}") != 0)) {
                    if (n_lmclass_used >= n_lmclass) {
                        E_FATAL("Too many LM classes specified for '%s'\n",
                                lmfile);
                    }

                    lmclass[n_lmclass_used] =
                        lmclass_get_lmclass(lmclass_set, str);
                    if (!(lmclass_isclass(lmclass[n_lmclass_used])))
                        E_FATAL("LM class '%s' not found\n", str);
                    n_lmclass_used++;
                }
                if (strcmp(str, "}") != 0)
                    E_FATAL("Unexpected EOF(%s)\n", ctlfile);
                if (fscanf(ctlfp, "%s", str) != 1)
                    str[0] = '\0';
            }
        }
        else
            str[0] = '\0';

        lm = (lm_t *) lm_read_advance(lmfile, lmname, lw, wip, uw,
                                      dict_size(dict), NULL, 1, logmath, FALSE, FALSE);


        if (n_lmclass_used > 0) {
            E_INFO("Did I enter here?\n");
            lm_build_lmclass_info(lm, lw, uw, wip, n_lmclass_used,
                                  lmclass);
        }

        if (lms->n_lm == lms->n_alloc_lm) {
            lms->lmarray =
                (lm_t **) ckd_realloc(lms->lmarray,
                                      (lms->n_alloc_lm +
                                       LM_ALLOC_BLOCK) * sizeof(lm_t *));
            lms->n_alloc_lm += LM_ALLOC_BLOCK;
        }

        lms->lmarray[lms->n_lm] = lm;
        lms->n_lm += 1;
        E_INFO("%d %d\n", lms->n_alloc_lm, lms->n_lm);
    }

    assert(lms);
    assert(lms->lmarray);
    E_INFO("No. of LM set allocated %d, no. of LM %d \n", lms->n_alloc_lm,
           lms->n_lm);


    if (dict != NULL) {
        for (i = 0; i < lms->n_lm; i++) {
            assert(lms->lmarray[i]);
            assert(dict);
            if ((lms->lmarray[i]->dict2lmwid =
                 wid_dict_lm_map(dict, lms->lmarray[i], lw)) == NULL)
                E_FATAL
                    ("Dict/LM word-id mapping failed for LM index %d, named %s\n",
                     i, lmset_idx_to_name(lms, i));
        }
    }
    else {
        E_FATAL
            ("Dict is specified to be NULL (dict_init is not called before lmset_read_lm?), dict2lmwid is not built inside lmset_read_lm\n");
    }


    ckd_free(lmclass_set);
    ckd_free(lmclass);
    fclose(ctlfp);
    return lms;
}
void kb (int argc, char *argv[],
         float ip,	/* word insertion penalty */
         float lw,	/* langauge weight */
         float pip)	/* phone insertion penalty */
{
    char *pname = argv[0];
    char hmm_file_name[256];
    int32 num_phones, num_ci_phones;
    int32 i, use_darpa_lm;

    /* FIXME: This is evil.  But if we do it, let's prototype it
       somewhere, OK? */
    unlimit ();		/* Remove memory size limits */

    language_weight = lw;
    insertion_penalty = ip;
    phone_insertion_penalty = pip;

    pconf (argc, argv, kb_param, 0, 0, 0);

    if ((phone_file_name == 0) ||
            (dict_file_name == 0))
        pusage (pname, (Config_t *)kb_param);

    log_info("%s(%d): Reading phone file [%s]\n",
             __FILE__, __LINE__, phone_file_name);
    if (phone_read (phone_file_name))
        exit (-1);
    if (useWDPhonesOnly)
        phone_add_diphones();

    num_ci_phones = phoneCiCount();

    /* Read the distribution map file */
    log_info("%s(%d): Reading map file [%s]\n", __FILE__, __LINE__, mapFileName);

    read_map (mapFileName, TRUE /* useCiTrans compress */);
    log_info("%s(%d): Reading dict file [%s]\n",
             __FILE__, __LINE__, dict_file_name);

    word_dict = dict_new ();
    if (dict_read (word_dict, dict_file_name, phrase_dict_file_name,
                   noise_dict_file_name, !useWDPhonesOnly))
        exit (-1);

    use_darpa_lm = TRUE;

    if (use_darpa_lm) {
        lmSetStartSym (lm_start_sym);
        lmSetEndSym (lm_end_sym);

        /*
         * Read control file describing multiple LMs, if specified.
         * File format (optional stuff is indicated by enclosing in []):
         *
         *   [{ LMClassFileName LMClassFilename ... }]
         *   TrigramLMFileName LMName [{ LMClassName LMClassName ... }]
         *   TrigramLMFileName LMName [{ LMClassName LMClassName ... }]
         *   ...
         * (There should be whitespace around the { and } delimiters.)
         *
         * This is an extension of the older format that had only TrigramLMFilenName
         * and LMName pairs.  The new format allows a set of LMClass files to be read
         * in and referred to by the trigram LMs.  (Incidentally, if one wants to use
         * LM classes in a trigram LM, one MUST use the -lmctlfn flag.  It is not
         * possible to read in a class-based trigram LM using the -lmfn flag.)
         *
         * No "comments" allowed in this file.
         */
        if (lm_ctl_filename) {
            FILE *ctlfp;
            char lmfile[4096], lmname[4096], str[4096];
            lmclass_set_t lmclass_set;
            lmclass_t *lmclass, cl;
            int32 n_lmclass, n_lmclass_used;

            lmclass_set = lmclass_newset();

            E_INFO("Reading LM control file '%s'\n", lm_ctl_filename);

            ctlfp = CM_fopen (lm_ctl_filename, "r");
            if (fscanf (ctlfp, "%s", str) == 1) {
                if (strcmp (str, "{") == 0) {
                    /* Load LMclass files */
                    while ((fscanf (ctlfp, "%s", str) == 1) && (strcmp (str, "}") != 0))
                        lmclass_set = lmclass_loadfile (lmclass_set, str);

                    if (strcmp (str, "}") != 0)
                        E_FATAL("Unexpected EOF(%s)\n", lm_ctl_filename);

                    if (fscanf (ctlfp, "%s", str) != 1)
                        str[0] = '\0';
                }
            } else
                str[0] = '\0';

            /* Fill in dictionary word id information for each LMclass word */
            for (cl = lmclass_firstclass(lmclass_set);
                    lmclass_isclass(cl);
                    cl = lmclass_nextclass(lmclass_set, cl)) {
                kb_init_lmclass_dictwid (cl);
            }

            /* At this point if str[0] != '\0', we have an LM filename */
            n_lmclass = lmclass_get_nclass(lmclass_set);
            lmclass = (lmclass_t *) CM_calloc (n_lmclass, sizeof(lmclass_t));

            /* Read in one LM at a time */
            while (str[0] != '\0') {
                strcpy (lmfile, str);
                if (fscanf (ctlfp, "%s", lmname) != 1)
                    E_FATAL("LMname missing after LMFileName '%s'\n", lmfile);

                n_lmclass_used = 0;

                if (fscanf (ctlfp, "%s", str) == 1) {
                    if (strcmp (str, "{") == 0) {
                        /* LM uses classes; read their names */
                        while ((fscanf (ctlfp, "%s", str) == 1) &&
                                (strcmp (str, "}") != 0)) {
                            if (n_lmclass_used >= n_lmclass)
                                E_FATAL("Too many LM classes specified for '%s'\n",
                                        lmfile);
                            lmclass[n_lmclass_used] = lmclass_get_lmclass (lmclass_set,
                                                      str);
                            if (! (lmclass_isclass(lmclass[n_lmclass_used])))
                                E_FATAL("LM class '%s' not found\n", str);
                            n_lmclass_used++;
                        }
                        if (strcmp (str, "}") != 0)
                            E_FATAL("Unexpected EOF(%s)\n", lm_ctl_filename);

                        if (fscanf (ctlfp, "%s", str) != 1)
                            str[0] = '\0';
                    }
                } else
                    str[0] = '\0';

                if (n_lmclass_used > 0)
                    lm_read_clm (lmfile, lmname,
                                 language_weight, unigramWeight, insertion_penalty,
                                 lmclass, n_lmclass_used);
                else
                    lm_read (lmfile, lmname,
                             language_weight, unigramWeight, insertion_penalty);
            }

            fclose (ctlfp);
            NoLangModel = FALSE;
        }

        /* Read "base" LM file, if specified */
        if (lm_file_name) {
            lmSetStartSym (lm_start_sym);
            lmSetEndSym (lm_end_sym);
            lm_read (lm_file_name, "", language_weight, unigramWeight, insertion_penalty);

            /* Make initial OOV list known to this base LM */
            lm_init_oov ();

            NoLangModel = FALSE;
        }

#ifdef USE_ILM
        /* Init ILM module (non-std-Darpa LM, eg ug/bg cache LM) */
        ilm_init ();
#endif
    }

#if 0
    /* Compute the phrase lm probabilities */
    computePhraseLMProbs ();
#endif

    num_phones = phone_count ();
    numSmds = hmm_num_sseq();
    smds = (SMD *) CM_calloc (numSmds, sizeof (SMD));

    /*
     * Read the hmm's into the SMD structures
     */
    if (useBigHmmFiles) {
        for (i = 0; i < num_ci_phones; i++) {
            sprintf (hmm_file_name, "%s.%s", phone_from_id (i),
                     hmm_ext);

            hmm_tied_read_big_bin (hmm_dir_list, hmm_file_name, smds,
                                   transSmooth, NUMOFCODEENTRIES, TRUE,
                                   transWeight);
        }
    } else {
        for (i = 0; i < num_phones; i++) {
            if ((!useCiTrans) || (phone_id_to_base_id(i) == i)) {
                sprintf (hmm_file_name, "%s.%s", phone_from_id (i), hmm_ext);
                hmm_tied_read_bin (hmm_dir_list, hmm_file_name,
                                   &smds[hmm_pid2sid(i)], transSmooth,
                                   NUMOFCODEENTRIES, TRUE, transWeight);
            }
        }
    }

    /*
     *  Use Ci transitions ?
     */
    if (useCiTrans) {
        for (i = 0; i < num_phones; i++) {
            if (hmm_pid2sid(phone_id_to_base_id(i)) != hmm_pid2sid(i)) {
                /*
                 * Just make a copy of the CI phone transitions
                 */
                memcpy (&smds[hmm_pid2sid(i)], &smds[hmm_pid2sid(phone_id_to_base_id(i))],
                        sizeof (SMD));
            }
        }
    }
    /*
     * Read the distributions
     */
    read_dists (hmm_dir, code1_ext, code2_ext, code3_ext, code4_ext,
                NUMOFCODEENTRIES, hmm_smooth_min, useCiPhonesOnly);
    if (Use8BitSenProb)
        SCVQSetSenoneCompression (8);

    /*
     * Map the distributions to the correct locations
     */
    remap (smds);
}