int phone_read (char *filename) /*------------------------------------------------------------* * read in the phone file filename *------------------------------------------------------------*/ { int32 retval = 0; FILE *fs = NULL; char phone_str[1024]; int32 phone_id = 0; int32 base_id = 0; int32 field1; int32 field2; int32 model_len, j; int32 tmp_num_phones; /* * Find #phones and set hash and list tables size hints. * (Otherwise, the simple-minded PC malloc library goes berserk.) */ fs = CM_fopen (filename, "r"); for (j = 1;; j++) if (fgets (phone_str, sizeof(phone_str), fs) == NULL) break; phones.size_hint = j; phones_list.size_hint = j; phone_base_map.size_hint = j; phone_model_len.size_hint = j; phone_type_map.size_hint = j; rewind(fs); while (EOF != fscanf (fs, "%s%d%d%d%d\n", phone_str, &field1, &field2, &base_id, &phone_id)) { /* * field1 is 0 for ciPhones */ if (field1 == 0) numCiPhones++; /* * Phones marked with a '-2' account for more than one model */ if (field1 == -2) { numWdPhones++; model_len = field2; } else model_len = 1; if (model_len <= 0) { E_WARN("%s has length %d\n", phone_str, model_len); } add_phone (phone_str, phone_id, base_id, field1, model_len); } tmp_num_phones = phone_id; /* * Next availiable phone id */ phone_id++; for (j = 0; j < tmp_num_phones; j++) { /* * If the phone is one of these within word phones then * create model_len-1 additional phones name ...(1) ...(2) and * and so on up to ...(model_len-1). It's these phones the * word refers to for it pronunciation */ if (phone_type(j) == -2) { int32 i; int32 model_len = phone_len (j); for (i = 1; i < model_len; i++) { char tmpstr[256]; sprintf (tmpstr, "%s(%d)", phone_from_id(j), i); add_phone (tmpstr, phone_id, j, 1000+i, 1); phone_id++; } } } mk_phone_map(); if (fs) fclose (fs); return (retval); }
void kb (int argc, char *argv[], float ip, /* word insertion penalty */ float lw, /* langauge weight */ float pip) /* phone insertion penalty */ { char *pname = argv[0]; char hmm_file_name[256]; int32 num_phones, num_ci_phones; int32 i, use_darpa_lm; /* FIXME: This is evil. But if we do it, let's prototype it somewhere, OK? */ unlimit (); /* Remove memory size limits */ language_weight = lw; insertion_penalty = ip; phone_insertion_penalty = pip; pconf (argc, argv, kb_param, 0, 0, 0); if ((phone_file_name == 0) || (dict_file_name == 0)) pusage (pname, (Config_t *)kb_param); log_info("%s(%d): Reading phone file [%s]\n", __FILE__, __LINE__, phone_file_name); if (phone_read (phone_file_name)) exit (-1); if (useWDPhonesOnly) phone_add_diphones(); num_ci_phones = phoneCiCount(); /* Read the distribution map file */ log_info("%s(%d): Reading map file [%s]\n", __FILE__, __LINE__, mapFileName); read_map (mapFileName, TRUE /* useCiTrans compress */); log_info("%s(%d): Reading dict file [%s]\n", __FILE__, __LINE__, dict_file_name); word_dict = dict_new (); if (dict_read (word_dict, dict_file_name, phrase_dict_file_name, noise_dict_file_name, !useWDPhonesOnly)) exit (-1); use_darpa_lm = TRUE; if (use_darpa_lm) { lmSetStartSym (lm_start_sym); lmSetEndSym (lm_end_sym); /* * Read control file describing multiple LMs, if specified. * File format (optional stuff is indicated by enclosing in []): * * [{ LMClassFileName LMClassFilename ... }] * TrigramLMFileName LMName [{ LMClassName LMClassName ... }] * TrigramLMFileName LMName [{ LMClassName LMClassName ... }] * ... * (There should be whitespace around the { and } delimiters.) * * This is an extension of the older format that had only TrigramLMFilenName * and LMName pairs. The new format allows a set of LMClass files to be read * in and referred to by the trigram LMs. (Incidentally, if one wants to use * LM classes in a trigram LM, one MUST use the -lmctlfn flag. It is not * possible to read in a class-based trigram LM using the -lmfn flag.) * * No "comments" allowed in this file. */ if (lm_ctl_filename) { FILE *ctlfp; char lmfile[4096], lmname[4096], str[4096]; lmclass_set_t lmclass_set; lmclass_t *lmclass, cl; int32 n_lmclass, n_lmclass_used; lmclass_set = lmclass_newset(); E_INFO("Reading LM control file '%s'\n", lm_ctl_filename); ctlfp = CM_fopen (lm_ctl_filename, "r"); if (fscanf (ctlfp, "%s", str) == 1) { if (strcmp (str, "{") == 0) { /* Load LMclass files */ while ((fscanf (ctlfp, "%s", str) == 1) && (strcmp (str, "}") != 0)) lmclass_set = lmclass_loadfile (lmclass_set, str); if (strcmp (str, "}") != 0) E_FATAL("Unexpected EOF(%s)\n", lm_ctl_filename); if (fscanf (ctlfp, "%s", str) != 1) str[0] = '\0'; } } else str[0] = '\0'; /* Fill in dictionary word id information for each LMclass word */ for (cl = lmclass_firstclass(lmclass_set); lmclass_isclass(cl); cl = lmclass_nextclass(lmclass_set, cl)) { kb_init_lmclass_dictwid (cl); } /* At this point if str[0] != '\0', we have an LM filename */ n_lmclass = lmclass_get_nclass(lmclass_set); lmclass = (lmclass_t *) CM_calloc (n_lmclass, sizeof(lmclass_t)); /* Read in one LM at a time */ while (str[0] != '\0') { strcpy (lmfile, str); if (fscanf (ctlfp, "%s", lmname) != 1) E_FATAL("LMname missing after LMFileName '%s'\n", lmfile); n_lmclass_used = 0; if (fscanf (ctlfp, "%s", str) == 1) { if (strcmp (str, "{") == 0) { /* LM uses classes; read their names */ while ((fscanf (ctlfp, "%s", str) == 1) && (strcmp (str, "}") != 0)) { if (n_lmclass_used >= n_lmclass) E_FATAL("Too many LM classes specified for '%s'\n", lmfile); lmclass[n_lmclass_used] = lmclass_get_lmclass (lmclass_set, str); if (! (lmclass_isclass(lmclass[n_lmclass_used]))) E_FATAL("LM class '%s' not found\n", str); n_lmclass_used++; } if (strcmp (str, "}") != 0) E_FATAL("Unexpected EOF(%s)\n", lm_ctl_filename); if (fscanf (ctlfp, "%s", str) != 1) str[0] = '\0'; } } else str[0] = '\0'; if (n_lmclass_used > 0) lm_read_clm (lmfile, lmname, language_weight, unigramWeight, insertion_penalty, lmclass, n_lmclass_used); else lm_read (lmfile, lmname, language_weight, unigramWeight, insertion_penalty); } fclose (ctlfp); NoLangModel = FALSE; } /* Read "base" LM file, if specified */ if (lm_file_name) { lmSetStartSym (lm_start_sym); lmSetEndSym (lm_end_sym); lm_read (lm_file_name, "", language_weight, unigramWeight, insertion_penalty); /* Make initial OOV list known to this base LM */ lm_init_oov (); NoLangModel = FALSE; } #ifdef USE_ILM /* Init ILM module (non-std-Darpa LM, eg ug/bg cache LM) */ ilm_init (); #endif } #if 0 /* Compute the phrase lm probabilities */ computePhraseLMProbs (); #endif num_phones = phone_count (); numSmds = hmm_num_sseq(); smds = (SMD *) CM_calloc (numSmds, sizeof (SMD)); /* * Read the hmm's into the SMD structures */ if (useBigHmmFiles) { for (i = 0; i < num_ci_phones; i++) { sprintf (hmm_file_name, "%s.%s", phone_from_id (i), hmm_ext); hmm_tied_read_big_bin (hmm_dir_list, hmm_file_name, smds, transSmooth, NUMOFCODEENTRIES, TRUE, transWeight); } } else { for (i = 0; i < num_phones; i++) { if ((!useCiTrans) || (phone_id_to_base_id(i) == i)) { sprintf (hmm_file_name, "%s.%s", phone_from_id (i), hmm_ext); hmm_tied_read_bin (hmm_dir_list, hmm_file_name, &smds[hmm_pid2sid(i)], transSmooth, NUMOFCODEENTRIES, TRUE, transWeight); } } } /* * Use Ci transitions ? */ if (useCiTrans) { for (i = 0; i < num_phones; i++) { if (hmm_pid2sid(phone_id_to_base_id(i)) != hmm_pid2sid(i)) { /* * Just make a copy of the CI phone transitions */ memcpy (&smds[hmm_pid2sid(i)], &smds[hmm_pid2sid(phone_id_to_base_id(i))], sizeof (SMD)); } } } /* * Read the distributions */ read_dists (hmm_dir, code1_ext, code2_ext, code3_ext, code4_ext, NUMOFCODEENTRIES, hmm_smooth_min, useCiPhonesOnly); if (Use8BitSenProb) SCVQSetSenoneCompression (8); /* * Map the distributions to the correct locations */ remap (smds); }