void phone_add_diphones (void)
/*------------------------*
* DESCRIPTION - figure out what the word begin/end diphones of intrest are
* and add them to the phone set.
*/
{
int32 phone_cnt = phone_count();
int32 pid;
int32 ret;
int32 new_phone_id = phone_cnt;
char tp[64];
char ci[32], lc[64], rc[64], pc[64];
for (pid = 0; pid < phone_cnt; pid++) {
strcpy (tp, phone_from_id (pid));
ret = parse_triphone (tp, ci, lc, rc, pc);
if (ret == 4) {
switch (pc[0]) {
case 'b':
sprintf (tp, "%s(%%s,%s)b", ci, rc);
if (phone_to_id (tp, FALSE) == NO_PHONE) {
add_phone (tp, new_phone_id, phone_to_id(ci, TRUE),
PT_DIPHONE, 1);
new_phone_id++;
}
break;
case 'e':
sprintf (tp, "%s(%s,%%s)e", ci, lc);
if (phone_to_id (tp, FALSE) == NO_PHONE) {
add_phone (tp, new_phone_id, phone_to_id(ci, TRUE),
PT_DIPHONE, 1);
new_phone_id++;
}
break;
case 's':
sprintf (tp, "%s(%%s,%%s)s", ci);
if (phone_to_id (tp, FALSE) == NO_PHONE) {
add_phone (tp, new_phone_id, phone_to_id(ci, TRUE),
PT_DIPHONE_S, 1);
new_phone_id++;
}
break;
case '\0':
break;
default:
E_FATAL("Unknown position context in %s == '%c'\n",
tp, pc[0]);
}
}
}
/*
* Remake the phone map to account for the diphones
*/
mk_phone_map ();
E_INFO("Added %d new begin/end word diphones\n",
new_phone_id - phone_cnt);
}
int
phone_read (char *filename)
/*------------------------------------------------------------*
* read in the phone file filename
*------------------------------------------------------------*/
{
int32 retval = 0;
FILE *fs = NULL;
char phone_str[1024];
int32 phone_id = 0;
int32 base_id = 0;
int32 field1;
int32 field2;
int32 model_len, j;
int32 tmp_num_phones;
/*
* Find #phones and set hash and list tables size hints.
* (Otherwise, the simple-minded PC malloc library goes berserk.)
*/
fs = CM_fopen (filename, "r");
for (j = 1;; j++)
if (fgets (phone_str, sizeof(phone_str), fs) == NULL)
break;
phones.size_hint = j;
phones_list.size_hint = j;
phone_base_map.size_hint = j;
phone_model_len.size_hint = j;
phone_type_map.size_hint = j;
rewind(fs);
while (EOF != fscanf (fs, "%s%d%d%d%d\n",
phone_str, &field1, &field2,
&base_id, &phone_id))
{
/*
* field1 is 0 for ciPhones
*/
if (field1 == 0)
numCiPhones++;
/*
* Phones marked with a '-2' account for more than one model
*/
if (field1 == -2) {
numWdPhones++;
model_len = field2;
}
else
model_len = 1;
if (model_len <= 0) {
E_WARN("%s has length %d\n", phone_str, model_len);
}
add_phone (phone_str, phone_id, base_id, field1, model_len);
}
tmp_num_phones = phone_id;
/*
* Next availiable phone id
*/
phone_id++;
for (j = 0; j < tmp_num_phones; j++) {
/*
* If the phone is one of these within word phones then
* create model_len-1 additional phones name ...(1) ...(2) and
* and so on up to ...(model_len-1). It's these phones the
* word refers to for it pronunciation
*/
if (phone_type(j) == -2) {
int32 i;
int32 model_len = phone_len (j);
for (i = 1; i < model_len; i++) {
char tmpstr[256];
sprintf (tmpstr, "%s(%d)", phone_from_id(j), i);
add_phone (tmpstr, phone_id, j, 1000+i, 1);
phone_id++;
}
}
}
mk_phone_map();
if (fs)
fclose (fs);
return (retval);
}
static void mk_phone_map (void)
{
int32 numPhones = phone_count();
int32 numPhoneMappings = 0; /* will be bogus without VERBOSE_PHONE_MAP! */
int32 pid;
#ifdef VERBOSE_PHONE_MAP
int32 pid1, pid2;
int32 numCiPhones = phoneCiCount();
int32 numVocPhones = sizeof(voc)/sizeof(char *);
int32 numUstPhones = sizeof(ust)/sizeof(char *);
int32 op_id, res_id;
char op[32], res[32];
#endif
/*
* In case we get called twice.
*/
if (PhoneMap)
free (PhoneMap);
PhoneMap = (int32 *) CM_calloc (numPhones, sizeof(int32));
for (pid = 0; pid < numPhones; pid++)
PhoneMap[pid] = pid;
#ifdef VERBOSE_PHONE_MAP
for (pid = 0; pid < numCiPhones; pid++) {
sprintf (op, "TD(%s,Y)e", phone_from_id(pid));
sprintf (res, "JH(%s,Y)e", phone_from_id(pid));
op_id = phone_to_id (op, FALSE);
res_id = phone_to_id (res, FALSE);
if ((op_id != NO_PHONE) && (res_id != NO_PHONE)) {
PhoneMap[op_id] = res_id;
numPhoneMappings++;
}
sprintf (op, "DD(%s,Y)e", phone_from_id(pid));
sprintf (res, "JH(%s,Y)e", phone_from_id(pid));
op_id = phone_to_id (op, FALSE);
res_id = phone_to_id (res, FALSE);
if ((op_id != NO_PHONE) && (res_id != NO_PHONE)) {
PhoneMap[op_id] = res_id;
numPhoneMappings++;
}
}
for (pid1 = 0; pid1 < numVocPhones; pid1++) {
for (pid2 = 0; pid2 < numUstPhones; pid2++) {
sprintf (op, "TD(%s,%s)e", voc[pid1], ust[pid2]);
sprintf (res, "DX(%s,%s)", voc[pid1], ust[pid2]);
op_id = phone_to_id (op, FALSE);
res_id = phone_to_id (res, FALSE);
if ((op_id != NO_PHONE) && (res_id != NO_PHONE)) {
PhoneMap[op_id] = res_id;
numPhoneMappings++;
}
sprintf (op, "DD(%s,%s)e", voc[pid1], ust[pid2]);
sprintf (res, "D(%s,%s)", voc[pid1], ust[pid2]);
op_id = phone_to_id (op, FALSE);
res_id = phone_to_id (res, FALSE);
if ((op_id != NO_PHONE) && (res_id != NO_PHONE)) {
PhoneMap[op_id] = res_id;
numPhoneMappings++;
}
}
for (pid2 = 0; pid2 < numCiPhones; pid2++) {
sprintf (op, "TD(%s,%s)e", phone_from_id(pid2), voc[pid1]);
sprintf (res, "T(%s,%s)", phone_from_id(pid2), voc[pid1]);
op_id = phone_to_id (op, FALSE);
res_id = phone_to_id (res, FALSE);
if ((op_id != NO_PHONE) && (res_id != NO_PHONE)) {
PhoneMap[op_id] = res_id;
numPhoneMappings++;
}
sprintf (op, "DD(%s,%s)e", phone_from_id(pid2), voc[pid1]);
sprintf (res, "D(%s,%s)", phone_from_id(pid2), voc[pid1]);
op_id = phone_to_id (op, FALSE);
res_id = phone_to_id (res, FALSE);
if ((op_id != NO_PHONE) && (res_id != NO_PHONE)) {
PhoneMap[op_id] = res_id;
numPhoneMappings++;
}
sprintf (op, "BD(%s,%s)e", phone_from_id(pid2), voc[pid1]);
sprintf (res, "B(%s,%s)", phone_from_id(pid2), voc[pid1]);
op_id = phone_to_id (op, FALSE);
res_id = phone_to_id (res, FALSE);
if ((op_id != NO_PHONE) && (res_id != NO_PHONE)) {
PhoneMap[op_id] = res_id;
numPhoneMappings++;
}
sprintf (op, "GD(%s,%s)e", phone_from_id(pid2), voc[pid1]);
sprintf (res, "G(%s,%s)", phone_from_id(pid2), voc[pid1]);
op_id = phone_to_id (op, FALSE);
res_id = phone_to_id (res, FALSE);
if ((op_id != NO_PHONE) && (res_id != NO_PHONE)) {
PhoneMap[op_id] = res_id;
numPhoneMappings++;
}
sprintf (op, "KD(%s,%s)e", phone_from_id(pid2), voc[pid1]);
sprintf (res, "K(%s,%s)", phone_from_id(pid2), voc[pid1]);
op_id = phone_to_id (op, FALSE);
res_id = phone_to_id (res, FALSE);
if ((op_id != NO_PHONE) && (res_id != NO_PHONE)) {
PhoneMap[op_id] = res_id;
numPhoneMappings++;
}
sprintf (op, "PD(%s,%s)e", phone_from_id(pid2), voc[pid1]);
sprintf (res, "P(%s,%s)", phone_from_id(pid2), voc[pid1]);
op_id = phone_to_id (op, FALSE);
res_id = phone_to_id (res, FALSE);
if ((op_id != NO_PHONE) && (res_id != NO_PHONE)) {
PhoneMap[op_id] = res_id;
numPhoneMappings++;
}
}
}
#endif /* VERBOSE_PHONE_MAP */
E_INFO ("Using %d phonological mappings\n", numPhoneMappings);
}
void kb (int argc, char *argv[],
float ip, /* word insertion penalty */
float lw, /* langauge weight */
float pip) /* phone insertion penalty */
{
char *pname = argv[0];
char hmm_file_name[256];
int32 num_phones, num_ci_phones;
int32 i, use_darpa_lm;
/* FIXME: This is evil. But if we do it, let's prototype it
somewhere, OK? */
unlimit (); /* Remove memory size limits */
language_weight = lw;
insertion_penalty = ip;
phone_insertion_penalty = pip;
pconf (argc, argv, kb_param, 0, 0, 0);
if ((phone_file_name == 0) ||
(dict_file_name == 0))
pusage (pname, (Config_t *)kb_param);
log_info("%s(%d): Reading phone file [%s]\n",
__FILE__, __LINE__, phone_file_name);
if (phone_read (phone_file_name))
exit (-1);
if (useWDPhonesOnly)
phone_add_diphones();
num_ci_phones = phoneCiCount();
/* Read the distribution map file */
log_info("%s(%d): Reading map file [%s]\n", __FILE__, __LINE__, mapFileName);
read_map (mapFileName, TRUE /* useCiTrans compress */);
log_info("%s(%d): Reading dict file [%s]\n",
__FILE__, __LINE__, dict_file_name);
word_dict = dict_new ();
if (dict_read (word_dict, dict_file_name, phrase_dict_file_name,
noise_dict_file_name, !useWDPhonesOnly))
exit (-1);
use_darpa_lm = TRUE;
if (use_darpa_lm) {
lmSetStartSym (lm_start_sym);
lmSetEndSym (lm_end_sym);
/*
* Read control file describing multiple LMs, if specified.
* File format (optional stuff is indicated by enclosing in []):
*
* [{ LMClassFileName LMClassFilename ... }]
* TrigramLMFileName LMName [{ LMClassName LMClassName ... }]
* TrigramLMFileName LMName [{ LMClassName LMClassName ... }]
* ...
* (There should be whitespace around the { and } delimiters.)
*
* This is an extension of the older format that had only TrigramLMFilenName
* and LMName pairs. The new format allows a set of LMClass files to be read
* in and referred to by the trigram LMs. (Incidentally, if one wants to use
* LM classes in a trigram LM, one MUST use the -lmctlfn flag. It is not
* possible to read in a class-based trigram LM using the -lmfn flag.)
*
* No "comments" allowed in this file.
*/
if (lm_ctl_filename) {
FILE *ctlfp;
char lmfile[4096], lmname[4096], str[4096];
lmclass_set_t lmclass_set;
lmclass_t *lmclass, cl;
int32 n_lmclass, n_lmclass_used;
lmclass_set = lmclass_newset();
E_INFO("Reading LM control file '%s'\n", lm_ctl_filename);
ctlfp = CM_fopen (lm_ctl_filename, "r");
if (fscanf (ctlfp, "%s", str) == 1) {
if (strcmp (str, "{") == 0) {
/* Load LMclass files */
while ((fscanf (ctlfp, "%s", str) == 1) && (strcmp (str, "}") != 0))
lmclass_set = lmclass_loadfile (lmclass_set, str);
if (strcmp (str, "}") != 0)
E_FATAL("Unexpected EOF(%s)\n", lm_ctl_filename);
if (fscanf (ctlfp, "%s", str) != 1)
str[0] = '\0';
}
} else
str[0] = '\0';
/* Fill in dictionary word id information for each LMclass word */
for (cl = lmclass_firstclass(lmclass_set);
lmclass_isclass(cl);
cl = lmclass_nextclass(lmclass_set, cl)) {
kb_init_lmclass_dictwid (cl);
}
/* At this point if str[0] != '\0', we have an LM filename */
n_lmclass = lmclass_get_nclass(lmclass_set);
lmclass = (lmclass_t *) CM_calloc (n_lmclass, sizeof(lmclass_t));
/* Read in one LM at a time */
while (str[0] != '\0') {
strcpy (lmfile, str);
if (fscanf (ctlfp, "%s", lmname) != 1)
E_FATAL("LMname missing after LMFileName '%s'\n", lmfile);
n_lmclass_used = 0;
if (fscanf (ctlfp, "%s", str) == 1) {
if (strcmp (str, "{") == 0) {
/* LM uses classes; read their names */
while ((fscanf (ctlfp, "%s", str) == 1) &&
(strcmp (str, "}") != 0)) {
if (n_lmclass_used >= n_lmclass)
E_FATAL("Too many LM classes specified for '%s'\n",
lmfile);
lmclass[n_lmclass_used] = lmclass_get_lmclass (lmclass_set,
str);
if (! (lmclass_isclass(lmclass[n_lmclass_used])))
E_FATAL("LM class '%s' not found\n", str);
n_lmclass_used++;
}
if (strcmp (str, "}") != 0)
E_FATAL("Unexpected EOF(%s)\n", lm_ctl_filename);
if (fscanf (ctlfp, "%s", str) != 1)
str[0] = '\0';
}
} else
str[0] = '\0';
if (n_lmclass_used > 0)
lm_read_clm (lmfile, lmname,
language_weight, unigramWeight, insertion_penalty,
lmclass, n_lmclass_used);
else
lm_read (lmfile, lmname,
language_weight, unigramWeight, insertion_penalty);
}
fclose (ctlfp);
NoLangModel = FALSE;
}
/* Read "base" LM file, if specified */
if (lm_file_name) {
lmSetStartSym (lm_start_sym);
lmSetEndSym (lm_end_sym);
lm_read (lm_file_name, "", language_weight, unigramWeight, insertion_penalty);
/* Make initial OOV list known to this base LM */
lm_init_oov ();
NoLangModel = FALSE;
}
#ifdef USE_ILM
/* Init ILM module (non-std-Darpa LM, eg ug/bg cache LM) */
ilm_init ();
#endif
}
#if 0
/* Compute the phrase lm probabilities */
computePhraseLMProbs ();
#endif
num_phones = phone_count ();
numSmds = hmm_num_sseq();
smds = (SMD *) CM_calloc (numSmds, sizeof (SMD));
/*
* Read the hmm's into the SMD structures
*/
if (useBigHmmFiles) {
for (i = 0; i < num_ci_phones; i++) {
sprintf (hmm_file_name, "%s.%s", phone_from_id (i),
hmm_ext);
hmm_tied_read_big_bin (hmm_dir_list, hmm_file_name, smds,
transSmooth, NUMOFCODEENTRIES, TRUE,
transWeight);
}
} else {
for (i = 0; i < num_phones; i++) {
if ((!useCiTrans) || (phone_id_to_base_id(i) == i)) {
sprintf (hmm_file_name, "%s.%s", phone_from_id (i), hmm_ext);
hmm_tied_read_bin (hmm_dir_list, hmm_file_name,
&smds[hmm_pid2sid(i)], transSmooth,
NUMOFCODEENTRIES, TRUE, transWeight);
}
}
}
/*
* Use Ci transitions ?
*/
if (useCiTrans) {
for (i = 0; i < num_phones; i++) {
if (hmm_pid2sid(phone_id_to_base_id(i)) != hmm_pid2sid(i)) {
/*
* Just make a copy of the CI phone transitions
*/
memcpy (&smds[hmm_pid2sid(i)], &smds[hmm_pid2sid(phone_id_to_base_id(i))],
sizeof (SMD));
}
}
}
/*
* Read the distributions
*/
read_dists (hmm_dir, code1_ext, code2_ext, code3_ext, code4_ext,
NUMOFCODEENTRIES, hmm_smooth_min, useCiPhonesOnly);
if (Use8BitSenProb)
SCVQSetSenoneCompression (8);
/*
* Map the distributions to the correct locations
*/
remap (smds);
}
