/**
* Look up a word on dictionary by string.
*
* @param keyword [in] keyword to search
* @param winfo [in] word dictionary
*
* @return the word id if found, or WORD_INVALID if not found.
*/
WORD_ID
voca_lookup_wid(char *keyword, WORD_INFO *winfo)
{
WORD_ID i, found;
int plen,totallen;
boolean numflag = TRUE;
int wid;
char *c;
if (keyword == NULL) return WORD_INVALID;
if (keyword[0] == '#') {
for(i=1;i<strlen(keyword);i++) {
if (keyword[i] < '0' || keyword[i] > '9') {
numflag = FALSE;
break;
}
}
if (numflag) {
wid = atoi(&(keyword[1]));
if (wid < 0 || wid >= winfo->num) {
return(WORD_INVALID);
} else {
return(wid);
}
} else {
return(WORD_INVALID);
}
}
found = WORD_INVALID;
totallen = strlen(keyword);
if ((c = strchr(keyword, '[')) != NULL) {
plen = c - keyword;
for (i=0;i<winfo->num;i++) {
if (strnmatch(keyword,winfo->wname[i], plen)
&& strnmatch(c+1, winfo->woutput[i], totallen-plen-2)) {
if (found == WORD_INVALID) {
found = i;
} else {
jlog("Warning: voca_lookup: several \"%s\" found in dictionary, use the first one..\n");
break;
}
}
}
} else {
for (i=0;i<winfo->num;i++) {
if (strmatch(keyword,winfo->wname[i])) {
if (found == WORD_INVALID) {
found = i;
} else {
jlog("Warning: voca_lookup: several \"%s\" found in dictionary, use the first one..\n");
break;
}
}
}
}
return found;
}
mBOOL DLLINTERNAL MConfig::load(const char *fn) {
FILE *fp;
char loadfile[PATH_MAX];
char line[MAX_CONF_LEN];
char *optname, *optval;
option_t *optp;
int ln;
// Make full pathname (from gamedir if relative, collapse "..",
// backslashes, etc).
full_gamedir_path(fn, loadfile);
fp=fopen(loadfile, "r");
if(!fp) {
META_WARNING("unable to open config file '%s': %s", loadfile,
strerror(errno));
RETURN_ERRNO(mFALSE, ME_NOFILE);
}
META_DEBUG(2, ("Loading from config file: %s", loadfile));
for(ln=1; !feof(fp) && fgets(line, sizeof(line), fp); ln++) {
if(line[0]=='#')
continue;
if(line[0]==';')
continue;
if(strnmatch(line, "//", 2))
continue;
if(!(optname=strtok(line, " \t\r\n"))) {
META_WARNING("'%s' line %d: bad config format: missing option",
loadfile, ln);
continue;
}
if(!(optval=strtok(NULL, "\r\n"))) {
META_WARNING("'%s' line %d: bad config format: missing value",
loadfile, ln);
continue;
}
if(!(optp=find(optname))) {
META_WARNING("'%s' line %d: unknown option name '%s'",
loadfile, ln, optname);
continue;
}
if(!set(optp, optval)) {
META_WARNING("'%s' line %d: unable to set option '%s' value '%s'",
loadfile, ln, optname, optval);
continue;
}
}
filename=strdup(loadfile);
fclose(fp);
return(mTRUE);
}
void FuncShowMaps(edict_t *pEntity)
{
FILE *file = NULL;
const char *path = "AdminMod/maps.txt";
char fullpath[PATH_MAX];
char line[MAX_CONF_LEN];
char buf[MAX_CONF_LEN];
sup_gamedir_path(path, fullpath);
file = fopen(fullpath, "r");
if(!file)
{
UTIL_LogPrintf("Unable to open map list file '%s': %s", fullpath, strerror(errno));
if(pEntity)
CLIENT_PRINTF(pEntity, print_console, "\nUnable to load map list\n");
return;
}
for(int i = 1, j = 0; !feof(file); i++)
{
if(!fgets(line, sizeof(line), file))
{
UTIL_LogPrintf("Unable to read %d line", i);
if(pEntity)
CLIENT_PRINTF(pEntity, print_console, "\nUnable to load map list\n");
fclose(file);
break;
}
if(line[0] == '#' || line[0] == ';')
continue;
if(strnmatch(line, "//", 2))
continue;
j = strlen(line) - 1;
if(line[j] == '\n')
line[j] = '\0';
if(!pEntity)
LOG_CONSOLE(NULL, "%d) %s", i, line);
else
{
sup_make_str(buf, sizeof(buf), "%d) %s\n", i, line);
CLIENT_PRINTF(pEntity, print_console, buf);
}
}
fclose(file);
}
/**
* Set number of N-gram entries, for reading the first LR 2-gram.
*
* @param fp [in] file pointer
* @param numlist [out] set the values to this buffer (malloc)
*
* @return the value of N, or -1 on error.
*/
static int
get_total_info(FILE *fp, NNID **numlist)
{
char *p;
int n;
int maxn;
unsigned long entry_num;
int numnum;
maxn = 0;
numnum = 10;
*numlist = (NNID *)mymalloc(sizeof(NNID) * numnum);
while (getl(buf, sizeof(buf), fp) != NULL && buf[0] != '\\') {
if (strnmatch(buf, "ngram", 5)) { /* n-gram num */
//p = strtok(buf, " =");
//n = atoi(p);
//p = strtok(NULL, " =");
//entry_num = atol(p);
//sscanf(p, "%lu", &entry_num);
sscanf(buf, "ngram %d = %lu", &n, &entry_num);
/* check maximum number */
if (entry_num > NNID_MAX) {
jlog("Error: too big %d-gram (exceeds %d bit)\n", n, sizeof(NNID) * 8);
return -1;
}
/* ignore empty entry */
if (entry_num == 0) {
jlog("Warning: empty %d-gram, skipped\n", n);
} else {
if (maxn < n) maxn = n;
if (n >= numnum) {
numnum *= 2;
*numlist = (NNID *)myrealloc(*numlist, sizeof(NNID) * numnum);
}
(*numlist)[n-1] = entry_num;
}
}
}
return(maxn);
}
bool FuncValidMap(const char *map)
{
FILE *file = NULL;
const char *path = "AdminMod/maps.txt";
char fullpath[PATH_MAX];
char line[MAX_CONF_LEN];
sup_gamedir_path(path, fullpath);
file = fopen(fullpath, "r");
if(!file)
{
UTIL_LogPrintf("Unable to open map list file '%s': %s", fullpath, strerror(errno));
return false;
}
for(int i = 1, j = 0; !feof(file); i++)
{
if(!fgets(line, sizeof(line), file))
{
UTIL_LogPrintf("Unable to read %d line", i);
break;
}
if(line[0] == '#' || line[0] == ';')
continue;
if(strnmatch(line, "//", 2))
continue;
j = strlen(line) - 1;
if(line[j] == '\n')
line[j] = '\0';
if(!strcmp(map, line))
{
fclose(file);
return true;
}
}
fclose(file);
return false;
}
char* FuncGetMap(int id)
{
FILE *file = NULL;
const char *path = "AdminMod/maps.txt";
char fullpath[PATH_MAX];
static char line[36];
sup_gamedir_path(path, fullpath);
file = fopen(fullpath, "r");
if(!file)
{
UTIL_LogPrintf("Unable to open map list file '%s': %s", fullpath, strerror(errno));
return NULL;
}
for(int i = 1, j = 0; !feof(file); i++)
{
if(!fgets(line, sizeof(line), file))
{
UTIL_LogPrintf("Unable to read %d line", i);
fclose(file);
break;
}
if(line[0] == '#' || line[0] == ';')
continue;
if(strnmatch(line, "//", 2))
continue;
j++;
if(j != id)
continue;
break;
}
if(line[strlen(line) - 1] == '\n')
line[strlen(line) - 1] = '\0';
fclose(file);
return line;
}
bool FuncIsBanned(const char *name, int &minutes)
{
FILE *file = NULL;
const char *path = "AdminMod/banned.txt";
char fullpath[PATH_MAX];
char line[MAX_CONF_LEN];
char *banned = NULL, *date = NULL, *cur = NULL;
time_t clock = time(NULL);
tm *ltm = localtime(&clock);
sup_gamedir_path(path, fullpath);
LOG_CONSOLE(NULL, "path - %s", fullpath);
file = fopen(fullpath, "r");
if(!file)
{
UTIL_LogPrintf("Unable to open banned list file '%s': %s", fullpath, strerror(errno));
return false;
}
for(int i = 1, j = 0; !feof(file); i++)
{
if(!fgets(line, sizeof(line), file))
{
UTIL_LogPrintf("Unable to read %d line in file %s", i, fullpath);
fclose(file);
return false;
}
if(line[0] == '#' || line[0] == ';')
continue;
if(strnmatch(line, "//", 2))
continue;
j = strlen(line) - 1;
if(line[j] == '\n')
line[j] = '\0';
banned = strtok(line, "\"");
if(!banned)
{
UTIL_LogPrintf("Invalid parameters at line %d in file %s", i, fullpath);
fclose(file);
return false;
}
sup_del_quotes(banned);
if(strcmp(banned, name) == 0)
{
minutes = 0;
date = strtok(NULL, "\n");
for(USHORT k = 0; k < strlen(date); k++)
if(date[k] != ' ' && date[k] != '\t')
{
STRNCPY(date, date + k, strlen(date));
break;
}
if(date[0] == '-')
{
fclose(file);
return true;
}
cur = strtok(date, ":");
minutes += sup_str_to_int(cur) - ltm->tm_min;
cur = strtok(NULL, " ");
minutes += 60 * (sup_str_to_int(cur) - ltm->tm_hour);
cur = strtok(NULL, ":");
minutes += 60 * 24 * (sup_str_to_int(cur) - ltm->tm_yday);
cur = strtok(NULL, ":");
minutes += 60 * 24 * 30 * (sup_str_to_int(cur) - ltm->tm_year);
fclose(file);
if(minutes <= 0)
{
FuncRemoveFromBanned(banned);
return false;
}
else return true;
}
}
fclose(file);
return false;
}
bool FuncRemoveFromBanned(const char *name)
{
FILE *file = NULL;
const char *path = "AdminMod/banned.txt";
char fullpath[PATH_MAX];
char line[MAX_CONF_LEN];
char *banned = NULL;
int offset = 0, cur = 0;
sup_gamedir_path(path, fullpath);
file = fopen(fullpath, "r+");
if(!file)
{
UTIL_LogPrintf("Unable to open banned list file '%s': %s", fullpath, strerror(errno));
return false;
}
for(int i = 1, j = 0; !feof(file); i++)
{
if(!fgets(line, sizeof(line), file))
{
UTIL_LogPrintf("Unable to read %d line in file %s", i, fullpath);
fclose(file);
return false;
}
if(line[0] == '#' || line[0] == ';')
continue;
if(strnmatch(line, "//", 2))
continue;
j = strlen(line) - 1;
if(line[j] == '\n')
line[j] = '\0';
offset = strlen(line);
banned = strtok(line, "\"");
if(!banned)
{
UTIL_LogPrintf("Invalid parameters at line %d in file %s", i, fullpath);
fclose(file);
return false;
}
sup_del_quotes(banned);
LOG_CONSOLE(NULL, "banned - %s; list name - %s", banned, name);
if(strcmp(banned, name) == 0)
{
LOG_CONSOLE(NULL, "size of offset - %d", offset);
while(1)
{
if(fgets(line, sizeof(line), file) == NULL)
{
fseek(file, -(offset + 3), SEEK_CUR);
fputs("\n", file);
chsize(fileno(file), ftell(file));
fclose(file);
return true;
}
if(feof(file))
break;
LOG_CONSOLE(NULL, "read line - %s", line);
cur = strlen(line);
fseek(file, -(cur + offset + 3), SEEK_CUR);
fputs(line, file);
fseek(file, offset + 2, SEEK_CUR);
}
LOG_CONSOLE(NULL, "read line - %s", line);
cur = strlen(line);
fseek(file, -(cur + offset + 2), SEEK_CUR);
fputs(line, file);
chsize(fileno(file), ftell(file));
break;
}
}
fclose(file);
return true;
}
uint8_t *_IVParse(SplineFont *sf, char *text, int *len,
void (*IVError) (void *, char *, int), void *iv) {
short numberstack[256];
int npos=0, nread, i;
int push_left=0, push_size=0;
char *pt;
char *end, *bend, *brack;
int icnt=0, imax=strlen(text)/2, val, temp;
uint8_t *instrs=malloc(imax);
for (pt=text; *pt; ++pt) {
npos=0;
while (npos<256) {
while (*pt==' ' || *pt=='\t')
++pt;
if (isdigit(*pt) || *pt=='-') {
val=strtol(pt, &end, 0);
if (val>32767 || val<-32768) {
IVError(iv,"A value must be between [-32768,32767]",
pt-text);
return (NULL);
}
pt=end;
if (*pt=='@') { /* a delta control byte */
if (val>8 || val<-8 || val==0) {
IVError(iv,"A value must be between [-8,-1] or [1,8]",
pt-text);
return (NULL);
}
pt++;
if (!isdigit(*pt)) {
IVError(iv,"Number expected",pt-text);
return (NULL);
}
temp=val;
val=strtol(pt, &end, 0);
if (val>15 || val<0) {
IVError(iv,"A value must be between [0,15]",pt-text);
return (NULL);
}
val *= 16;
if (temp<0)
temp += 8;
else
temp += 7;
val += temp;
pt=end;
}
numberstack[npos++]=val;
} else if (strnmatch(pt, "cvt", 3)==0) {
pt += 3;
while (*pt==' ' || *pt=='\t')
++pt;
if (*pt != '(') {
IVError(iv,
"Missing left parenthesis in command to get a cvt index",
pt-text);
return (NULL);
}
temp=strtol(pt+1, &end, 0);
pt=end;
while (*pt==' ' || *pt=='\t')
++pt;
if (*pt != ')') {
IVError(iv,
"Missing right paren in command to get a cvt index",
pt-text);
return (NULL);
}
numberstack[npos++]=TTF__getcvtval(sf, temp);
++pt;
} else
break;
}
while (*pt==' ' || *pt=='\t')
++pt;
if (npos==0 && (*pt=='\n' || *pt=='\0'))
continue;
nread=0;
if (push_left==-1) {
/* we need a push count */
if (npos==0)
IVError(iv, "Expected a number for a push count", pt-text);
else if (numberstack[0]>255 || numberstack[0] <= 0) {
IVError(iv,
"The push count must be a number between 0 and 255",
pt-text);
return (NULL);
} else {
nread=1;
instrs[icnt++]=numberstack[0];
push_left=numberstack[0];
}
}
if (push_left != 0 && push_left<npos-nread
&& (*pt=='\n' || *pt=='\0')) {
IVError(iv, "More pushes specified than needed", pt-text);
return (NULL);
}
while (push_left>0 && nread<npos) {
if (push_size==2) {
instrs[icnt++]=numberstack[nread] >> 8;
instrs[icnt++]=numberstack[nread++]&0xff;
} else if (numberstack[0]>255 || numberstack[0]<0) {
IVError(iv,
"A value to be pushed by a byte push must be between 0 and 255",
pt-text);
return (NULL);
} else
instrs[icnt++]=numberstack[nread++];
--push_left;
}
// Parse a player name, userid, wonid, and team,
// or a Team name,
// or "World".
//
// Returns an allocated event_player_t struct, which should be freed by the
// main thread after it's no longer needed. But note that the strings
// contained in the struct refer to the "buf" in the parent event_args_t
// and should _not_ be freed directly; instead, the buf itself should be
// freed when the event_args_t struct is freed by the main thread.
event_player_t *parse_player(char *start, int *retlen) {
char *cp, *begin, *end;
int len;
event_player_t *pl;
// Create and init the event_player struct
pl=(event_player_t *) malloc(sizeof(event_player_t));
if(!pl)
return(NULL);
memset(pl, 0, sizeof(event_player_t));
cp=start;
*retlen=0;
cp+=strspn(cp, " ");
// ... World ...
if(strnmatch(cp, "World", 5)) {
// store team "World"
begin=cp;
end=begin+5;
len=end-begin;
*end='\0';
pl->team=begin;
*retlen=end-start+strspn(end, " ");
return(pl);
}
// ... Team "TERRORIST" ...
else if(strnmatch(cp, "Team ", 5)) {
// find team
begin=strchr(cp+5, '"')+1;
end=strchr(begin, '"');
len=end-begin;
if(len <= 0)
return(pl);
// store team
*end='\0';
pl->team=begin;
*retlen=end-start+strspn(end, " ");
return(pl);
}
// ... "Joe<15><785><TERRORIST>" ...
else if(cp[0] == '"') {
// find player name
begin=cp+1;
end=strchr(begin, '<');
len=end-begin;
if(len <= 0) {
*retlen=begin-1-start;
return(pl);
}
// store player name
*end='\0';
pl->name=begin;
// find userid
begin=end+1;
end=strchr(begin, '>');
len=end-begin;
if(len <= 0) {
*retlen=begin-1-start;
return(pl);
}
// store userid
*end='\0';
pl->userid=atoi(begin);
// find wonid
begin=strchr(end+1, '<')+1;
end=strchr(begin, '>');
len=end-begin;
if(len <= 0) {
*retlen=begin-1-start;
return(pl);
}
// store wonid
*end='\0';
pl->wonid=atoi(begin);
// find team
begin=strchr(end+1, '<')+1;
end=strchr(begin, '>');
len=end-begin;
if(len <= 0) {
*retlen=begin-1-start;
return(pl);
}
// store team
*end='\0';
pl->team=begin;
*retlen=end-start+strspn(end, " ");
return(pl);
}
else {
// No player strings found, so we go ahead and free the struct we
// malloc'd at the beginning.
free(pl);
return(NULL);
}
}
// Pre-parse logmsg line for args and to determine some possible type of
// event.
//
// Returns an allocated event_args_t struct, which should be freed by the
// main thread after it's no longer needed. Note that this struct gets
// passed to (possibly) multiple hooks.
event_args_t *parse_event_args(const char *logline) {
event_args_t *args;
char *cp, *begin, *end;
int len;
// Create and init the event_args struct.
args=(event_args_t *) malloc(sizeof(event_args_t));
if(!args)
return(NULL);
memset(args, 0, sizeof(*args));
// Make a copy of the logline, as we're going to be setting NULLs in
// the string to mark ends of various substrings.
args->buf=strdup(logline);
if(!args->buf)
return(NULL);
cp=args->buf;
// Grab the player name and attributes, or "world" or "team" name, ie:
// Joe<15><785><CT>
// World
// Team "CT"
args->player=parse_player(cp, &len);
cp+=len;
// Look for one of several pre-determined actions that we recognize.
// ... triggered "some action" ...
// ie: "Joe<15><785><CT>" triggered "Killed_A_Hostage"
if(strnmatch(cp, "triggered ", 10)) {
cp+=10;
args->action=parse_quoted(cp, &len);
cp+=len;
}
// ... killed Joe<15><785><CT> ...
// ie: "Joe<15><785><CT>" killed "Sam<17><197><TERRORIST>" with "sg552"
else if(strnmatch(cp, "killed ", 7)) {
cp+=7;
args->target=parse_player(cp, &len);
if(strmatch(args->player->team, args->target->team)) {
args->action="team_kill";
args->evtype=EV_TEAM_KILL;
}
else {
args->action="kill";
}
cp+=len;
}
// ... committed suicide ...
// ie: "Joe<15><785><CT>" committed suicide with "worldspawn"
else if(strnmatch(cp, "committed suicide", 17)) {
cp+=17;
args->action="suicide";
cp+=strspn(cp, " ");
args->evtype=EV_PLAYER_SUICIDE;
}
// ... joined team "someteam" ...
// ie: "Joe<15><785><>" joined team "CT"
else if(strnmatch(cp, "joined team ", 12)) {
cp+=12;
args->action="join_team";
args->target=(event_player_t *) malloc(sizeof(event_player_t));
// Note:
// old team is in: args->player->team
// new team is in: args->target->team
if(args->target) {
memset(args->target, 0, sizeof(event_player_t));
args->target->team=parse_quoted(cp, &len);
cp+=len;
}
args->evtype=EV_PLAYER_JOIN_TEAM;
}
// ... changed role to "something" ...
// from TFC, ie: "Joe<15><785><Red>" changed role to "Pyro"
else if(strnmatch(cp, "changed role to ", 16)) {
cp+=16;
args->action="change_role";
// Note:
// new role is in: args->with
args->with=parse_quoted(cp, &len);
cp+=len;
args->evtype=EV_PLAYER_CHANGE_ROLE;
}
// ... scored ...
// ie: Team "CT" scored "7" with "2" players
else if(strnmatch(cp, "scored ", 7)) {
cp+=7;
if(args->player && args->player->team && !args->player->name) {
args->action="team_score";
args->evtype=EV_TEAM_SCORE;
}
else {
args->action="score";
}
cp+=strspn(cp, " ");
}
// anything else...
// Consider any words up to a quote (") to be the 'action'.
else {
begin=cp;
end=strchr(begin, '"')-1;
len=end-begin;
if(len > 0) {
*end='\0';
args->action=begin;
cp=end+1;
cp+=strspn(cp, " ");
}
}
// Look for associated phrases...
// ... against ...
// ie: "Joe<15><785><Red>" triggered "Medic_Heal" against "Bob<27><954><Red>"
if(strnmatch(cp, "against ", 8)) {
cp+=8;
args->target=parse_player(cp, &len);
cp+=len;
}
// ... with ...
// ie: "Joe<15><785><CT>" killed "Sam<17><197><TERRORIST>" with "sg552"
// or: "Joe<15><785><Blue>" triggered "Sentry_Destroyed" against "Sam<17><197><Red>" with "mirvgrenade"
if(strnmatch(cp, "with ", 5)) {
cp+=5;
args->with=parse_quoted(cp, &len);
if(strmatch(args->action, "kill")) {
args->action="weapon_kill";
args->evtype=EV_WEAPON_KILL;
}
cp+=len;
}
return(args);
}
/**
* Check header to see whether the version matches.
*
* @param fp [in] file pointer
*/
static boolean
check_header(FILE *fp)
{
char buf[BINGRAM_HDSIZE], *p;
rdn(fp, buf, 1, BINGRAM_HDSIZE);
p = buf;
#ifdef WORDS_INT
need_conv = FALSE;
#endif
/* version check */
if (strnmatch(p, BINGRAM_IDSTR, strlen(BINGRAM_IDSTR))) {
/* bingram file made by mkbingram before 3.4.2 */
file_version = 3;
p += strlen(BINGRAM_IDSTR) + 1;
} else if (strnmatch(p, BINGRAM_IDSTR_V4, strlen(BINGRAM_IDSTR_V4))) {
/* bingram file made by mkbingram later than 3.5 */
file_version = 4;
p += strlen(BINGRAM_IDSTR_V4) + 1;
} else if (strnmatch(p, BINGRAM_IDSTR_V5, strlen(BINGRAM_IDSTR_V5))) {
/* bingram file made by JuliusLib-4 and later */
file_version = 5;
p += strlen(BINGRAM_IDSTR_V5) + 1;
} else {
/* not a bingram file */
jlog("Error: ngram_read_bin: invalid header\n");
return FALSE;
}
/* word size check (for bingram build by mkbingram 3.3p5 and later */
if (strnmatch(p, BINGRAM_SIZESTR_HEAD, strlen(BINGRAM_SIZESTR_HEAD))) {
p += strlen(BINGRAM_SIZESTR_HEAD);
if (! strnmatch(p, BINGRAM_SIZESTR_BODY, strlen(BINGRAM_SIZESTR_BODY))) {
/* word size does not match (int / short) */
#ifdef WORDS_INT
if (strnmatch(p, BINGRAM_SIZESTR_BODY_2BYTE, strlen(BINGRAM_SIZESTR_BODY_2BYTE))) {
/* this is 2-byte word ID, will convert while reading */
jlog("Warning: ngram_read_bin: 2-bytes bingram, converting to 4 bytes\n");
need_conv = TRUE;
p += strlen(BINGRAM_SIZESTR_BODY_2BYTE) + 1;
} else {
jlog("Error: ngram_read_bin: unknown word byte size!\n");
return FALSE;
}
#else
if (strnmatch(p, BINGRAM_SIZESTR_BODY_4BYTE, strlen(BINGRAM_SIZESTR_BODY_4BYTE))) {
/*** 4bytes to 2bytes not implemented, just terminate here... ***/
jlog("Error: ngram_read_bin: cannot handle 4-bytes bingram\n");
jlog("Error: ngram_read_bin: please use Julius compiled with --enable-words-int\n");
return FALSE;
//p += strlen(BINGRAM_SIZESTR_BODY_4BYTE) + 1;
} else {
jlog("Error: ngram_read_bin: unknown word byte size!\n");
return FALSE;
}
#endif
} else {
p += strlen(BINGRAM_SIZESTR_BODY) + 1;
}
/* byte order check (v4 (rev.3.5) and later) */
if (file_version >= 4) {
if (!strnmatch(p, BINGRAM_BYTEORDER_HEAD, strlen(BINGRAM_BYTEORDER_HEAD))) {
jlog("Error: ngram_read_bin: no information for byte order??\n");
return FALSE;
}
p += strlen(BINGRAM_BYTEORDER_HEAD);
if (! strnmatch(p, BINGRAM_NATURAL_BYTEORDER, strlen(BINGRAM_NATURAL_BYTEORDER))) {
/* file endian and running endian is different, need swapping */
need_swap = TRUE;
} else {
need_swap = FALSE;
}
p += strlen(BINGRAM_NATURAL_BYTEORDER) + 1;
}
} /* if no BINGRAM_SIZESTR_HEAD found, just pass it */
/* in case of V3 bingram file, the unit size of word_id and its byte order
cannot be determined from the header. In that case, we assume
byteorder to be a BIG ENDIAN. The word_id unit size (2byte in normal,
or 4byte if bingram generated with mkbingram with --enable-words-int)
will be automagically detected.
*/
if (file_version < 4) {
/* assume input as big endian */
#ifdef WORDS_BIGENDIAN
need_swap = FALSE;
#else
need_swap = TRUE;
#endif
}
/*jlog("%s",buf);*/
return TRUE;
}
/**
* Sub function to Add a dictionary entry line to the word dictionary.
*
* @param buf [i/o] buffer to hold the input string, will be modified in this function
* @param vnum_p [in] current number of words in @a winfo
* @param linenum [in] current line number of the input
* @param winfo [out] pointer to word dictionary to append the data.
* @param hmminfo [in] HTK %HMM definition data. if NULL, phonemes are ignored.
* @param do_conv [in] TRUE if performing triphone conversion
* @param ok_flag [out] will be set to FALSE if an error occured for this input.
* @param headphone [in] word head silence model name
* @param tailphone [in] word tail silence model name
* @param contextphone [in] silence context name to be used at head and tail
*
* @return FALSE if buf == "DICEND", else TRUE will be returned.
*/
boolean
voca_load_wordlist_line(char *buf, WORD_ID *vnum_p, int linenum, WORD_INFO *winfo, HTK_HMM_INFO *hmminfo, boolean do_conv, boolean *ok_flag, char *headphone, char *tailphone, char *contextphone)
{
char *ptmp, *lp = NULL, *p;
static char cbuf[MAX_HMMNAME_LEN];
static HMM_Logical **tmpwseq = NULL;
static int tmpmaxlen;
int len;
HMM_Logical *tmplg;
boolean pok, first;
int vnum;
vnum = *vnum_p;
if (strmatch(buf, "DICEND")) return FALSE;
/* allocate temporal work area for the first call */
if (tmpwseq == NULL) {
tmpmaxlen = PHONEMELEN_STEP;
tmpwseq = (HMM_Logical **)mymalloc(sizeof(HMM_Logical *) * tmpmaxlen);
}
/* backup whole line for debug output */
strcpy(bufbak, buf);
/* Output string */
if ((ptmp = mystrtok_quote(buf, " \t\n")) == NULL) {
jlog("Error: voca_load_wordlist: line %d: corrupted data:\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
winfo->wname[vnum] = strcpy((char *)mybmalloc2(strlen(ptmp)+1, &(winfo->mroot)), ptmp);
/* reset transparent flag */
winfo->is_transparent[vnum] = FALSE;
/* just move pointer to next token */
if ((ptmp = mystrtok_movetonext(NULL, " \t\n")) == NULL) {
jlog("Error: voca_load_wordlist: line %d: corrupted data:\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
#ifdef CLASS_NGRAM
winfo->cprob[vnum] = 0.0; /* prob = 1.0, logprob = 0.0 */
#endif
if (ptmp[0] == '@') { /* class N-gram prob */
#ifdef CLASS_NGRAM
/* word probability within the class (for class N-gram) */
/* format: classname @classprob wordname [output] phoneseq */
/* classname equals to wname, and wordname will be omitted */
/* format: @%f (log scale) */
/* if "@" not found or "@0", it means class == word */
if ((ptmp = mystrtok(NULL, " \t\n")) == NULL) {
jlog("Error: voca_load_wordlist: line %d: corrupted data:\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
if (ptmp[1] == '\0') { /* space between '@' and figures */
jlog("Error: voca_load_wordlist: line %d: value after '@' missing, maybe wrong space?\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
winfo->cprob[vnum] = atof(&(ptmp[1]));
if (winfo->cprob[vnum] != 0.0) winfo->cwnum++;
/* read next word entry (just skip them) */
if ((ptmp = mystrtok(NULL, " \t\n")) == NULL) {
jlog("Error: voca_load_wordlist: line %d: corrupted data:\n> %s\n", linenum,bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
/* move to the next word entry */
if ((ptmp = mystrtok_movetonext(NULL, " \t\n")) == NULL) {
jlog("Error: voca_load_wordlist: line %d: corrupted data:\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
#else /* ~CLASS_NGRAM */
jlog("Error: voca_load_wordlist: line %d: cannot handle in-class word probability\n> %s\n", linenum, ptmp, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
#endif /* CLASS_NGRAM */
}
/* OutputString */
switch(ptmp[0]) {
case '[': /* ignore transparency */
ptmp = mystrtok_quotation(NULL, " \t\n", '[', ']', 0);
break;
case '{': /* ignore transparency */
ptmp = mystrtok_quotation(NULL, " \t\n", '{', '}', 0);
break;
default:
/* ALLOW no entry for output */
/* same as wname is used */
ptmp = winfo->wname[vnum];
}
if (ptmp == NULL) {
jlog("Error: voca_load_htkdict: line %d: corrupted data:\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
winfo->woutput[vnum] = strcpy((char *)mybmalloc2(strlen(ptmp)+1, &(winfo->mroot)), ptmp);
/* phoneme sequence */
if (hmminfo == NULL) {
/* don't read */
winfo->wseq[vnum] = NULL;
winfo->wlen[vnum] = 0;
} else {
len = 0;
first = TRUE;
pok = TRUE;
for (;;) {
if (do_conv) {
if (first) {
/* init phone cycler */
cycle_triphone(NULL);
/* insert head phone at beginning of word */
if (contextphone) {
if (strlen(contextphone) >= MAX_HMMNAME_LEN) {
jlog("Error: voca_load_htkdict: line %d: too long phone name: %s\n", linenum, contextphone);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
cycle_triphone(contextphone);
} else {
cycle_triphone("NULL_C");
}
if ((lp = mystrtok(NULL, " \t\n")) == NULL) {
jlog("Error: voca_load_wordlist: line %d: word %s has no phoneme:\n> %s\n", linenum, winfo->wname[vnum], bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
if (strlen(lp) >= MAX_HMMNAME_LEN) {
jlog("Error: voca_load_htkdict: line %d: too long phone name: %s\n", linenum, lp);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
p = cycle_triphone(lp);
first = FALSE;
} else { /* do_conv, not first */
if (lp != NULL) { /* some token processed at last loop */
lp = mystrtok(NULL, " \t\n");
if (lp != NULL) {
/* token exist */
if (strlen(lp) >= MAX_HMMNAME_LEN) {
jlog("Error: voca_load_htkdict: line %d: too long phone name: %s\n", linenum, lp);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
p = cycle_triphone(lp);
} else {
/* no more token, insert tail phone at end of word */
if (contextphone) {
if (strlen(contextphone) >= MAX_HMMNAME_LEN) {
jlog("Error: voca_load_htkdict: line %d: too long phone name: %s\n", linenum, contextphone);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
p = cycle_triphone(contextphone);
} else {
p = cycle_triphone("NULL_C");
}
}
} else { /* no more token at last input */
/* flush tone cycler */
p = cycle_triphone_flush();
}
}
} else { /* not do_conv */
if (first) {
p = lp = headphone;
first = FALSE;
} else {
if (lp != NULL) { /* some token processed at last loop */
p = lp = mystrtok(NULL, " \t\n");
/* if no more token, use tailphone */
if (lp == NULL) p = tailphone;
} else {
/* no more token at last input, exit loop */
p = NULL;
}
}
}
if (p == NULL) break;
/* for headphone and tailphone, their context should not be handled */
/* and when they appear as context they should be replaced by contextphone */
if (do_conv) {
center_name(p, cbuf);
if (contextphone) {
if (strmatch(cbuf, contextphone)) {
if (len == 0) {
p = headphone;
} else if (lp == NULL) {
p = tailphone;
}
}
} else {
if (strmatch(cbuf, "NULL_C")) {
if (len == 0) {
p = headphone;
} else if (lp == NULL) {
p = tailphone;
}
} else {
if (strnmatch(p, "NULL_C", 6)) {
if (strnmatch(&(p[strlen(p)-6]), "NULL_C", 6)) {
p = cbuf;
} else {
p = rightcenter_name(p, cbuf);
}
} else if (strnmatch(&(p[strlen(p)-6]), "NULL_C", 6)) {
p = leftcenter_name(p, cbuf);
}
}
}
}
//printf("[[%s]]\n", p);
/* both defined/pseudo phone is allowed */
tmplg = htk_hmmdata_lookup_logical(hmminfo, p);
if (tmplg == NULL) {
/* not found */
if (do_conv) {
/* logical phone was not found */
jlog("Error: voca_load_wordlist: line %d: logical phone \"%s\" not found\n", linenum, p);
snprintf(cbuf,MAX_HMMNAME_LEN,"%s", p);
} else {
jlog("Error: voca_load_wordlist: line %d: phone \"%s\" not found\n", linenum, p);
snprintf(cbuf, MAX_HMMNAME_LEN, "%s", p);
}
add_to_error(winfo, cbuf);
pok = FALSE;
} else {
/* found */
if (len >= tmpmaxlen) {
/* expand wseq area by PHONEMELEN_STEP */
tmpmaxlen += PHONEMELEN_STEP;
tmpwseq = (HMM_Logical **)myrealloc(tmpwseq, sizeof(HMM_Logical *) * tmpmaxlen);
}
/* store to temporal buffer */
tmpwseq[len] = tmplg;
}
len++;
}
if (!pok) { /* error in phoneme */
jlog("Error: voca_load_wordlist: the line content was: %s\n", bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
if (len == 0) {
jlog("Error: voca_load_wordlist: line %d: no phone specified:\n> %s\n", linenum, bufbak);
winfo->errnum++;
*ok_flag = FALSE;
return TRUE;
}
/* store to winfo */
winfo->wseq[vnum] = (HMM_Logical **)mybmalloc2(sizeof(HMM_Logical *) * len, &(winfo->mroot));
memcpy(winfo->wseq[vnum], tmpwseq, sizeof(HMM_Logical *) * len);
winfo->wlen[vnum] = len;
winfo->wton[vnum] = 0;
}
vnum++;
*vnum_p = vnum;
return(TRUE);
}
/*
** A "local" function of MatchFilter(). Returns 1 or 0 depending on whether or
** not #item# contains the first attribute specified by #filter#. Updates
** #filter# to point to the next attribute.
*/
static int
MatchFilterAndUpdate(const char *item,
const char **filter) {
char firstChar;
const char *itemAttribute;
char operator;
int returnValue;
char simpleName[127 + 1];
char simplePattern[127 + 1];
const char *value;
const char *valueEnd;
size_t valueLen;
/* Allow leading and trailing spaces for legibility. */
while(**filter == ' ')
(*filter)++;
firstChar = **filter;
if(firstChar == '(') {
/* Nested pattern. */
(*filter)++;
returnValue = MatchFilterAndUpdate(item, filter);
if(**filter == ')')
(*filter)++;
}
else if((firstChar == '!') || (firstChar == '|') || (firstChar == '&')) {
/* Prefix operator: unary not, n-ary or, n-ary and */
(*filter)++;
if(firstChar == '!') {
returnValue = !MatchFilterAndUpdate(item, filter);
}
else {
returnValue = firstChar == '&';
/* To advance filter correctly we must avoid short-circuit matches. */
while((**filter != '\0') && (**filter != ')')) {
if(MatchFilterAndUpdate(item, filter)) {
if(firstChar == '|')
returnValue = 1;
}
else if(firstChar == '&')
returnValue = 0;
}
}
}
else {
/* Grab the individual parts of the <attribute><op><pattern> pair. */
if(!GETTOK(simpleName, *filter, "<=>~", filter))
FAIL1("MatchFilterAndUpdate: parse error in %s\n", filter);
strcat(simpleName, ":");
operator = **filter;
if(operator != '=') {
(*filter)++; /* Skip < > or ~ */
if(**filter != '=')
FAIL1("MatchFilterAndUpdate: parse error in %s\n", filter);
}
(*filter)++; /* Skip '=' */
if(!GETTOK(simplePattern, *filter, ") ", filter))
FAIL1("MatchFilterAndUpdate: parse error in %s\n", filter);
/* Now match against the value of each occurence of <attribute> in item. */
for(itemAttribute = strstr(item, simpleName); itemAttribute != NULL;
itemAttribute = strstr(itemAttribute + 1, simpleName)) {
value = itemAttribute + strlen(simpleName);
valueEnd = strchr(value, '\t');
valueLen = (valueEnd == NULL) ? strlen(value) : (valueEnd - value);
if((operator == '=') && strnmatch(value, simplePattern, valueLen))
break;
else if((operator == '<') &&
(strncmp(value, simplePattern, valueLen) <= 0))
break;
else if((operator == '>') &&
(strncmp(value, simplePattern, valueLen) >= 0))
break;
else if((operator == '~') && strnmatch(value, simplePattern, valueLen))
/* We recognize ~= but treat it as simple = */
break;
}
returnValue = itemAttribute != NULL;
}
while(**filter == ' ')
(*filter)++;
return returnValue;
}