int oe_02_main (int argc, char **argv) {
ng_t ng;
arpa_lm_t arpa_ng;
char input_string[500];
int num_of_args;
char *args[MAX_ARGS];
char *lm_filename_arpa;
char *lm_filename_binary;
flag told_to_quit;
flag inconsistant_parameters;
flag backoff_from_unk_inc;
flag backoff_from_unk_exc;
flag backoff_from_ccs_inc;
flag backoff_from_ccs_exc;
flag arpa_lm;
flag binary_lm;
flag include_unks;
char *fb_list_filename;
char *probs_stream_filename;
char *annotation_filename;
char *text_stream_filename;
char *oov_filename;
char *ccs_filename;
int generate_size;
int random_seed;
double log_base;
char wlist_entry[1024];
char current_cc[200];
vocab_sz_t current_cc_id;
FILE *context_cues_fp;
int n;
/* Process command line */
report_version(&argc,argv);
if (pc_flagarg(&argc, argv,"-help") ||
argc == 1 ||
(strcmp(argv[1],"-binary") && strcmp(argv[1],"-arpa"))) {
oe_02_help_message();
exit(1);
}
lm_filename_arpa = rr_salloc(pc_stringarg(&argc, argv,"-arpa",""));
if (strcmp(lm_filename_arpa,""))
arpa_lm = 1;
else
arpa_lm = 0;
lm_filename_binary = rr_salloc(pc_stringarg(&argc, argv,"-binary",""));
if (strcmp(lm_filename_binary,""))
binary_lm = 1;
else
binary_lm = 0;
if (arpa_lm && binary_lm)
quit(-1,"Error : Can't use both -arpa and -binary flags.\n");
if (!arpa_lm && !binary_lm)
quit(-1,"Error : Must specify either a binary or an arpa format language model.\n");
ccs_filename = rr_salloc(pc_stringarg(&argc, argv,"-context",""));
if (binary_lm && strcmp(ccs_filename,""))
fprintf(stderr,"Warning - context cues file not needed with binary language model file.\nWill ignore it.\n");
pc_report_unk_args(&argc,argv,2);
/* Load language model */
if (arpa_lm) {
fprintf(stderr,"Reading in language model from file %s\n",
lm_filename_arpa);
load_arpa_lm(&arpa_ng,lm_filename_arpa);
}else {
fprintf(stderr,"Reading in language model from file %s\n",
lm_filename_binary);
load_lm(&ng,lm_filename_binary);
}
fprintf(stderr,"\nDone.\n");
n=arpa_lm?
arpa_ng.n:
ng.n;
if (arpa_lm) {
arpa_ng.context_cue =
(flag *) rr_calloc(arpa_ng.table_sizes[0],sizeof(flag));
arpa_ng.no_of_ccs = 0;
if (strcmp(ccs_filename,"")) {
context_cues_fp = rr_iopen(ccs_filename);
while (fgets (wlist_entry, sizeof (wlist_entry),context_cues_fp)) {
if (strncmp(wlist_entry,"##",2)==0) continue;
sscanf (wlist_entry, "%s ",current_cc);
warn_on_wrong_vocab_comments(wlist_entry);
if (sih_lookup(arpa_ng.vocab_ht,current_cc,¤t_cc_id) == 0)
quit(-1,"Error : %s in the context cues file does not appear in the vocabulary.\n",current_cc);
arpa_ng.context_cue[(unsigned short) current_cc_id] = 1;
arpa_ng.no_of_ccs++;
fprintf(stderr,"Context cue word : %s id = %lld\n",current_cc,current_cc_id);
}
rr_iclose(context_cues_fp);
}
}
/* Process commands */
told_to_quit = 0;
num_of_args = 0;
while (!feof(stdin) && !told_to_quit) {
printf("evallm : \n");
fgets(input_string, sizeof(input_string), stdin);
if(strlen(input_string) < sizeof(input_string)-1)
input_string[strlen(input_string)-1] = '\0'; //chop new-line
else
quit(1, "evallm input exceeds size of input buffer");
if (!feof(stdin)) {
parse_comline(input_string,&num_of_args,args);
log_base = pc_doublearg(&num_of_args,args,"-log_base",10.0);
backoff_from_unk_inc = pc_flagarg(&num_of_args,args,"-backoff_from_unk_inc");
backoff_from_ccs_inc = pc_flagarg(&num_of_args,args,"-backoff_from_ccs_inc");
backoff_from_unk_exc = pc_flagarg(&num_of_args,args,"-backoff_from_unk_exc");
backoff_from_ccs_exc = pc_flagarg(&num_of_args,args,"-backoff_from_ccs_exc");
include_unks = pc_flagarg(&num_of_args,args,"-include_unks");
fb_list_filename = rr_salloc(pc_stringarg(&num_of_args,args,"-backoff_from_list",""));
text_stream_filename =
rr_salloc(pc_stringarg(&num_of_args,args,"-text",""));
probs_stream_filename =
rr_salloc(pc_stringarg(&num_of_args,args,"-probs",""));
annotation_filename =
rr_salloc(pc_stringarg(&num_of_args,args,"-annotate",""));
oov_filename = rr_salloc(pc_stringarg(&num_of_args,args,"-oovs",""));
generate_size = pc_intarg(&num_of_args,args,"-size",10000);
random_seed = pc_intarg(&num_of_args,args,"-seed",-1);
inconsistant_parameters = 0;
if (backoff_from_unk_inc && backoff_from_unk_exc) {
fprintf(stderr,"Error : Cannot specify both exclusive and inclusive forced backoff.\n");
fprintf(stderr,"Use only one of -backoff_from_unk_exc and -backoff_from_unk_inc\n");
inconsistant_parameters = 1;
}
if (backoff_from_ccs_inc && backoff_from_ccs_exc) {
fprintf(stderr,"Error : Cannot specify both exclusive and inclusive forced backoff.\n");
fprintf(stderr,"Use only one of -backoff_from_ccs_exc and -backoff_from_ccs_inc\n");
inconsistant_parameters = 1;
}
if (num_of_args > 0) {
if (!inconsistant_parameters) {
if (!strcmp(args[0],"perplexity")) {
compute_perplexity(&ng,
&arpa_ng,
text_stream_filename,
probs_stream_filename,
annotation_filename,
oov_filename,
fb_list_filename,
backoff_from_unk_inc,
backoff_from_unk_exc,
backoff_from_ccs_inc,
backoff_from_ccs_exc,
arpa_lm,
include_unks,
log_base);
}else
/* do perplexity sentence by sentence [20090612] (air) */
if (!strcmp(args[0],"uttperp")) {
FILE *uttfh,*tempfh;
char utt[4096]; /* live dangerously... */
char tmpfil[128];
if ((uttfh = fopen(text_stream_filename,"r")) == NULL) {
printf("Error: can't open %s\n",text_stream_filename);
exit(1);
}
char *template = "uttperp_XXXXXX";// CHANGED HLW
mkstemp(template);// CHANGED HLW
int oe_11_main(int argc,char** argv)
{
arpa_lm_t arpa_lm,lm1,lm2;
char header1[MAX_HEADER];
char header2[MAX_HEADER];
flag backoff_from_unk_inc,backoff_from_unk_exc,backoff_from_ccs_inc,backoff_from_ccs_exc;
char *lmfile1,*lmfile2,*newlmfile, *wtfile;
char *fb_list_filename,*ccs_filename;
fb_info *fb1,*fb2,*fb;
double w1,w2;
if (pc_flagarg(&argc, argv,"-help") || argc == 1) {
oe_05_help_message();
exit(1);
}
lmfile1 = rr_salloc(pc_stringarg(&argc, argv,"-lm1",""));
if (0 == strcmp(lmfile1, "")) {
fprintf(stderr, "ERROR: Please specify a first input file with -lm1.\n");
oe_05_help_message();
}
lmfile2 = rr_salloc(pc_stringarg(&argc, argv,"-lm2",""));
if (0 == strcmp(lmfile2, "")) {
fprintf(stderr, "ERROR: Please specify a second input file with -lm2.\n");
oe_05_help_message();
}
newlmfile = rr_salloc(pc_stringarg(&argc, argv,"-lm",""));
if (0 == strcmp(newlmfile, "")) {
fprintf(stderr, "ERROR: Please specify a destination file with -lm.\n");
oe_05_help_message();
}
fb_list_filename = rr_salloc(pc_stringarg(&argc, argv,"-forced_backoff",""));
wtfile= rr_salloc(pc_stringarg(&argc, argv,"-weight",""));
if (0 == strcmp(wtfile, "")) {
fprintf(stderr, "ERROR: Please specify a weights file with -weight.\n");
oe_05_help_message();
}
ccs_filename= rr_salloc(pc_stringarg(&argc, argv,"-context",""));
backoff_from_unk_inc = pc_flagarg(&argc,argv,"-backoff_from_unk_inc");
backoff_from_ccs_inc = pc_flagarg(&argc,argv,"-backoff_from_ccs_inc");
backoff_from_unk_exc = pc_flagarg(&argc,argv,"-backoff_from_unk_exc");
backoff_from_ccs_exc = pc_flagarg(&argc,argv,"-backoff_from_ccs_exc");
robust_load_arpa_lm(&lm1,lmfile1,header1,MAX_HEADER);
robust_load_arpa_lm(&lm2,lmfile2,header2,MAX_HEADER);
load_weights(&w1,&w2,wtfile);
printf ("\ncombine lms\n");
combine_lm(&arpa_lm,&lm1,&lm2);
printf ("\nloading context cues.\n");
load_context_cue(&arpa_lm,ccs_filename);
load_context_cue(&lm1,ccs_filename);
load_context_cue(&lm2,ccs_filename);
fb=gen_fb_list(arpa_lm.vocab_ht,
arpa_lm.vocab_size,
arpa_lm.vocab,
arpa_lm.context_cue,
backoff_from_unk_inc,
backoff_from_unk_exc,
backoff_from_ccs_inc,
backoff_from_ccs_exc,
fb_list_filename);
fb1=gen_fb_list(lm1.vocab_ht,
lm1.vocab_size,
lm1.vocab,
lm1.context_cue,
backoff_from_unk_inc,
backoff_from_unk_exc,
backoff_from_ccs_inc,
backoff_from_ccs_exc,
fb_list_filename);
fb2=gen_fb_list(lm2.vocab_ht,
lm2.vocab_size,
lm2.vocab,
lm2.context_cue,
backoff_from_unk_inc,
backoff_from_unk_exc,
backoff_from_ccs_inc,
backoff_from_ccs_exc,
fb_list_filename);
printf ("\nrecaculate oov probabilities.\n");
recalc_oov_prob(&arpa_lm,&lm1,&lm2);
printf ("\ncheck probabilities\n");
check_prob(&arpa_lm,&lm1,&lm2,fb1,fb2,w1,w2);
printf ("\ncalculate interpolated probabilities\n");
calc_interpolated_prob(&arpa_lm,&lm1,&lm2,fb1,fb2,w1,w2);
printf ("\ncalculate backoff weights\n");
calc_backoff_weight(&arpa_lm,fb);
printf ("\nwrite interpolated lm\n");
write_interpolated_lm(&arpa_lm,newlmfile,header1,header2,2);
printf ("\nfinished\n");
return 0;
}
void main(int argc, char *argv[]) {
int i,j;
char *vocab_filename;
FILE *tempfile;
char tempfiles_directory[1000];
int vocab_size;
FILE *vocab_file;
int verbosity;
int buffer_size;
int position_in_buffer;
int number_of_tempfiles;
int max_files;
int fof_size;
unsigned short *buffer;
unsigned short *placeholder;
unsigned short *temp_ngram;
int temp_count;
char temp_word[500];
char temp_word2[500];
char *temp_file_root;
char *temp_file_ext;
char *host_name;
int proc_id;
struct utsname uname_info;
flag write_ascii;
/* Vocab hash table things */
struct hash_table vocabulary;
unsigned long hash_size;
unsigned long M;
tempfile = NULL; /* Just to prevent compilation warnings. */
report_version(&argc,argv);
verbosity = pc_intarg(&argc,argv,"-verbosity",DEFAULT_VERBOSITY);
/* Process command line */
if (pc_flagarg( &argc, argv,"-help") || argc==1) {
fprintf(stderr,"text2idngram - Convert a text stream to an id n-gram stream.\n");
fprintf(stderr,"Usage : text2idngram -vocab .vocab \n");
fprintf(stderr," [ -buffer 100 ]\n");
fprintf(stderr," [ -hash %d ]\n",DEFAULT_HASH_SIZE);
fprintf(stderr," [ -temp %s ]\n",DEFAULT_TEMP);
fprintf(stderr," [ -files %d ]\n",DEFAULT_MAX_FILES);
fprintf(stderr," [ -gzip | -compress ]\n");
fprintf(stderr," [ -verbosity %d ]\n",
DEFAULT_VERBOSITY);
fprintf(stderr," [ -n 3 ]\n");
fprintf(stderr," [ -write_ascii ]\n");
fprintf(stderr," [ -fof_size 10 ]\n");
exit(1);
}
pc_message(verbosity,2,"text2idngram\n");
n = pc_intarg( &argc, argv, "-n",DEFAULT_N);
placeholder = (unsigned short *) rr_malloc(sizeof(unsigned short)*n);
temp_ngram = (unsigned short *) rr_malloc(sizeof(unsigned short)*n);
hash_size = pc_intarg( &argc, argv, "-hash",DEFAULT_HASH_SIZE);
buffer_size = pc_intarg( &argc, argv, "-buffer",STD_MEM);
write_ascii = pc_flagarg(&argc,argv,"-write_ascii");
fof_size = pc_intarg(&argc,argv,"-fof_size",10);
max_files = pc_intarg( &argc, argv, "-files",DEFAULT_MAX_FILES);
vocab_filename = salloc(pc_stringarg( &argc, argv, "-vocab", "" ));
if (!strcmp("",vocab_filename)) {
quit(-1,"text2idngram : Error : Must specify a vocabulary file.\n");
}
strcpy(tempfiles_directory,pc_stringarg( &argc, argv, "-temp",
DEFAULT_TEMP));
if (pc_flagarg(&argc,argv,"-compress")) {
temp_file_ext = salloc(".Z");
}
else {
if (pc_flagarg(&argc,argv,"-gzip")) {
temp_file_ext = salloc(".gz");
}
else {
temp_file_ext = salloc("");
}
}
uname(&uname_info);
host_name = salloc(uname_info.nodename);
proc_id = getpid();
sprintf(temp_word,"%s%s.%d.",TEMP_FILE_ROOT,host_name,proc_id);
temp_file_root = salloc(temp_word);
pc_report_unk_args(&argc,argv,verbosity);
/* If the last charactor in the directory name isn't a / then add one. */
if (tempfiles_directory[strlen(tempfiles_directory)-1] != '/') {
strcat(tempfiles_directory,"/");
}
pc_message(verbosity,2,"Vocab : %s\n",vocab_filename);
pc_message(verbosity,2,"N-gram buffer size : %d\n",buffer_size);
pc_message(verbosity,2,"Hash table size : %d\n",hash_size);
pc_message(verbosity,2,"Temp directory : %s\n",tempfiles_directory);
pc_message(verbosity,2,"Max open files : %d\n",max_files);
pc_message(verbosity,2,"FOF size : %d\n",fof_size);
pc_message(verbosity,2,"n : %d\n",n);
buffer_size *= (1000000/(sizeof(unsigned short)*n));
/* Allocate memory for hash table */
fprintf(stderr,"Initialising hash table...\n");
M = nearest_prime(hash_size);
new_hashtable(&vocabulary,M);
/* Read in the vocabulary */
vocab_size = 0;
vocab_file = rr_iopen(vocab_filename);
pc_message(verbosity,2,"Reading vocabulary...\n");
while (fgets (temp_word, sizeof(temp_word),vocab_file)) {
if (strncmp(temp_word,"##",2)==0) continue;
sscanf (temp_word, "%s ",temp_word2);
/* Check for repeated words in the vocabulary */
if (index2(&vocabulary,temp_word2) != 0) {
fprintf(stderr,"======================================================\n");
fprintf(stderr,"WARNING: word %s is repeated in the vocabulary.\n",temp_word);
fprintf(stderr,"=======================================================\n");
}
if (strncmp(temp_word,"#",1)==0) {
fprintf(stderr,"\n\n===========================================================\n");
fprintf(stderr,":\nWARNING: line assumed NOT a comment:\n");
fprintf(stderr, ">>> %s <<<\n",temp_word);
fprintf(stderr, " '%s' will be included in the vocabulary.\n",temp_word2);
fprintf(stderr, " (comments must start with '##')\n");
fprintf(stderr,"===========================================================\n\n");
}
vocab_size++;
add_to_hashtable(&vocabulary,hash(temp_word2,M),temp_word2,vocab_size);
}
if (vocab_size > MAX_VOCAB_SIZE) {
quit(-1,"text2idngram : Error : Vocabulary size exceeds maximum.\n");
}
pc_message(verbosity,2,"Allocating memory for the n-gram buffer...\n");
buffer=(unsigned short*) rr_malloc(n*(buffer_size+1)*sizeof(unsigned short));
number_of_tempfiles = 0;
/* Read text into buffer */
/* Read in the first ngram */
position_in_buffer = 0;
for (i=0;i<=n-1;i++) {
get_word(stdin,temp_word);
add_to_buffer(index2(&vocabulary,temp_word),0,i,buffer);
}
while (!rr_feof(stdin)) {
/* Fill up the buffer */
pc_message(verbosity,2,"Reading text into the n-gram buffer...\n");
pc_message(verbosity,2,"20,000 n-grams processed for each \".\", 1,000,000 for each line.\n");
while ((position_in_buffer<buffer_size) && (!rr_feof(stdin))) {
position_in_buffer++;
if (position_in_buffer % 20000 == 0) {
if (position_in_buffer % 1000000 == 0) {
pc_message(verbosity,2,".\n");
}
else {
pc_message(verbosity,2,".");
}
}
for (i=1;i<=n-1;i++) {
add_to_buffer(buffer_contents(position_in_buffer-1,i,buffer),
position_in_buffer,i-1,buffer);
}
if (get_word(stdin,temp_word) == 1) {
add_to_buffer(index2(&vocabulary,temp_word),position_in_buffer,
n-1,buffer);
}
}
for (i=0;i<=n-1;i++) {
placeholder[i] = buffer_contents(position_in_buffer,i,buffer);
}
/* Sort buffer */
pc_message(verbosity,2,"\nSorting n-grams...\n");
qsort((void*) buffer,(size_t) position_in_buffer,
n*sizeof(unsigned short),compare_ngrams);
/* Output the buffer to temporary BINARY file */
number_of_tempfiles++;
sprintf(temp_word,"%s%s%hu%s",tempfiles_directory,temp_file_root,
number_of_tempfiles,temp_file_ext);
pc_message(verbosity,2,"Writing sorted n-grams to temporary file %s\n",
temp_word);
tempfile = rr_oopen(temp_word);
for (i=0;i<=n-1;i++) {
temp_ngram[i] = buffer_contents(0,i,buffer);
if (temp_ngram[i] > MAX_VOCAB_SIZE) {
quit(-1,"Invalid trigram in buffer.\nAborting");
}
}
temp_count = 1;
for (i=1;i<=position_in_buffer;i++) {
if (!compare_ngrams(temp_ngram,&buffer[i*n])) {
temp_count++;
}
else {
for (j=0;j<=n-1;j++) {
rr_fwrite(&temp_ngram[j],sizeof(unsigned short),1,
tempfile,"temporary n-gram ids");
temp_ngram[j] = buffer_contents(i,j,buffer);
}
rr_fwrite(&temp_count,sizeof(int),1,tempfile,
"temporary n-gram counts");
temp_count = 1;
}
}
rr_oclose(tempfile);
for (i=0;i<=n-1;i++) {
add_to_buffer(placeholder[i],0,i,buffer);
}
position_in_buffer = 0;
}
/* Merge the temporary files, and output the result to standard output */
pc_message(verbosity,2,"Merging temporary files...\n");
merge_tempfiles(1,
number_of_tempfiles,
temp_file_root,
temp_file_ext,
max_files,
tempfiles_directory,
stdout,
write_ascii,
fof_size);
pc_message(verbosity,0,"text2idngram : Done.\n");
exit(0);
}
int main(int argc, char *argv[]) {
int verbosity;
int vocab_size;
FILE *vocab_file;
int buffer_size;
flag write_ascii;
int max_files;
int number_of_tempfiles;
char *vocab_filename;
char *idngram_filename;
char temp_word[MAX_WORD_LENGTH];
char temp_word2[MAX_WORD_LENGTH];
char temp_word3[MAX_WORD_LENGTH];
flag contains_unks;
int position_in_buffer;
FILE *outfile;
FILE *tempfile;
FILE *non_unk_fp;
ngram_rec *buffer;
flag same_ngram;
int i;
int j;
int fof_size;
int size_of_rec;
char temp_directory[1000];
char *temp_file_ext;
/* Vocab hash table things */
struct idngram_hash_table vocabulary;
unsigned long hash_size;
unsigned long M;
wordid_t *current_ngram;
int current_count;
wordid_t *sort_ngram;
int sort_count;
/* Process command line */
report_version(&argc,argv);
if (argc == 1 || pc_flagarg(&argc, argv,"-help")) {
/* Display help message */
help_message();
exit(1);
}
n = pc_intarg( &argc, argv, "-n",DEFAULT_N);
hash_size = pc_intarg( &argc, argv, "-hash",DEFAULT_HASH_SIZE);
buffer_size = pc_intarg( &argc, argv, "-buffer",STD_MEM);
write_ascii = pc_flagarg(&argc,argv,"-write_ascii");
verbosity = pc_intarg(&argc,argv,"-verbosity",DEFAULT_VERBOSITY);
max_files = pc_intarg( &argc, argv, "-files",DEFAULT_MAX_FILES);
fof_size = pc_intarg(&argc,argv,"-fof_size",10);
vocab_filename = salloc(pc_stringarg( &argc, argv, "-vocab", "" ));
idngram_filename = salloc(pc_stringarg( &argc, argv, "-idngram", "" ));
if (!strcmp("",vocab_filename))
quit(-1,"Error : Must specify a vocabulary file.\n");
if (!strcmp("",idngram_filename))
quit(-1,"text2idngram : Error : Must specify idngram file.\n");
if (pc_flagarg(&argc,argv,"-compress"))
temp_file_ext = salloc(".Z");
else {
if (pc_flagarg(&argc,argv,"-gzip"))
temp_file_ext = salloc(".gz");
else
temp_file_ext = salloc("");
}
strcpy(temp_directory, "cmuclmtk-XXXXXX");
if (mkdtemp(temp_directory) == NULL) {
quit(-1, "Failed to create temporary folder: %s\n", strerror(errno));
}
pc_report_unk_args(&argc,argv,verbosity);
outfile = rr_fopen(idngram_filename,"wb");
pc_message(verbosity,2,"Vocab : %s\n",vocab_filename);
pc_message(verbosity,2,"Output idngram : %s\n",idngram_filename);
pc_message(verbosity,2,"Buffer size : %d\n",buffer_size);
pc_message(verbosity,2,"Hash table size : %d\n",hash_size);
pc_message(verbosity,2,"Max open files : %d\n",max_files);
pc_message(verbosity,2,"n : %d\n",n);
pc_message(verbosity,2,"FOF size : %d\n",fof_size);
size_of_rec = (sizeof(wordid_t) * n) + 16 - (( n* sizeof(wordid_t)) % 16);
buffer_size *= (1000000/((sizeof(ngram_rec) + size_of_rec)));
fprintf(stderr,"buffer size = %d\n",buffer_size);
/* Allocate memory for hash table */
fprintf(stderr,"Initialising hash table...\n");
M = nearest_prime(hash_size);
new_idngram_hashtable(&vocabulary,M);
/* Read in the vocabulary */
vocab_size = 0;
vocab_file = rr_iopen(vocab_filename);
pc_message(verbosity,2,"Reading vocabulary...\n");
while (fgets (temp_word, sizeof(temp_word),vocab_file)) {
if (strncmp(temp_word,"##",2)==0) continue;
sscanf (temp_word, "%s ",temp_word2);
/* Check for vocabulary order */
if (vocab_size > 0 && strcmp(temp_word2,temp_word3)<0)
quit(-1,"wngram2idngram : Error : Vocabulary is not alphabetically ordered.\n");
/* Check for repeated words in the vocabulary */
if (index2(&vocabulary,temp_word2) != 0)
warn_on_repeated_words(temp_word);
warn_on_wrong_vocab_comments(temp_word);
vocab_size++;
add_to_idngram_hashtable(&vocabulary,idngram_hash(temp_word2,M),temp_word2,vocab_size);
strcpy(temp_word3,temp_word2);
}
if (vocab_size > MAX_VOCAB_SIZE)
quit(-1,"Error : Vocabulary size exceeds maximum.\n");
pc_message(verbosity,2,"Allocating memory for the buffer...\n");
buffer=(ngram_rec *) rr_malloc((buffer_size+1)*sizeof(ngram_rec));
for (i=0;i<=buffer_size;i++)
buffer[i].word = (wordid_t *) rr_malloc(n*sizeof(wordid_t));
/* Open the "non-OOV" tempfile */
sprintf(temp_word, "%s/1%s", temp_directory, temp_file_ext);
non_unk_fp = rr_fopen(temp_word,"w");
pc_message(verbosity,2,"Writing non-OOV counts to temporary file %s\n",
temp_word);
number_of_tempfiles = 1;
current_ngram = (wordid_t *) rr_malloc(n*sizeof(wordid_t));
sort_ngram = (wordid_t *) rr_malloc(n*sizeof(wordid_t));
/* Read text into buffer */
position_in_buffer = 0;
while (!rr_feof(stdin)) {
for (i=0;i<=n-1;i++) {
get_word(stdin,temp_word);
current_ngram[i]=index2(&vocabulary,temp_word);
}
if (scanf("%d",¤t_count) != 1)
if (!rr_feof(stdin))
quit(-1,"Error reading n-gram count from stdin.\n");
if (!rr_feof(stdin)) {
contains_unks = 0;
for (i=0;i<=n-1;i++) {
if (!current_ngram[i])
contains_unks = 1;
}
if (contains_unks) {
/* Write to buffer */
position_in_buffer++;
if (position_in_buffer >= buffer_size) {
/* Sort buffer */
pc_message(verbosity,2,
"Sorting n-grams which include an OOV word...\n");
qsort((void*) buffer,(size_t) position_in_buffer,
sizeof(ngram_rec),compare_ngrams2);
pc_message(verbosity,2,"Done.\n");
/* Write buffer to temporary file */
number_of_tempfiles++;
sprintf(temp_word,"%s/%hu%s", temp_directory,
number_of_tempfiles,temp_file_ext);
pc_message(verbosity,2,
"Writing sorted OOV-counts buffer to temporary file %s\n",
temp_word);
tempfile = rr_fopen(temp_word,"w");
for (i=0;i<=n-1;i++)
sort_ngram[i] = buffer[0].word[i];
sort_count = buffer[0].count;
for (i=0;i<=position_in_buffer-2;i++) {
same_ngram = 1;
for (j=n-1;j>=0;j--) {
if (buffer[i].word[j] != sort_ngram[j]) {
same_ngram = 0;
j = -1;
}
}
if (same_ngram)
sort_count += buffer[i].count;
else {
for (j=0;j<=n-1;j++) {
rr_fwrite((char*)&sort_ngram[j],sizeof(wordid_t),1,
tempfile,"temporary n-gram ids");
sort_ngram[j] = buffer[i].word[j];
}
rr_fwrite((char*)&sort_count,sizeof(int),1,tempfile,
"temporary n-gram counts");
sort_count = buffer[i].count;
}
}
for (j=0;j<=n-1;j++)
rr_fwrite((char*)&sort_ngram[j],sizeof(wordid_t),1,
tempfile,"temporary n-gram ids");
rr_fwrite((char*)&sort_count,sizeof(int),1,tempfile,
"temporary n-gram counts");
rr_oclose(tempfile);
position_in_buffer = 1;
}
for (i=0;i<=n-1;i++)
buffer[position_in_buffer-1].word[i] = current_ngram[i];
buffer[position_in_buffer-1].count = current_count;
}else {
/* Write to temporary file */
for (i=0;i<=n-1;i++)
rr_fwrite((char*)¤t_ngram[i],sizeof(wordid_t),1,
non_unk_fp,"temporary n-gram ids");
rr_fwrite((char*)¤t_count,sizeof(int),1,non_unk_fp,
"temporary n-gram counts");
}
}
}
if (position_in_buffer > 0) {
/* Only do this bit if we have actually seen some OOVs */
/* Sort final buffer */
pc_message(verbosity,2,"Sorting final buffer...\n");
qsort((void*) buffer,(size_t) position_in_buffer,
sizeof(ngram_rec),compare_ngrams2);
/* Write final buffer */
number_of_tempfiles++;
sprintf(temp_word,"%s/%hu%s", temp_directory,
number_of_tempfiles,temp_file_ext);
pc_message(verbosity,2,"Writing sorted buffer to temporary file %s\n", temp_word);
tempfile = rr_fopen(temp_word,"w");
for (i=0;i<=n-1;i++)
sort_ngram[i] = buffer[0].word[i];
sort_count = buffer[0].count;
for (i=1;i<=position_in_buffer-1;i++) {
same_ngram = 1;
for (j=n-1;j>=0;j--) {
if (buffer[i].word[j] != sort_ngram[j]) {
same_ngram = 0;
j = -1;
}
}
if (same_ngram)
sort_count += buffer[i].count;
else {
for (j=0;j<=n-1;j++) {
rr_fwrite((char*)&sort_ngram[j],sizeof(wordid_t),1,
tempfile,"temporary n-gram ids");
sort_ngram[j] = buffer[i].word[j];
}
rr_fwrite((char*)&sort_count,sizeof(int),1,tempfile,
"temporary n-gram counts");
sort_count = buffer[i].count;
}
}
for (j=0;j<=n-1;j++)
rr_fwrite((char*)&sort_ngram[j],sizeof(wordid_t),1,
tempfile,"temporary n-gram ids");
rr_fwrite((char*)&sort_count,sizeof(int),1,tempfile,
"temporary n-gram counts");
fclose(tempfile);
}
/* Merge the temporary files, and output the result */
fclose(non_unk_fp);
pc_message(verbosity,2,"Merging temporary files...\n");
merge_idngramfiles(1,
number_of_tempfiles,
temp_directory,
temp_file_ext,
max_files,
outfile,
write_ascii,
fof_size,
n);
fclose(outfile);
rmdir(temp_directory);
pc_message(verbosity,0,"wngram2idngram : Done.\n");
return 0;
}
void main (int argc, char **argv) {
ng_t ng;
arpa_lm_t arpa_ng;
char input_string[500];
int num_of_args;
char *args[MAX_ARGS];
char *lm_filename_arpa;
char *lm_filename_binary;
flag told_to_quit;
flag inconsistant_parameters;
flag backoff_from_unk_inc;
flag backoff_from_unk_exc;
flag backoff_from_ccs_inc;
flag backoff_from_ccs_exc;
flag arpa_lm;
flag binary_lm;
flag include_unks;
char *fb_list_filename;
char *probs_stream_filename;
char *annotation_filename;
char *text_stream_filename;
char *oov_filename;
char *ccs_filename;
double log_base;
char wlist_entry[1024];
char current_cc[200];
int current_cc_id;
FILE *context_cues_fp;
int n;
int generate_size = 10000;
int random_seed;
/* Process command line */
report_version(&argc,argv);
if (pc_flagarg(&argc, argv,"-help") ||
argc == 1 ||
(strcmp(argv[1],"-binary") && strcmp(argv[1],"-arpa"))) {
fprintf(stderr,"evallm : Evaluate a language model.\n");
fprintf(stderr,"Usage : evallm [ -binary .binlm | \n");
fprintf(stderr," -arpa .arpa [ -context .ccs ] ]\n");
exit(1);
}
lm_filename_arpa = salloc(pc_stringarg(&argc, argv,"-arpa",""));
if (strcmp(lm_filename_arpa,"")) {
arpa_lm = 1;
}
else {
arpa_lm = 0;
}
lm_filename_binary = salloc(pc_stringarg(&argc, argv,"-binary",""));
if (strcmp(lm_filename_binary,"")) {
binary_lm = 1;
}
else {
binary_lm = 0;
}
if (arpa_lm && binary_lm) {
quit(-1,"Error : Can't use both -arpa and -binary flags.\n");
}
if (!arpa_lm && !binary_lm) {
quit(-1,"Error : Must specify either a binary or an arpa format language model.\n");
}
ccs_filename = salloc(pc_stringarg(&argc, argv,"-context",""));
if (binary_lm && strcmp(ccs_filename,"")) {
fprintf(stderr,"Warning - context cues file not needed with binary language model file.\nWill ignore it.\n");
}
pc_report_unk_args(&argc,argv,2);
/* Load language model */
if (arpa_lm) {
fprintf(stderr,"Reading in language model from file %s\n",
lm_filename_arpa);
load_arpa_lm(&arpa_ng,lm_filename_arpa);
}
else {
fprintf(stderr,"Reading in language model from file %s\n",
lm_filename_binary);
load_lm(&ng,lm_filename_binary);
}
fprintf(stderr,"\nDone.\n");
if (!arpa_lm) {
n=ng.n;
}
else {
n=arpa_ng.n;
}
if (arpa_lm) {
arpa_ng.context_cue =
(flag *) rr_calloc(arpa_ng.table_sizes[0],sizeof(flag));
arpa_ng.no_of_ccs = 0;
if (strcmp(ccs_filename,"")) {
context_cues_fp = rr_iopen(ccs_filename);
while (fgets (wlist_entry, sizeof (wlist_entry),context_cues_fp)) {
if (strncmp(wlist_entry,"##",2)==0) continue;
sscanf (wlist_entry, "%s ",current_cc);
if (strncmp(wlist_entry,"#",1)==0) {
fprintf(stderr,"\n\n===========================================================\n");
fprintf(stderr,":\nWARNING: line assumed NOT a comment:\n");
fprintf(stderr, ">>> %s <<<\n",wlist_entry);
fprintf(stderr, " '%s' will be included in the context cues list\n",current_cc);
fprintf(stderr, " (comments must start with '##')\n");
fprintf(stderr,"===========================================================\n\n");
}
if (sih_lookup(arpa_ng.vocab_ht,current_cc,¤t_cc_id) == 0) {
quit(-1,"Error : %s in the context cues file does not appear in the vocabulary.\n",current_cc);
}
arpa_ng.context_cue[(unsigned short) current_cc_id] = 1;
arpa_ng.no_of_ccs++;
fprintf(stderr,"Context cue word : %s id = %d\n",current_cc,current_cc_id);
}
rr_iclose(context_cues_fp);
}
}
/* Process commands */
told_to_quit = 0;
num_of_args = 0;
while (!feof(stdin) && !told_to_quit) {
printf("evallm : ");
gets(input_string);
if (!feof(stdin)) {
parse_comline(input_string,&num_of_args,args);
random_seed = pc_intarg(&num_of_args,args,"-seed",-1);
generate_size = pc_intarg(&num_of_args,args,"-size",10000);
log_base = pc_doublearg(&num_of_args,args,"-log_base",10.0);
backoff_from_unk_inc = pc_flagarg(&num_of_args,args,
"-backoff_from_unk_inc");
backoff_from_ccs_inc = pc_flagarg(&num_of_args,args,
"-backoff_from_ccs_inc");
backoff_from_unk_exc = pc_flagarg(&num_of_args,args,
"-backoff_from_unk_exc");
backoff_from_ccs_exc = pc_flagarg(&num_of_args,args,
"-backoff_from_ccs_exc");
include_unks = pc_flagarg(&num_of_args,args,"-include_unks");
fb_list_filename = salloc(pc_stringarg(&num_of_args,args,
"-backoff_from_list",""));
text_stream_filename =
salloc(pc_stringarg(&num_of_args,args,"-text",""));
probs_stream_filename =
salloc(pc_stringarg(&num_of_args,args,"-probs",""));
annotation_filename =
salloc(pc_stringarg(&num_of_args,args,"-annotate",""));
oov_filename = salloc(pc_stringarg(&num_of_args,args,"-oovs",""));
inconsistant_parameters = 0;
if (backoff_from_unk_inc && backoff_from_unk_exc) {
fprintf(stderr,"Error : Cannot specify both exclusive and inclusive forced backoff.\n");
fprintf(stderr,"Use only one of -backoff_from_unk_exc and -backoff_from_unk_inc\n");
inconsistant_parameters = 1;
}
if (backoff_from_ccs_inc && backoff_from_ccs_exc) {
fprintf(stderr,"Error : Cannot specify both exclusive and inclusive forced backoff.\n");
fprintf(stderr,"Use only one of -backoff_from_ccs_exc and -backoff_from_ccs_inc\n");
inconsistant_parameters = 1;
}
if (num_of_args > 0) {
if (!inconsistant_parameters) {
if (!strcmp(args[0],"perplexity")) {
compute_perplexity(&ng,
&arpa_ng,
text_stream_filename,
probs_stream_filename,
annotation_filename,
oov_filename,
fb_list_filename,
backoff_from_unk_inc,
backoff_from_unk_exc,
backoff_from_ccs_inc,
backoff_from_ccs_exc,
arpa_lm,
include_unks,
log_base);
}
else {
if (!strcmp(args[0],"validate")) {
if (num_of_args != n) {
fprintf(stderr,"Error : must specify %d words of context.\n",
n-1);
}
else {
/* Assume last n-1 parameters form context */
validate(&ng,
&arpa_ng,
&(args[num_of_args-n+1]),
backoff_from_unk_inc,
backoff_from_unk_exc,
backoff_from_ccs_inc,
backoff_from_ccs_exc,
arpa_lm,
fb_list_filename);
}
}
else {
if (!strcmp(args[0],"stats")) {
if (arpa_lm) {
display_arpa_stats(&arpa_ng);
}
else {
display_stats(&ng);
}
}
else {
if (!strcmp(args[0],"quit")) {
told_to_quit=1;
}
else if (!strcmp(args[0],"generate")) {
if(arpa_lm)
generate_words(NULL,&arpa_ng,generate_size,random_seed,text_stream_filename);
else
generate_words(&ng,NULL,generate_size,random_seed,text_stream_filename);
}
else {
if (!strcmp(args[0],"help")) {
printf("The user may specify one of the following commands: \n");
printf("\n");
printf(" - perplexity\n");
printf("\n");
printf("Computes the perplexity of a given text. May optionally specify words\n");
printf("from which to force back-off.\n");
printf("\n");
printf("Syntax: \n");
printf("\n");
printf("perplexity -text .text\n");
printf(" [ -probs .fprobs ]\n");
printf(" [ -oovs .oov_file ]\n");
printf(" [ -annotate .annotation_file ] \n");
printf(" [ -backoff_from_unk_inc | -backoff_from_unk_exc ]\n");
printf(" [ -backoff_from_ccs_inc | -backoff_from_ccs_exc ] \n");
printf(" [ -backoff_from_list .fblist ]\n");
printf(" [ -include_unks ]\n");
printf("\n");
printf(" - validate\n");
printf(" \n");
printf("Calculate the sum of the probabilities of all the words in the\n");
printf("vocabulary given the context specified by the user.\n");
printf("\n");
printf("Syntax: \n");
printf("\n");
printf("validate [ -backoff_from_unk -backoff_from_ccs |\n");
printf(" -backoff_from_list .fblist ]\n");
printf(" [ -forced_backoff_inc | -forced_back_off_exc ] \n");
printf(" word1 word2 ... word_(n-1)\n");
printf("\n");
printf("Where n is the n in n-gram. \n");
printf("\n");
printf(" - help\n");
printf("\n");
printf("Displays this help message.\n");
printf("\n");
printf("Syntax: \n");
printf("\n");
printf("help\n");
printf("\n");
printf(" - quit\n");
printf("\n");
printf("Exits the program.\n");
printf("\n");
printf("Syntax: \n");
printf("\n");
printf("quit\n");
}
else {
fprintf(stderr,"Unknown command : %s\nType \'help\'\n",
args[0]);
}
}
}
}
}
}
}
}
}
fprintf(stderr,"evallm : Done.\n");
exit(0);
}
ng_t * init_ng(
int* argc,
char **argv,
int verbosity
)
{
int i;
ng_t* ng;
ng=(ng_t*) rr_calloc(1,sizeof(ng_t));
ng->disc_meth=NULL;
/* -n */
ng->n = pc_intarg(argc, argv,"-n",DEFAULT_N);
if (ng->n<1)
quit(-1,"Error: Value of n must be larger than zero.\n");
/* -cutoffs */
ng->cutoffs = (cutoff_t *) pc_shortarrayarg(argc, argv, "-cutoffs",ng->n-1,ng->n-1);
if (ng->cutoffs == NULL)
ng->cutoffs = (cutoff_t *) rr_calloc((ng->n-1)+1,sizeof(cutoff_t)); /* +1 for the sake of the correction in writing in write_lms.c */
for (i=0;i<=ng->n-3;i++) {
if (ng->cutoffs[i+1] < ng->cutoffs[i]) {
quit(-1,"Error - cutoffs for (n+1)-gram must be greater than or equal to those for \nn-gram. You have %d-gram cutoff = %d > %d-gram cutoff = %d.\n",i+2,ng->cutoffs[i],i+3,ng->cutoffs[i+1]);
}
}
/* -min_unicount */
ng->min_unicount = pc_intarg(argc, argv, "-min_unicount",0);
/* -idngram */
ng->id_gram_filename = salloc(pc_stringarg(argc, argv,"-idngram",""));
if (!strcmp(ng->id_gram_filename,""))
quit(-1,"Error: id ngram file not specified. Use the -idngram flag.\n");
/* -arpa & -bin */
ng->arpa_filename = salloc(pc_stringarg(argc, argv,"-arpa",""));
ng->bin_filename = salloc(pc_stringarg(argc, argv,"-binary",""));
ng->write_arpa = strcmp("",ng->arpa_filename);
ng->write_bin = strcmp("",ng->bin_filename);
if (!(ng->write_arpa || ng->write_bin))
quit(-1,"Error : must specify either an arpa, or a binary output file.\n");
ng->count_table_size = DEFAULT_COUNT_TABLE_SIZE;
/* -vocab */
ng->vocab_filename = salloc(pc_stringarg(argc,argv,"-vocab",""));
if (!strcmp("",ng->vocab_filename))
quit(-1,"Error : vocabulary file not specified. Use the -vocab option.\n");
/* -context */
ng->context_cues_filename = salloc(pc_stringarg(argc,argv,"-context",""));
ng->context_set = strcmp("", ng->context_cues_filename);
/* -vocab_type */
ng->vocab_type = pc_intarg(argc,argv,"-vocab_type",1);
/* -oov_fraction */
ng->oov_fraction = pc_doublearg(argc, argv,"-oov_fraction",-1.0);
if (ng->oov_fraction == -1.0)
ng->oov_fraction=DEFAULT_OOV_FRACTION;
else {
if (ng->vocab_type != 2)
pc_message(verbosity,1,"Warning : OOV fraction specified, but will not be used, since vocab type is not 2.\n");
}
if (ng->vocab_type == 0)
ng->first_id = 1;
else
ng->first_id = 0;
/* Allow both "min_alpha" etc and "min_bo_weight" etc as valid
syntax. The "bo_weight" form is preferred, but the "alpha" form is
maintained as it was present in version 2.00 */
ng->min_alpha = pc_doublearg(argc,argv,"-min_alpha",DEFAULT_MIN_ALPHA);
ng->max_alpha = pc_doublearg(argc,argv,"-max_alpha",DEFAULT_MAX_ALPHA);
ng->out_of_range_alphas = pc_intarg(argc,argv,"-out_of_range_alphas",
DEFAULT_OUT_OF_RANGE_ALPHAS);
ng->min_alpha = pc_doublearg(argc,argv,"-min_bo_weight",ng->min_alpha);
ng->max_alpha = pc_doublearg(argc,argv,"-max_bo_weight",ng->max_alpha);
ng->out_of_range_alphas = pc_intarg(argc,argv,"-out_of_range_bo_weights",
ng->out_of_range_alphas);
if (ng->min_alpha >= ng->max_alpha)
quit(-1,"Error : Minimum of alpha range must be less than the maximum.\n");
init_ng_disc_method(ng,
pc_flagarg(argc, argv,"-linear"),
pc_flagarg(argc,argv,"-absolute"),
pc_flagarg(argc,argv,"-witten_bell"),
pc_flagarg(argc,argv,"-good_turing"));
ng->disc_range = (unsigned short *) pc_shortarrayarg(argc, argv, "-disc_ranges",ng->n,ng->n);
ng->disc_range_set = (ng->disc_range != NULL);
if (ng->discounting_method == GOOD_TURING) {
if (!ng->disc_range_set) {
ng->disc_range = (unsigned short *) rr_malloc(sizeof(unsigned short) * ng->n);
ng->disc_range[0] = DEFAULT_DISC_RANGE_1;
for (i=1;i<=ng->n-1;i++)
ng->disc_range[i] = DEFAULT_DISC_RANGE_REST;
}
ng->fof_size = (fof_sz_t *) rr_malloc(sizeof(fof_sz_t) * ng->n);
for (i=0;i<=ng->n-1;i++)
ng->fof_size[i] = ng->disc_range[i]+1;
}else {
if (ng->disc_range_set)
pc_message(verbosity,2,"Warning : discount ranges specified will be ignored, since they only apply\nto Good Turing discounting.\n");
}
ng->four_byte_alphas = !(pc_flagarg(argc, argv, "-two_byte_alphas") ||
pc_flagarg(argc, argv, "-two_byte_bo_weights"));
ng->four_byte_counts = pc_flagarg(argc, argv, "-four_byte_counts");
if(ng->four_byte_counts){
pc_message(verbosity,2,"Using Four byte counts.\n");
}
ng->zeroton_fraction = pc_doublearg(argc,argv,"-zeroton_fraction",1.0);
/* Attempt to open all the files that we will need for input and
output. It is better to do it here than to spend a few hours of
CPU processing id-gram counts, only to find that the output path
is invalid. */
ng->id_gram_fp = rr_iopen(ng->id_gram_filename);
/* Vocab is read by Roni's function which does the file opening for
us, so no need to do it here. Don't need to worry about time
being lost if file doesn't exist, since vocab is first thing to
be read anyway. */
if (ng->context_set)
ng->context_cues_fp = rr_iopen(ng->context_cues_filename);
if (ng->write_arpa)
ng->arpa_fp = rr_oopen(ng->arpa_filename);
if (ng->write_bin)
ng->bin_fp = rr_oopen(ng->bin_filename);
return ng;
}
