void main(int argc, char *argv[]) {
int i,j;
char *vocab_filename;
FILE *tempfile;
char tempfiles_directory[1000];
int vocab_size;
FILE *vocab_file;
int verbosity;
int buffer_size;
int position_in_buffer;
int number_of_tempfiles;
int max_files;
int fof_size;
unsigned short *buffer;
unsigned short *placeholder;
unsigned short *temp_ngram;
int temp_count;
char temp_word[500];
char temp_word2[500];
char *temp_file_root;
char *temp_file_ext;
char *host_name;
int proc_id;
struct utsname uname_info;
flag write_ascii;
/* Vocab hash table things */
struct hash_table vocabulary;
unsigned long hash_size;
unsigned long M;
tempfile = NULL; /* Just to prevent compilation warnings. */
report_version(&argc,argv);
verbosity = pc_intarg(&argc,argv,"-verbosity",DEFAULT_VERBOSITY);
/* Process command line */
if (pc_flagarg( &argc, argv,"-help") || argc==1) {
fprintf(stderr,"text2idngram - Convert a text stream to an id n-gram stream.\n");
fprintf(stderr,"Usage : text2idngram -vocab .vocab \n");
fprintf(stderr," [ -buffer 100 ]\n");
fprintf(stderr," [ -hash %d ]\n",DEFAULT_HASH_SIZE);
fprintf(stderr," [ -temp %s ]\n",DEFAULT_TEMP);
fprintf(stderr," [ -files %d ]\n",DEFAULT_MAX_FILES);
fprintf(stderr," [ -gzip | -compress ]\n");
fprintf(stderr," [ -verbosity %d ]\n",
DEFAULT_VERBOSITY);
fprintf(stderr," [ -n 3 ]\n");
fprintf(stderr," [ -write_ascii ]\n");
fprintf(stderr," [ -fof_size 10 ]\n");
exit(1);
}
pc_message(verbosity,2,"text2idngram\n");
n = pc_intarg( &argc, argv, "-n",DEFAULT_N);
placeholder = (unsigned short *) rr_malloc(sizeof(unsigned short)*n);
temp_ngram = (unsigned short *) rr_malloc(sizeof(unsigned short)*n);
hash_size = pc_intarg( &argc, argv, "-hash",DEFAULT_HASH_SIZE);
buffer_size = pc_intarg( &argc, argv, "-buffer",STD_MEM);
write_ascii = pc_flagarg(&argc,argv,"-write_ascii");
fof_size = pc_intarg(&argc,argv,"-fof_size",10);
max_files = pc_intarg( &argc, argv, "-files",DEFAULT_MAX_FILES);
vocab_filename = salloc(pc_stringarg( &argc, argv, "-vocab", "" ));
if (!strcmp("",vocab_filename)) {
quit(-1,"text2idngram : Error : Must specify a vocabulary file.\n");
}
strcpy(tempfiles_directory,pc_stringarg( &argc, argv, "-temp",
DEFAULT_TEMP));
if (pc_flagarg(&argc,argv,"-compress")) {
temp_file_ext = salloc(".Z");
}
else {
if (pc_flagarg(&argc,argv,"-gzip")) {
temp_file_ext = salloc(".gz");
}
else {
temp_file_ext = salloc("");
}
}
uname(&uname_info);
host_name = salloc(uname_info.nodename);
proc_id = getpid();
sprintf(temp_word,"%s%s.%d.",TEMP_FILE_ROOT,host_name,proc_id);
temp_file_root = salloc(temp_word);
pc_report_unk_args(&argc,argv,verbosity);
/* If the last charactor in the directory name isn't a / then add one. */
if (tempfiles_directory[strlen(tempfiles_directory)-1] != '/') {
strcat(tempfiles_directory,"/");
}
pc_message(verbosity,2,"Vocab : %s\n",vocab_filename);
pc_message(verbosity,2,"N-gram buffer size : %d\n",buffer_size);
pc_message(verbosity,2,"Hash table size : %d\n",hash_size);
pc_message(verbosity,2,"Temp directory : %s\n",tempfiles_directory);
pc_message(verbosity,2,"Max open files : %d\n",max_files);
pc_message(verbosity,2,"FOF size : %d\n",fof_size);
pc_message(verbosity,2,"n : %d\n",n);
buffer_size *= (1000000/(sizeof(unsigned short)*n));
/* Allocate memory for hash table */
fprintf(stderr,"Initialising hash table...\n");
M = nearest_prime(hash_size);
new_hashtable(&vocabulary,M);
/* Read in the vocabulary */
vocab_size = 0;
vocab_file = rr_iopen(vocab_filename);
pc_message(verbosity,2,"Reading vocabulary...\n");
while (fgets (temp_word, sizeof(temp_word),vocab_file)) {
if (strncmp(temp_word,"##",2)==0) continue;
sscanf (temp_word, "%s ",temp_word2);
/* Check for repeated words in the vocabulary */
if (index2(&vocabulary,temp_word2) != 0) {
fprintf(stderr,"======================================================\n");
fprintf(stderr,"WARNING: word %s is repeated in the vocabulary.\n",temp_word);
fprintf(stderr,"=======================================================\n");
}
if (strncmp(temp_word,"#",1)==0) {
fprintf(stderr,"\n\n===========================================================\n");
fprintf(stderr,":\nWARNING: line assumed NOT a comment:\n");
fprintf(stderr, ">>> %s <<<\n",temp_word);
fprintf(stderr, " '%s' will be included in the vocabulary.\n",temp_word2);
fprintf(stderr, " (comments must start with '##')\n");
fprintf(stderr,"===========================================================\n\n");
}
vocab_size++;
add_to_hashtable(&vocabulary,hash(temp_word2,M),temp_word2,vocab_size);
}
if (vocab_size > MAX_VOCAB_SIZE) {
quit(-1,"text2idngram : Error : Vocabulary size exceeds maximum.\n");
}
pc_message(verbosity,2,"Allocating memory for the n-gram buffer...\n");
buffer=(unsigned short*) rr_malloc(n*(buffer_size+1)*sizeof(unsigned short));
number_of_tempfiles = 0;
/* Read text into buffer */
/* Read in the first ngram */
position_in_buffer = 0;
for (i=0;i<=n-1;i++) {
get_word(stdin,temp_word);
add_to_buffer(index2(&vocabulary,temp_word),0,i,buffer);
}
while (!rr_feof(stdin)) {
/* Fill up the buffer */
pc_message(verbosity,2,"Reading text into the n-gram buffer...\n");
pc_message(verbosity,2,"20,000 n-grams processed for each \".\", 1,000,000 for each line.\n");
while ((position_in_buffer<buffer_size) && (!rr_feof(stdin))) {
position_in_buffer++;
if (position_in_buffer % 20000 == 0) {
if (position_in_buffer % 1000000 == 0) {
pc_message(verbosity,2,".\n");
}
else {
pc_message(verbosity,2,".");
}
}
for (i=1;i<=n-1;i++) {
add_to_buffer(buffer_contents(position_in_buffer-1,i,buffer),
position_in_buffer,i-1,buffer);
}
if (get_word(stdin,temp_word) == 1) {
add_to_buffer(index2(&vocabulary,temp_word),position_in_buffer,
n-1,buffer);
}
}
for (i=0;i<=n-1;i++) {
placeholder[i] = buffer_contents(position_in_buffer,i,buffer);
}
/* Sort buffer */
pc_message(verbosity,2,"\nSorting n-grams...\n");
qsort((void*) buffer,(size_t) position_in_buffer,
n*sizeof(unsigned short),compare_ngrams);
/* Output the buffer to temporary BINARY file */
number_of_tempfiles++;
sprintf(temp_word,"%s%s%hu%s",tempfiles_directory,temp_file_root,
number_of_tempfiles,temp_file_ext);
pc_message(verbosity,2,"Writing sorted n-grams to temporary file %s\n",
temp_word);
tempfile = rr_oopen(temp_word);
for (i=0;i<=n-1;i++) {
temp_ngram[i] = buffer_contents(0,i,buffer);
if (temp_ngram[i] > MAX_VOCAB_SIZE) {
quit(-1,"Invalid trigram in buffer.\nAborting");
}
}
temp_count = 1;
for (i=1;i<=position_in_buffer;i++) {
if (!compare_ngrams(temp_ngram,&buffer[i*n])) {
temp_count++;
}
else {
for (j=0;j<=n-1;j++) {
rr_fwrite(&temp_ngram[j],sizeof(unsigned short),1,
tempfile,"temporary n-gram ids");
temp_ngram[j] = buffer_contents(i,j,buffer);
}
rr_fwrite(&temp_count,sizeof(int),1,tempfile,
"temporary n-gram counts");
temp_count = 1;
}
}
rr_oclose(tempfile);
for (i=0;i<=n-1;i++) {
add_to_buffer(placeholder[i],0,i,buffer);
}
position_in_buffer = 0;
}
/* Merge the temporary files, and output the result to standard output */
pc_message(verbosity,2,"Merging temporary files...\n");
merge_tempfiles(1,
number_of_tempfiles,
temp_file_root,
temp_file_ext,
max_files,
tempfiles_directory,
stdout,
write_ascii,
fof_size);
pc_message(verbosity,0,"text2idngram : Done.\n");
exit(0);
}
int main (int argc, char **argv) {
int n;
int verbosity;
int max_files;
int max_words;
int max_chars;
int current_word;
int current_char;
int start_char; /* start boundary (possibly > than 0) */
int no_of_spaces;
int pos_in_string;
int i;
char *current_string;
char current_temp_filename[500];
int current_file_number;
FILE *temp_file;
flag text_buffer_full;
char *text_buffer;
char **pointers;
char current_ngram[500];
int current_count;
int counter;
char temp_directory[1000];
char *temp_file_ext;
flag words_set;
flag chars_set;
/* Process command line */
verbosity = pc_intarg(&argc, argv,"-verbosity",DEFAULT_VERBOSITY);
pc_message(verbosity,2,"text2wngram\n");
report_version(&argc,argv);
if (pc_flagarg( &argc, argv,"-help")) {
help_message();
exit(1);
}
n = pc_intarg(&argc, argv,"-n",DEFAULT_N);
/* max_words = pc_intarg(&argc, argv,"-words",STD_MEM*1000000/11);
max_chars = pc_intarg(&argc, argv,"-chars",STD_MEM*7000000/11); */
max_words = pc_intarg(&argc, argv,"-words",-1);
max_chars = pc_intarg(&argc, argv,"-chars",-1);
if (max_words == -1) {
words_set = 0;
max_words = STD_MEM*1000000/11;
}else
words_set = 1;
if (max_chars == -1) {
chars_set = 0;
max_chars = STD_MEM*7000000/11;
}else
chars_set = 1;
max_files = pc_intarg(&argc, argv,"-files",DEFAULT_MAX_FILES);
if (pc_flagarg(&argc,argv,"-compress"))
temp_file_ext = salloc(".Z");
else {
if (pc_flagarg(&argc,argv,"-gzip"))
temp_file_ext = salloc(".gz");
else
temp_file_ext = salloc("");
}
strcpy(temp_directory, "cmuclmtk-XXXXXX");
if (mkdtemp(temp_directory) == NULL) {
quit(-1, "Failed to create temporary folder: %s\n", strerror(errno));
}
pc_report_unk_args(&argc,argv,verbosity);
if (words_set && !chars_set)
max_chars = max_words * 7;
if (!words_set && chars_set)
max_words = max_chars / 7;
/* If the last charactor in the directory name isn't a / then add one. */
pc_message(verbosity,2,"n = %d\n",n);
pc_message(verbosity,2,"Number of words in buffer = %d\n",max_words);
pc_message(verbosity,2,"Number of chars in buffer = %d\n",max_chars);
pc_message(verbosity,2,"Max number of files open at once = %d\n",max_files);
pc_message(verbosity,2,"Temporary directory = %s\n",temp_directory);
/* Allocate memory for the buffers */
text_buffer = (char *) rr_malloc(sizeof(char)*max_chars);
pc_message(verbosity,2,"Allocated %d bytes to text buffer.\n",
sizeof(char)*max_chars);
pointers = (char **) rr_malloc(sizeof(char *)*max_words);
pc_message(verbosity,2,"Allocated %d bytes to pointer array.\n",
sizeof(char *)*max_words);
current_file_number = 0;
current_word = 1;
start_char = 0;
current_char = 0;
counter = 0;
pointers[0] = text_buffer;
while (!feof(stdin)) {
current_file_number++;
/* Read text into buffer */
pc_message(verbosity,2,"Reading text into buffer...\n");
pc_message(verbosity,2,"Reading text into the n-gram buffer...\n");
pc_message(verbosity,2,"20,000 words processed for each \".\", 1,000,000 for each line.\n");
pointers[0] = text_buffer;
while ((!rr_feof(stdin)) &&
(current_word < max_words) &&
(current_char < max_chars)) {
text_buffer[current_char] = getchar();
if (text_buffer[current_char] == '\n' ||
text_buffer[current_char] == '\t' ) {
text_buffer[current_char] = ' ';
}
if (text_buffer[current_char] == ' ') {
if (current_char > start_char) {
if (text_buffer[current_char-1] == ' ') {
current_word--;
current_char--;
}
pointers[current_word] = &(text_buffer[current_char+1]);
current_word++;
counter++;
if (counter % 20000 == 0) {
if (counter % 1000000 == 0)
pc_message(verbosity,2,"\n");
else
pc_message(verbosity,2,".");
}
}
}
if (text_buffer[current_char] != ' ' || current_char > start_char)
current_char++;
}
text_buffer[current_char]='\0';
if (current_word == max_words || rr_feof(stdin)) {
for (i=current_char+1;i<=max_chars-1;i++)
text_buffer[i] = ' ';
text_buffer_full = 0;
}else
text_buffer_full = 1;
/* Sort buffer */
pc_message(verbosity,2,"\nSorting pointer array...\n");
qsort((void *) pointers,(size_t) current_word-n,sizeof(char *),cmp_strings);
/* Write out temporary file */
sprintf(current_temp_filename,"%s/%hu%s",temp_directory, current_file_number, temp_file_ext);
pc_message(verbosity,2,"Writing out temporary file %s...\n",current_temp_filename);
temp_file = rr_oopen(current_temp_filename);
text_buffer[current_char] = ' ';
current_count = 0;
strcpy(current_ngram,"");
for (i = 0; i <= current_word-n; i++) {
current_string = pointers[i];
/* Find the nth space */
no_of_spaces = 0;
pos_in_string = 0;
while (no_of_spaces < n) {
if (current_string[pos_in_string] == ' ')
no_of_spaces++;
pos_in_string++;
}
if (!strncmp(current_string,current_ngram,pos_in_string))
current_count++;
else {
if (strcmp(current_ngram,""))
if (fprintf(temp_file,"%s %d\n",current_ngram,current_count) < 0)
quit(-1,"Error writing to temporary file %s\n",current_temp_filename);
current_count = 1;
strncpy(current_ngram,current_string,pos_in_string);
current_ngram[pos_in_string] = '\0';
}
}
rr_oclose(temp_file);
/* Move the last n-1 words to the beginning of the buffer, and set
correct current_word and current_char things */
strcpy(text_buffer,pointers[current_word-n]);
pointers[0]=text_buffer;
/* Find the (n-1)th space */
no_of_spaces=0;
pos_in_string=0;
if (!text_buffer_full){
while (no_of_spaces<(n-1)) {
if (pointers[0][pos_in_string]==' ') {
no_of_spaces++;
pointers[no_of_spaces] = &pointers[0][pos_in_string+1];
}
pos_in_string++;
}
}else {
while (no_of_spaces<n) {
if (pointers[0][pos_in_string]==' ') {
no_of_spaces++;
pointers[no_of_spaces] = &pointers[0][pos_in_string+1];
}
pos_in_string++;
}
pos_in_string--;
}
current_char = pos_in_string;
current_word = n;
/* mark boundary beyond which counting pass cannot backup */
start_char = current_char;
}
/* Merge temporary files */
pc_message(verbosity,2,"Merging temporary files...\n");
merge_tempfiles(1,
current_file_number,
temp_directory,
temp_file_ext,
max_files,
stdout,
n,
verbosity);
rmdir(temp_directory);
pc_message(verbosity,0,"text2wngram : Done.\n");
return 0;
}
void merge_tempfiles (int start_file,
int end_file,
char *temp_file_root,
char *temp_file_ext,
int max_files,
FILE *outfile,
int n,
int verbosity) {
FILE *new_temp_file;
char *new_temp_filename;
FILE **temp_file;
char **temp_filename;
char **current_ngram;
char smallest_ngram[1000];
int *current_ngram_count;
flag *finished;
flag all_finished;
int temp_count;
char temp_word[500];
int i,j;
pc_message(verbosity,2,"Merging temp files %d through %d...\n", start_file,
end_file);
/*
* If we try to do more than max_files, then merge into groups,
* then merge groups recursively.
*/
if (end_file-start_file+1 > max_files) {
int new_start_file, new_end_file;
int n_file_groups = 1 + (end_file-start_file)/max_files;
fprintf(stderr, "%d files to do, in %d groups\n", end_file-start_file,
n_file_groups);
new_temp_filename = (char *) rr_malloc(300*sizeof(char));
/*
* These n_file_groups sets of files will be done in groups of
* max_files batches each, as temp files numbered
* end_file+1 ... end_file+n_file_groups,
* and then these will be merged into the final result.
*/
for (i = 0; i < n_file_groups; i++) {
/* do files i*max_files through min((i+1)*max_files-1,end_file); */
new_start_file = start_file + (i*max_files);
new_end_file = start_file + ((i+1)*max_files) - 1;
if (new_end_file > end_file) new_end_file = end_file;
sprintf(new_temp_filename,
"%s/%hu%s",
temp_file_root,
end_file+i+1,
temp_file_ext);
new_temp_file = rr_oopen(new_temp_filename);
merge_tempfiles(new_start_file,
new_end_file,
temp_file_root,
temp_file_ext,
max_files,
new_temp_file,
n,
verbosity);
rr_iclose(new_temp_file);
}
merge_tempfiles(end_file+1,
end_file+n_file_groups,
temp_file_root,
temp_file_ext,
max_files,
outfile,
n,
verbosity);
return;
}
/*
* We know we are now doing <= max_files.
*/
temp_file = (FILE **) rr_malloc((end_file+1)*sizeof(FILE *));
temp_filename = (char **) rr_malloc((end_file+1)*sizeof(char *));
for (i=start_file;i<=end_file;i++) {
temp_filename[i] = (char *) rr_malloc(300*sizeof(char));
}
current_ngram = (char **) rr_malloc((end_file+1)*sizeof(char *));
for (i=start_file;i<=end_file;i++) {
current_ngram[i] = (char *) rr_malloc(1000*sizeof(char));
}
current_ngram_count = (int *) rr_malloc((end_file+1)*sizeof(int));
finished = (flag *) rr_malloc(sizeof(flag)*(end_file+1));
/* Open all the temp files for reading */
for (i=start_file;i<=end_file;i++) {
sprintf(temp_filename[i],"%s/%hu%s",
temp_file_root,i,temp_file_ext);
temp_file[i] = rr_iopen(temp_filename[i]);
}
/* Now go through the files simultaneously, and write out the appropriate
ngram counts to the output file. */
for (i=start_file;i<=end_file;i++) {
finished[i] = 0;
if (!rr_feof(temp_file[i])) {
for (j=0;j<=n-1;j++) {
if (fscanf(temp_file[i],"%s",temp_word) != 1) {
if (!rr_feof(temp_file[i]))
quit(-1,"Error reading temp file %s\n",temp_filename[i]);
}else {
if (j==0)
strcpy(current_ngram[i],temp_word);
else {
strcat(current_ngram[i]," ");
strcat(current_ngram[i],temp_word);
}
}
}
if (fscanf(temp_file[i],"%d",¤t_ngram_count[i]) != 1) {
if (!rr_feof(temp_file[i]))
quit(-1,"Error reading temp file %s\n",temp_filename[i]);
}
}
}
all_finished = 0;
while (!all_finished) {
/* Find the smallest current ngram */
strcpy(smallest_ngram,"");
for (i=start_file;i<=end_file;i++) {
if (!finished[i]) {
if (strcmp(smallest_ngram,current_ngram[i]) > 0 ||
(smallest_ngram[0] == '\0'))
strcpy(smallest_ngram,current_ngram[i]);
}
}
/* For each of the files that are currently holding this ngram,
add its count to the temporary count, and read in a new ngram
from the files. */
temp_count = 0;
for (i=start_file;i<=end_file;i++) {
if (!finished[i]) {
if (!strcmp(smallest_ngram,current_ngram[i])) {
temp_count += current_ngram_count[i];
if (!rr_feof(temp_file[i])) {
for (j=0;j<=n-1;j++) {
if (fscanf(temp_file[i],"%s",temp_word) != 1) {
if (!rr_feof(temp_file[i])) {
quit(-1,"Error reading temp file %s\n",temp_filename[i]);
}
}else {
if (j==0)
strcpy(current_ngram[i],temp_word);
else {
strcat(current_ngram[i]," ");
strcat(current_ngram[i],temp_word);
}
}
}
if (fscanf(temp_file[i],"%d",¤t_ngram_count[i]) != 1) {
if (!rr_feof(temp_file[i])) {
quit(-1,"Error reading temp file count %s\n",
temp_filename[i]);
}
}
}
/*
* PWP: Note that the fscanf may have changed the state of
* temp_file[i], so we re-ask the question rather than just
* doing an "else".
*/
if (rr_feof(temp_file[i])) {
finished[i] = 1;
all_finished = 1;
for (j=start_file;j<=end_file;j++) {
if (!finished[j]) {
all_finished = 0;
}
}
}
}
}
}
/*
* PWP: We cannot conditionalize this on (!all_finished) because
* if we do we may have lost the very last count. (Consider the
* case when several files have ran out of data, but the last
* couple have the last count in them.)
*/
if (fprintf(outfile,"%s %d\n",smallest_ngram,temp_count) < 0) {
quit(-1,"Write error encountered while attempting to merge temporary files.\nAborting, but keeping temporary files.\n");
}
}
for (i=start_file;i<=end_file;i++) {
rr_iclose(temp_file[i]);
remove(temp_filename[i]);
}
free(temp_file);
for (i=start_file;i<=end_file;i++) {
free(temp_filename[i]);
}
free(temp_filename);
for (i=start_file;i<=end_file;i++) {
free(current_ngram[i]);
}
free(current_ngram);
free(current_ngram_count);
free(finished);
}
void merge_idngramfiles (int start_file,
int end_file,
char *temp_file_root,
char *temp_file_ext,
int max_files,
FILE *outfile,
flag write_ascii,
int fof_size,
int n_order) {
FILE *new_temp_file;
char temp_string[1000];
char *new_temp_filename;
FILE **temp_file;
char **temp_filename;
wordid_t **current_ngram;
wordid_t *smallest_ngram;
wordid_t *previous_ngram;
int *current_ngram_count;
flag *finished;
flag all_finished;
int temp_count;
int i,j;
flag first_ngram;
fof_t **fof_array;
ngram_sz_t *num_kgrams;
int *ng_count;
int pos_of_novelty;
n = n_order;
pos_of_novelty = n; /* Simply for warning-free compilation */
num_kgrams = (ngram_sz_t *) rr_calloc(n-1,sizeof(ngram_sz_t));
ng_count = (int *) rr_calloc(n-1,sizeof(int));
first_ngram = 1;
previous_ngram = (wordid_t *) rr_calloc(n,sizeof(wordid_t));
temp_file = (FILE **) rr_malloc(sizeof(FILE *) * (end_file-start_file+1));
temp_filename = (char **) rr_malloc(sizeof(char *) *
(end_file-start_file+1));
/* should change to 2d array*/
current_ngram = (wordid_t **) rr_malloc(sizeof(wordid_t *) *
(end_file-start_file+1));
for (i=0;i<=end_file-start_file;i++)
current_ngram[i] = (wordid_t *) rr_malloc(sizeof(wordid_t)*n);
current_ngram_count = (int *) rr_malloc(sizeof(int)*(end_file-start_file+1));
finished = (flag *) rr_malloc(sizeof(flag)*(end_file-start_file+1));
smallest_ngram = (wordid_t *) rr_malloc(sizeof(wordid_t)*n);
/* should change to 2d array*/
fof_array = (fof_t **) rr_malloc(sizeof(fof_t *)*(n-1));
for (i=0;i<=n-2;i++)
fof_array[i] = (fof_t *) rr_calloc(fof_size+1,sizeof(fof_t));
if (end_file-start_file+1 > max_files) {
sprintf(temp_string,"%s/%hu%s",temp_file_root,
end_file+1,temp_file_ext);
new_temp_filename = salloc(temp_string);
new_temp_file = rr_oopen(new_temp_filename);
merge_tempfiles(start_file,start_file+max_files-1,
temp_file_root,temp_file_ext,max_files,
new_temp_file,write_ascii,0);
merge_tempfiles(start_file+max_files,end_file+1,
temp_file_root,temp_file_ext,max_files,
outfile,write_ascii,0);
}else {
/* Open all the temp files for reading */
for (i=0;i<=end_file-start_file;i++) {
sprintf(temp_string,"%s/%hu%s",temp_file_root,
i+start_file,temp_file_ext);
temp_filename[i] = salloc(temp_string);
temp_file[i] = rr_iopen(temp_filename[i]);
}
/* Now go through the files simultaneously, and write out the appropriate
ngram counts to the output file. */
for (i=end_file-start_file;i>=0;i--) {
finished[i] = 0;
if (!rr_feof(temp_file[i])) {
for (j=0;j<=n-1;j++) {
rr_fread((char*) ¤t_ngram[i][j], sizeof(wordid_t),1,
temp_file[i],"temporary n-gram ids",0);
}
rr_fread((char*) ¤t_ngram_count[i], sizeof(int),1,
temp_file[i],"temporary n-gram counts",0);
}
}
all_finished = 0;
while (!all_finished) {
/* Find the smallest current ngram */
for (i=0;i<=n-1;i++)
smallest_ngram[i] = MAX_WORDID;
for (i=0;i<=end_file-start_file;i++) {
if (!finished[i]) {
if (compare_ngrams3(smallest_ngram,current_ngram[i]) < 0) {
for (j=0;j<n;j++)
smallest_ngram[j] = current_ngram[i][j];
}
}
}
#if MAX_VOCAB_SIZE < 65535
/* This check is well-meaning but completely useless since
smallest_ngram[i] by definition cannot contain any value
greater than MAX_VOCAB_SIZE (dhuggins@cs, 2006-03) */
for (i=0;i<=n-1;i++) {
if (smallest_ngram[i] > MAX_VOCAB_SIZE) {
quit(-1,"Error : Temporary files corrupted, invalid n-gram found.\n");
}
}
#endif
/* For each of the files that are currently holding this ngram,
add its count to the temporary count, and read in a new ngram
from the files. */
temp_count = 0;
for (i=0;i<=end_file-start_file;i++) {
if (!finished[i]) {
if (compare_ngrams3(smallest_ngram,current_ngram[i]) == 0) {
temp_count = temp_count + current_ngram_count[i];
if (!rr_feof(temp_file[i])) {
for (j=0;j<=n-1;j++) {
rr_fread((char*) ¤t_ngram[i][j],sizeof(wordid_t),1,
temp_file[i],"temporary n-gram ids",0);
}
rr_fread((char*)¤t_ngram_count[i],sizeof(int),1,
temp_file[i],"temporary n-gram count",0);
}else {
finished[i] = 1;
all_finished = 1;
for (j=0;j<=end_file-start_file;j++) {
if (!finished[j])
all_finished = 0;
}
}
}
}
}
if (write_ascii) {
for (i=0;i<=n-1;i++) {
if (fprintf(outfile,"%d ",smallest_ngram[i]) < 0)
{
quit(-1,"Write error encountered while attempting to merge temporary files.\nAborting, but keeping temporary files.\n");
}
}
if (fprintf(outfile,"%d\n",temp_count) < 0)
quit(-1,"Write error encountered while attempting to merge temporary files.\nAborting, but keeping temporary files.\n");
}else {
for (i=0;i<=n-1;i++) {
rr_fwrite((char*)&smallest_ngram[i],sizeof(wordid_t),1,
outfile,"n-gram ids");
}
rr_fwrite((char*)&temp_count,sizeof(count_t),1,outfile,"n-gram counts");
}
if (fof_size > 0 && n>1) { /* Add stuff to fof arrays */
/* Code from idngram2stats */
pos_of_novelty = n;
for (i=0;i<=n-1;i++) {
if (smallest_ngram[i] > previous_ngram[i]) {
pos_of_novelty = i;
i=n;
}
}
/* Add new N-gram */
num_kgrams[n-2]++;
if (temp_count <= fof_size)
fof_array[n-2][temp_count]++;
if (!first_ngram) {
for (i=n-2;i>=MAX(1,pos_of_novelty);i--) {
num_kgrams[i-1]++;
if (ng_count[i-1] <= fof_size) {
fof_array[i-1][ng_count[i-1]]++;
}
ng_count[i-1] = temp_count;
}
}else {
for (i=n-2;i>=MAX(1,pos_of_novelty);i--) {
ng_count[i-1] = temp_count;
}
first_ngram = 0;
}
for (i=0;i<=pos_of_novelty-2;i++)
ng_count[i] += temp_count;
for (i=0;i<=n-1;i++)
previous_ngram[i]=smallest_ngram[i];
}
}
for (i=0;i<=end_file-start_file;i++) {
fclose(temp_file[i]);
remove(temp_filename[i]);
}
}
if (fof_size > 0 && n>1) { /* Display fof arrays */
/* Process last ngram */
for (i=n-2;i>=MAX(1,pos_of_novelty);i--) {
num_kgrams[i-1]++;
if (ng_count[i-1] <= fof_size)
fof_array[i-1][ng_count[i-1]]++;
ng_count[i-1] = temp_count;
}
for (i=0;i<=pos_of_novelty-2;i++)
ng_count[i] += temp_count;
display_fof_array(num_kgrams,fof_array,fof_size,stderr, n);
}
}
int main(int argc, char *argv[]) {
int verbosity;
int vocab_size;
FILE *vocab_file;
int buffer_size;
flag write_ascii;
int max_files;
int number_of_tempfiles;
char *vocab_filename;
char *idngram_filename;
char tempfiles_directory[1000];
char temp_word[MAX_WORD_LENGTH];
char temp_word2[MAX_WORD_LENGTH];
char temp_word3[MAX_WORD_LENGTH];
flag contains_unks;
int position_in_buffer;
FILE *outfile;
FILE *tempfile;
FILE *non_unk_fp;
ngram_rec *buffer;
flag same_ngram;
int i;
int j;
int fof_size;
int size_of_rec;
char *temp_file_root;
char *temp_file_ext;
/* Vocab hash table things */
struct idngram_hash_table vocabulary;
unsigned long hash_size;
unsigned long M;
wordid_t *current_ngram;
int current_count;
wordid_t *sort_ngram;
int sort_count;
/* Process command line */
report_version(&argc,argv);
if (argc == 1 || pc_flagarg(&argc, argv,"-help")) {
/* Display help message */
help_message();
exit(1);
}
n = pc_intarg( &argc, argv, "-n",DEFAULT_N);
hash_size = pc_intarg( &argc, argv, "-hash",DEFAULT_HASH_SIZE);
buffer_size = pc_intarg( &argc, argv, "-buffer",STD_MEM);
write_ascii = pc_flagarg(&argc,argv,"-write_ascii");
verbosity = pc_intarg(&argc,argv,"-verbosity",DEFAULT_VERBOSITY);
max_files = pc_intarg( &argc, argv, "-files",DEFAULT_MAX_FILES);
fof_size = pc_intarg(&argc,argv,"-fof_size",10);
vocab_filename = salloc(pc_stringarg( &argc, argv, "-vocab", "" ));
idngram_filename = salloc(pc_stringarg( &argc, argv, "-idngram", "" ));
if (!strcmp("",vocab_filename))
quit(-1,"Error : Must specify a vocabulary file.\n");
if (!strcmp("",idngram_filename))
quit(-1,"text2idngram : Error : Must specify idngram file.\n");
strcpy(tempfiles_directory,pc_stringarg( &argc, argv, "-temp",DEFAULT_TEMP));
if (pc_flagarg(&argc,argv,"-compress"))
temp_file_ext = salloc(".Z");
else {
if (pc_flagarg(&argc,argv,"-gzip"))
temp_file_ext = salloc(".gz");
else
temp_file_ext = salloc("");
}
temp_file_root = tempnam(tempfiles_directory,TEMP_FILE_ROOT);
pc_report_unk_args(&argc,argv,verbosity);
outfile = rr_fopen(idngram_filename,"wb");
pc_message(verbosity,2,"Vocab : %s\n",vocab_filename);
pc_message(verbosity,2,"Output idngram : %s\n",idngram_filename);
pc_message(verbosity,2,"Buffer size : %d\n",buffer_size);
pc_message(verbosity,2,"Hash table size : %d\n",hash_size);
pc_message(verbosity,2,"Temp directory : %s\n",tempfiles_directory);
pc_message(verbosity,2,"Max open files : %d\n",max_files);
pc_message(verbosity,2,"n : %d\n",n);
pc_message(verbosity,2,"FOF size : %d\n",fof_size);
size_of_rec = (sizeof(wordid_t) * n) + 16 - (( n* sizeof(wordid_t)) % 16);
buffer_size *= (1000000/((sizeof(ngram_rec) + size_of_rec)));
fprintf(stderr,"buffer size = %d\n",buffer_size);
/* Allocate memory for hash table */
fprintf(stderr,"Initialising hash table...\n");
M = nearest_prime(hash_size);
new_idngram_hashtable(&vocabulary,M);
/* Read in the vocabulary */
vocab_size = 0;
vocab_file = rr_iopen(vocab_filename);
pc_message(verbosity,2,"Reading vocabulary...\n");
while (fgets (temp_word, sizeof(temp_word),vocab_file)) {
if (strncmp(temp_word,"##",2)==0) continue;
sscanf (temp_word, "%s ",temp_word2);
/* Check for vocabulary order */
if (vocab_size > 0 && strcmp(temp_word2,temp_word3)<0)
quit(-1,"wngram2idngram : Error : Vocabulary is not alphabetically ordered.\n");
/* Check for repeated words in the vocabulary */
if (index2(&vocabulary,temp_word2) != 0)
warn_on_repeated_words(temp_word);
warn_on_wrong_vocab_comments(temp_word);
vocab_size++;
add_to_idngram_hashtable(&vocabulary,idngram_hash(temp_word2,M),temp_word2,vocab_size);
strcpy(temp_word3,temp_word2);
}
if (vocab_size > MAX_VOCAB_SIZE)
quit(-1,"Error : Vocabulary size exceeds maximum.\n");
pc_message(verbosity,2,"Allocating memory for the buffer...\n");
buffer=(ngram_rec *) rr_malloc((buffer_size+1)*sizeof(ngram_rec));
for (i=0;i<=buffer_size;i++)
buffer[i].word = (wordid_t *) rr_malloc(n*sizeof(wordid_t));
/* Open the "non-OOV" tempfile */
sprintf(temp_word,"%s1%s",temp_file_root,temp_file_ext);
non_unk_fp = rr_fopen(temp_word,"w");
pc_message(verbosity,2,"Writing non-OOV counts to temporary file %s\n",
temp_word);
number_of_tempfiles = 1;
current_ngram = (wordid_t *) rr_malloc(n*sizeof(wordid_t));
sort_ngram = (wordid_t *) rr_malloc(n*sizeof(wordid_t));
/* Read text into buffer */
position_in_buffer = 0;
while (!rr_feof(stdin)) {
for (i=0;i<=n-1;i++) {
get_word(stdin,temp_word);
current_ngram[i]=index2(&vocabulary,temp_word);
}
if (scanf("%d",¤t_count) != 1)
if (!rr_feof(stdin))
quit(-1,"Error reading n-gram count from stdin.\n");
if (!rr_feof(stdin)) {
contains_unks = 0;
for (i=0;i<=n-1;i++) {
if (!current_ngram[i])
contains_unks = 1;
}
if (contains_unks) {
/* Write to buffer */
position_in_buffer++;
if (position_in_buffer >= buffer_size) {
/* Sort buffer */
pc_message(verbosity,2,
"Sorting n-grams which include an OOV word...\n");
qsort((void*) buffer,(size_t) position_in_buffer,
sizeof(ngram_rec),compare_ngrams2);
pc_message(verbosity,2,"Done.\n");
/* Write buffer to temporary file */
number_of_tempfiles++;
sprintf(temp_word,"%s%hu%s",temp_file_root,
number_of_tempfiles,temp_file_ext);
pc_message(verbosity,2,
"Writing sorted OOV-counts buffer to temporary file %s\n",
temp_word);
tempfile = rr_fopen(temp_word,"w");
for (i=0;i<=n-1;i++)
sort_ngram[i] = buffer[0].word[i];
sort_count = buffer[0].count;
for (i=0;i<=position_in_buffer-2;i++) {
same_ngram = 1;
for (j=n-1;j>=0;j--) {
if (buffer[i].word[j] != sort_ngram[j]) {
same_ngram = 0;
j = -1;
}
}
if (same_ngram)
sort_count += buffer[i].count;
else {
for (j=0;j<=n-1;j++) {
rr_fwrite((char*)&sort_ngram[j],sizeof(wordid_t),1,
tempfile,"temporary n-gram ids");
sort_ngram[j] = buffer[i].word[j];
}
rr_fwrite((char*)&sort_count,sizeof(int),1,tempfile,
"temporary n-gram counts");
sort_count = buffer[i].count;
}
}
for (j=0;j<=n-1;j++)
rr_fwrite((char*)&sort_ngram[j],sizeof(wordid_t),1,
tempfile,"temporary n-gram ids");
rr_fwrite((char*)&sort_count,sizeof(int),1,tempfile,
"temporary n-gram counts");
rr_oclose(tempfile);
position_in_buffer = 1;
}
for (i=0;i<=n-1;i++)
buffer[position_in_buffer-1].word[i] = current_ngram[i];
buffer[position_in_buffer-1].count = current_count;
}else {
/* Write to temporary file */
for (i=0;i<=n-1;i++)
rr_fwrite((char*)¤t_ngram[i],sizeof(unsigned short),1,
non_unk_fp,"temporary n-gram ids");
rr_fwrite((char*)¤t_count,sizeof(int),1,non_unk_fp,
"temporary n-gram counts");
}
}
}
if (position_in_buffer > 0) {
/* Only do this bit if we have actually seen some OOVs */
/* Sort final buffer */
pc_message(verbosity,2,"Sorting final buffer...\n");
qsort((void*) buffer,(size_t) position_in_buffer,
sizeof(ngram_rec),compare_ngrams2);
/* Write final buffer */
number_of_tempfiles++;
sprintf(temp_word,"%s%hu%s",temp_file_root,
number_of_tempfiles,temp_file_ext);
pc_message(verbosity,2,"Writing sorted buffer to temporary file %s\n", temp_word);
tempfile = rr_fopen(temp_word,"w");
for (i=0;i<=n-1;i++)
sort_ngram[i] = buffer[0].word[i];
sort_count = buffer[0].count;
for (i=1;i<=position_in_buffer-1;i++) {
same_ngram = 1;
for (j=n-1;j>=0;j--) {
if (buffer[i].word[j] != sort_ngram[j]) {
same_ngram = 0;
j = -1;
}
}
if (same_ngram)
sort_count += buffer[i].count;
else {
for (j=0;j<=n-1;j++) {
rr_fwrite((char*)&sort_ngram[j],sizeof(wordid_t),1,
tempfile,"temporary n-gram ids");
sort_ngram[j] = buffer[i].word[j];
}
rr_fwrite((char*)&sort_count,sizeof(int),1,tempfile,
"temporary n-gram counts");
sort_count = buffer[i].count;
}
}
for (j=0;j<=n-1;j++)
rr_fwrite((char*)&sort_ngram[j],sizeof(wordid_t),1,
tempfile,"temporary n-gram ids");
rr_fwrite((char*)&sort_count,sizeof(int),1,tempfile,
"temporary n-gram counts");
fclose(tempfile);
/* Merge the temporary files, and output the result to standard output */
fclose(non_unk_fp);
pc_message(verbosity,2,"Merging temporary files...\n");
merge_tempfiles(1,
number_of_tempfiles,
temp_file_root,
temp_file_ext,
max_files,
outfile,
write_ascii,
fof_size);
}else {
/* Just write out the none OOV buffer to stdout */
fclose(non_unk_fp);
merge_tempfiles(1,
1,
temp_file_root,
temp_file_ext,
max_files,
outfile,
write_ascii,
fof_size);
}
pc_message(verbosity,0,"wngram2idngram : Done.\n");
fclose(outfile);
exit(0);
}
