int tokenize(FILE* fp)
{
int ch, previous;
struct array *word;
struct array *input;
int num_characters=0;
// Allocate space to store the tokens
if( (word = malloc(sizeof(struct array))) == NULL ) { end(-1, "Reader Tokenize: malloc error "); }
word->size = WORDSIZE;
if( (word->word = malloc(word->size*sizeof(char))) == NULL ) { end(-1, "Reader Tokenize: malloc error "); }
// Allocate space to store the unprocessed input
if( (input = malloc(sizeof(struct array))) == NULL ) { end(-1, "Reader Tokenize: malloc error "); }
input->word = NULL;
// Initialise vars
initialise(word);
previous=-1;
ch = consume(fp, input);
while(ch!=EOF) {
if(character(ch)==TRUE) {
construct(word, ch);
}
else if(special_character(ch)==TRUE) {
if(character(previous)==TRUE) {
create_statistics(word);
send(word);
initialise(word);
}
construct(word, ch);
create_statistics(word);
send(word);
initialise(word);
}
num_characters++;
previous = ch;
ch = consume(fp, input);
}
// Make statistics accessible by printer
send_statistics();
// Inform text formatter that we are done
initialise(word);
send(word);
// Deallocate space
free(word->word);
free(word);
free(input);
}
void extract_set_component (
char *line
)
{
char *ce;
char *cs = &line[4];
ce = strstr (line, "/bld/");
if (ce) {
*ce = '\0';
}
if ((component == NULL) || (strcmp (component, cs) != 0)) {
component = f_strdup (cs);
create_statistics (component);
}
}
