void Scanner::load_strand_machines(unsigned int within) {
// comments and strings, take that!
// add nested strings and comments feature using within variable
auto comment_machine = FSMachinePtr(
new FiniteMachineContainer(get_token_info(TokType::MP_COMMENT).first, true));
// add states
comment_machine->add_state("0", true, false);
comment_machine->add_state("1", false, false);
comment_machine->add_state("2", false, true);
// for state 1
comment_machine->add_alphabet("1", "1");
comment_machine->add_symbols("1", "1");
comment_machine->add_digits("1", "1");
comment_machine->add_transition("1", ' ', "1");
comment_machine->add_transition("1", '\n', "1");
comment_machine->add_transition("1", '\t', "1");
comment_machine->add_transition("1", '\v', "1");
comment_machine->add_transition("1", '\r', "1");
comment_machine->add_transition("1", '\f', "1");
comment_machine->remove_transition("1", '{');
comment_machine->remove_transition("1", '}');
comment_machine->add_transition("0", '{', "1");
comment_machine->add_transition("1", '}', "2");
// new machine
auto string_machine = FSMachinePtr(
new FiniteMachineContainer(get_token_info(TokType::MP_STRING_LITERAL).first, true));
// add states
string_machine->add_state("0", true, false);
string_machine->add_state("1", false, false);
string_machine->add_state("2", false, true);
// add transitions
string_machine->add_transition("0", '\'', "1");
string_machine->add_alphabet("1", "1");
string_machine->add_symbols("1", "1");
string_machine->add_digits("1", "1");
string_machine->add_transition("1", ' ', "1");
string_machine->remove_transition("1", '\'');
string_machine->remove_transition("1", '\'');
string_machine->add_transition("1", '\'', "2");
// set machine priority
comment_machine->set_priority(1);
string_machine->set_priority(1);
// add the machines to our FSMachines list
this->fsmachines->push_back(comment_machine);
this->fsmachines->push_back(string_machine);
}
static void
process_token_result (const gchar *xml_result, gpointer user_data)
{
xmlDocPtr doc;
xmlNodePtr node;
GFlickrData *data = (GFlickrData *) user_data;
GHashTable *token;
doc = xmlReadMemory (xml_result, xmlStrlen ((xmlChar*) xml_result), NULL,
NULL, XML_PARSE_RECOVER | XML_PARSE_NOBLANKS);
node = xmlDocGetRootElement (doc);
/* Check if result is OK */
if (!node || !result_is_correct (node)) {
data->hashtable_cb (data->flickr, NULL, data->user_data);
} else {
node = node->xmlChildrenNode;
token = get_token_info (node);
data->hashtable_cb (data->flickr, token, data->user_data);
g_hash_table_unref (token);
}
g_object_unref (data->flickr);
g_slice_free (GFlickrData, data);
xmlFreeDoc (doc);
}
void Scanner::display_tokens() {
for (vector<TokenPtr>::iterator i = this->consumed->begin();
i != consumed->end(); i++) {
report_error_lc("Found", "'" + (*i)->get_lexeme()
+ "' : " + get_token_info((*i)->get_token()).first, (*i)->get_line(), (*i)->get_column());
}
}
// load automata for scanning valid identifiers
void Scanner::load_id_machine() {
// create the id scan automata
auto id_machine = FSMachinePtr(
new FiniteMachineContainer(get_token_info(TokType::MP_ID).first,
true));
// detects ids without two underscores
id_machine->add_state("0", true, false);
id_machine->add_state("1", false, true);
id_machine->add_state("2", false, true);
id_machine->add_state("3", false, false);
id_machine->add_state("4", false, false);
// add transitions
id_machine->add_alphabet("0", "1");
id_machine->add_alphabet("1", "1");
id_machine->add_digits("1", "1");
id_machine->add_transition("1", '_', "4");
id_machine->add_alphabet("4", "1");
id_machine->add_digits("4", "1");
// other part
id_machine->add_transition("0", '_', "3");
id_machine->add_alphabet("3", "2");
id_machine->add_digits("3", "2");
id_machine->add_alphabet("2", "2");
id_machine->add_digits("2", "2");
id_machine->add_transition("2", '_', "3");
// set priority
id_machine->set_priority(2);
// add to finite machines
this->fsmachines->push_back(FSMachinePtr(id_machine));
}
void Scanner::load_num_machines() {
// create floating point automata for scanning numeric literals
auto float_machine = FSMachinePtr(
new FiniteMachineContainer(get_token_info(TokType::MP_FLOAT_LITERAL).first, true));
// add states to this DFA, first and final
float_machine->add_state("0", true, false);
float_machine->add_state("1", false, false);
float_machine->add_state("2", false, true);
float_machine->add_state("3", false, false);
float_machine->add_state("4", false, false);
float_machine->add_state("5", false, true);
float_machine->add_digits("0", "1");
float_machine->add_digits("1", "1");
float_machine->add_transition("1", '.', "2");
float_machine->add_digits("2", "2");
float_machine->add_transition("2", 'e', "3");
float_machine->add_transition("2", 'E', "3");
float_machine->add_transition("3", '+', "4");
float_machine->add_transition("3", '-', "4");
float_machine->add_digits("3", "5");
float_machine->add_digits("4", "5");
float_machine->add_digits("5", "5");
// create integer automata
auto integer_machine = FSMachinePtr(
new FiniteMachineContainer(get_token_info(TokType::MP_INT_LITERAL).first, true));
// add states to this DFA
integer_machine->add_state("0", true, false);
integer_machine->add_state("1", false, true);
integer_machine->add_digits("0", "1");
integer_machine->add_digits("1", "1");
// reset both
float_machine->reset();
integer_machine->reset();
// set machine priorities
float_machine->set_priority(1);
integer_machine->set_priority(1);
// add both to the finite machines list
this->fsmachines->push_back(FSMachinePtr(float_machine));
this->fsmachines->push_back(FSMachinePtr(integer_machine));
}
void Scanner::load_keyword_machines() {
// get the token type for all keyworded or single char types
for (auto i = (int) TokType::MP_SEMI_COLON;
i <= (int) TokType::MP_BOOLEAN; i++) {
// create new automata for all types
FSMachinePtr new_fa = FSMachinePtr(
new FiniteMachineContainer(get_token_info((TokType)i).first, true));
// build the keyword into the automata
new_fa->build_keyword(get_token_info((TokType)i).second);
// set priority: keywords are always more important than ids
new_fa->set_priority(1);
// add to the vector of automata
this->fsmachines->push_back(FSMachinePtr(new_fa));
}
}