ptokenizer::token_t ptokenizer::get_token_internal()
{
/* skip ws */
char c = getc();
while (m_whitespace.find(c)>=0)
{
c = getc();
if (eof())
{
return token_t(ENDOFFILE);
}
}
if (m_number_chars_start.find(c)>=0)
{
/* read number while we receive number or identifier chars
* treat it as an identifier when there are identifier chars in it
*
*/
token_type ret = NUMBER;
pstring tokstr = "";
while (true) {
if (m_identifier_chars.find(c)>=0 && m_number_chars.find(c)<0)
ret = IDENTIFIER;
else if (m_number_chars.find(c)<0)
break;
tokstr += c;
c = getc();
}
ungetc();
return token_t(ret, tokstr);
}
else if (m_identifier_chars.find(c)>=0)
{
/* read identifier till non identifier char */
pstring tokstr = "";
while (m_identifier_chars.find(c)>=0) {
tokstr += c;
c = getc();
}
ungetc();
token_id_t id = token_id_t(m_tokens.indexof(tokstr));
if (id.id() >= 0)
return token_t(id, tokstr);
else
{
return token_t(IDENTIFIER, tokstr);
}
}
else if (c == m_string)
{
pstring tokstr = "";
c = getc();
while (c != m_string)
{
tokstr += c;
c = getc();
}
return token_t(STRING, tokstr);
}
else
{
/* read identifier till first identifier char or ws */
pstring tokstr = "";
while ((m_identifier_chars.find(c)) < 0 && (m_whitespace.find(c) < 0)) {
tokstr += c;
/* expensive, check for single char tokens */
if (tokstr.len() == 1)
{
token_id_t id = token_id_t(m_tokens.indexof(tokstr));
if (id.id() >= 0)
return token_t(id, tokstr);
}
c = getc();
}
ungetc();
token_id_t id = token_id_t(m_tokens.indexof(tokstr));
if (id.id() >= 0)
return token_t(id, tokstr);
else
{
return token_t(UNKNOWN, tokstr);
}
}
}
ptokenizer::token_t ptokenizer::get_token_internal()
{
/* skip ws */
char c = getc();
while (m_whitespace.find(c)>=0)
{
c = getc();
if (eof())
{
return token_t(ENDOFFILE);
}
}
if (m_identifier_chars.find(c)>=0)
{
/* read identifier till non identifier char */
pstring tokstr = "";
while (m_identifier_chars.find(c)>=0) {
tokstr += c;
c = getc();
}
ungetc();
token_id_t id = token_id_t(m_tokens.indexof(tokstr));
if (id.id() >= 0)
return token_t(id, tokstr);
else
{
return token_t(IDENTIFIER, tokstr);
}
}
else if (c == m_string)
{
pstring tokstr = "";
c = getc();
while (c != m_string)
{
tokstr += c;
c = getc();
}
return token_t(STRING, tokstr);
}
else
{
/* read identifier till first identifier char or ws */
pstring tokstr = "";
while ((m_identifier_chars.find(c)) < 0 && (m_whitespace.find(c) < 0)) {
tokstr += c;
/* expensive, check for single char tokens */
if (tokstr.len() == 1)
{
token_id_t id = token_id_t(m_tokens.indexof(tokstr));
if (id.id() >= 0)
return token_t(id, tokstr);
}
c = getc();
}
ungetc();
token_id_t id = token_id_t(m_tokens.indexof(tokstr));
if (id.id() >= 0)
return token_t(id, tokstr);
else
{
return token_t(UNKNOWN, tokstr);
}
}
}
/*
Add one line of CLI to the parse tree.
root Root node of the parse tree.
line A line of command from a CLI file.
returns the end of the nodes created
*/
node_t *add_cli(node_t *root, const std::string &filename, const std::string &line)
{
nodes_t nodes;
int flags = 0;
int num_opt_start = 0;
strings_t tokens;
std::stringstream ss(line);
ss.imbue(std::locale(std::locale(), new tokenizer_t(' ')));
std::istream_iterator<std::string> begin(ss);
std::istream_iterator<std::string> end;
tokens.assign(begin, end);
if(tokens.size() == 0)
return root; // blank line
// Convert tokens to parse tree nodes. '{' and '}' do not produce tree
// nodes.But they do affect the flags used in some nodes.
bool start_flag = false;
for(strings_t::const_iterator it = tokens.begin(); it != tokens.end(); it++)
{
// parse each token and look for '{' which identifies it as an optional node
if(*it == "{")
{
// In the last node, its flags field is a reference to
// 'flags'. So, if we append to it, the last node will get
// a new flag.
start_flag = true;
num_opt_start++;
continue;
}
else if(*it == "}")
{
if(nodes.empty())
throw std::string("Empty nodes inside {}");
nodes.back()->flags(nodes.back()->flags() | CLI_NODE_FLAGS_OPT_END);
num_opt_start--;
continue;
}
if(num_opt_start > 0)
flags = CLI_NODE_FLAGS_OPT_PARTIAL;
if(start_flag)
flags |= CLI_NODE_FLAGS_OPT_START;
token_t tt = token_t(it->c_str());
nodes.push_back(new node_t(tt.node_type(), tt.variable_type(), tt.get_function_name(), tt.name(), tt.enum_type(), tt.desc(), flags, filename));
start_flag = false;
}
// Insert them into the parse tree
node_t *cur_node = root;
int num_braces;
for(nodes_t::const_iterator it = nodes.begin(); it != nodes.end(); it++)
{
// determine if the node has a child with the same name as the current node
// if so we use that node.
cur_node = cur_node->add_child(*it);
}
// put a marker on the end
end_node = new node_t("END", "", "", "end", "", "", CLI_NODE_FLAGS_OPT_END, filename);
cur_node->add_child(end_node);
return end_node;
}
ptokenizer::token_t ptokenizer::get_token_internal()
{
/* skip ws */
pstring::value_type c = getc();
while (m_whitespace.find(c) != pstring::npos)
{
c = getc();
if (eof())
{
return token_t(ENDOFFILE);
}
}
if (m_number_chars_start.find(c) != pstring::npos)
{
/* read number while we receive number or identifier chars
* treat it as an identifier when there are identifier chars in it
*
*/
token_type ret = NUMBER;
pstring tokstr = "";
while (true) {
if (m_identifier_chars.find(c) != pstring::npos && m_number_chars.find(c) == pstring::npos)
ret = IDENTIFIER;
else if (m_number_chars.find(c) == pstring::npos)
break;
tokstr += c;
c = getc();
}
ungetc(c);
return token_t(ret, tokstr);
}
else if (m_identifier_chars.find(c) != pstring::npos)
{
/* read identifier till non identifier char */
pstring tokstr = "";
while (m_identifier_chars.find(c) != pstring::npos)
{
tokstr += c;
c = getc();
}
ungetc(c);
auto id = m_tokens.find(tokstr);
if (id != m_tokens.end())
return token_t(id->second, tokstr);
else
return token_t(IDENTIFIER, tokstr);
}
else if (c == m_string)
{
pstring tokstr = "";
c = getc();
while (c != m_string)
{
tokstr += c;
c = getc();
}
return token_t(STRING, tokstr);
}
else
{
/* read identifier till first identifier char or ws */
pstring tokstr = "";
while ((m_identifier_chars.find(c) == pstring::npos) && (m_whitespace.find(c) == pstring::npos))
{
tokstr += c;
/* expensive, check for single char tokens */
if (tokstr.length() == 1)
{
auto id = m_tokens.find(tokstr);
if (id != m_tokens.end())
return token_t(id->second, tokstr);
}
c = getc();
}
ungetc(c);
auto id = m_tokens.find(tokstr);
if (id != m_tokens.end())
return token_t(id->second, tokstr);
else
return token_t(UNKNOWN, tokstr);
}
}
