void CTokenizer::Tokenize(const std::string& _expression, std::list<Token>& tokens)
{
std::string expression = _expression;
std::smatch match;
leftTrimString(expression, 0);
while (!expression.empty())
{
bool valid_token = false;
for (const auto& pattern : token_patterns)
{
if (std::regex_search(expression, match, pattern.first))
{
#ifdef ENABLE_DEBUG
std::cout << match[1];
#endif
tokens.push_back(std::make_pair(match[1], pattern.second));
leftTrimString(expression, (int)match[1].length());
valid_token = true;
break;
}
}
if (!valid_token)
{
std::cout << "Error: wrong output format near : " << expression << "\n";
exit(1);
}
}
}
void trimString(std::string & str) noexcept {
leftTrimString(str);
rightTrimString(str);
}
