NABoolean
ComSqlTextHandle::isIdentifierPartOrUnderscore(const char &aChar) const
{
if (isIdentifierPart(aChar) OR isUnderscore(aChar))
{
return TRUE;
}
return FALSE;
}
int main() {
std::cout << "Ascii letters are identifier start " <<
(isIdentifierStart('a') ? "[OK]" : "[FAIL]") <<
std::endl;
std::cout << "Ascii letters are identifier part " <<
(isIdentifierPart('a') ? "[OK]" : "[FAIL]") <<
std::endl;
std::cout << "Numbers are not identifier start " <<
(!isIdentifierStart('0') ? "[OK]" : "[FAIL]") <<
std::endl;
std::cout << "Numbers are identifier part " <<
(isIdentifierPart('0') ? "[OK]" : "[FAIL]") <<
std::endl;
}
bool tokeniseIdentifier(std::string::const_iterator& s, std::string::const_iterator& e, Token& tok)
{
// Be sure that first char is alphanumeric or _ or $
if ( s==e || !isIdentifierStart(*s) ) return false;
std::string::const_iterator t = s;
while ( s!=e && isIdentifierPart(*++s) );
tok = Token(T_IDENTIFIER, t, s);
return true;
}
Token Lexer::parseIdentifier()
{
// Add the first character to the identifier
std::string identifier{next_};
advance();
// Match more identifier parts
while (!atEof_ && isIdentifierPart(next_)) {
identifier += next_;
advance();
}
skipSpaces();
return Token(TokenType::IDENTIFIER, identifier);
}
bool tokenise(std::string::const_iterator& s, std::string::const_iterator& e, Token& tok)
{
std::string::const_iterator t = s;
// Hand constructed state machine recogniser
enum {
START,
REJECT,
IDENTIFIER,
ZERO,
DIGIT,
HEXDIGIT_START,
HEXDIGIT,
OCTDIGIT,
BINDIGIT_START,
BINDIGIT,
DECIMAL_START,
DECIMAL,
EXPONENT_SIGN,
EXPONENT_START,
EXPONENT,
ACCEPT_IDENTIFIER,
ACCEPT_INC,
ACCEPT_NOINC
} state = START;
TokenType tokType = T_EOS;
while (true)
switch (state) {
case START:
if (t==e) {tok = Token(T_EOS, s, "<END>"); return true;}
else if (std::isspace(*t)) {++t; ++s; continue;}
else switch (*t) {
case '(': tokType = T_LPAREN; state = ACCEPT_INC; continue;
case ')': tokType = T_RPAREN; state = ACCEPT_INC; continue;
case ',': tokType = T_COMMA; state = ACCEPT_INC; continue;
case '+': tokType = T_PLUS; state = ACCEPT_INC; continue;
case '-': tokType = T_MINUS; state = ACCEPT_INC; continue;
case '*': tokType = T_MULT; state = ACCEPT_INC; continue;
case '/': tokType = T_DIV; state = ACCEPT_INC; continue;
case '=': tokType = T_EQUAL; state = ACCEPT_INC; continue;
case '<':
++t;
if (t==e || (*t!='>' && *t!='='))
{tokType = T_LESS; state = ACCEPT_NOINC; continue; }
else
{tokType = (*t=='>') ? T_NEQ : T_LSEQ; state = ACCEPT_INC; continue; }
case '>':
++t;
if (t==e || *t!='=')
{tokType = T_GRT; state = ACCEPT_NOINC; continue;}
else
{tokType = T_GREQ; state = ACCEPT_INC; continue;}
default:
break;
}
if (isIdentifierStart(*t)) {++t; state = IDENTIFIER;}
else if (*t=='\'') {return processString(s, e, '\'', T_STRING, tok);}
else if (*t=='\"') {return processString(s, e, '\"', T_IDENTIFIER, tok);}
else if (*t=='0') {++t; state = ZERO;}
else if (std::isdigit(*t)) {++t; state = DIGIT;}
else if (*t=='.') {++t; state = DECIMAL_START;}
else state = REJECT;
continue;
case IDENTIFIER:
if (t==e) {state = ACCEPT_IDENTIFIER;}
else if (isIdentifierPart(*t)) {++t; state = IDENTIFIER;}
else state = ACCEPT_IDENTIFIER;
continue;
case DECIMAL_START:
if (t==e) {state = REJECT;}
else if (std::isdigit(*t)) {++t; state = DECIMAL;}
else state = REJECT;
continue;
case EXPONENT_SIGN:
if (t==e) {state = REJECT;}
else if (*t=='-' || *t=='+') {++t; state = EXPONENT_START;}
else if (std::isdigit(*t)) {++t; state = EXPONENT;}
else state = REJECT;
continue;
case EXPONENT_START:
if (t==e) {state = REJECT;}
else if (std::isdigit(*t)) {++t; state = EXPONENT;}
else state = REJECT;
continue;
case ZERO:
if (t==e) {tokType = T_NUMERIC_EXACT; state = ACCEPT_NOINC;}
else if (*t=='.') {++t; state = DECIMAL;}
else if (*t=='x' || *t=='X') {++t; state = HEXDIGIT_START;}
else if (*t=='b' || *t=='B') {++t; state = BINDIGIT_START;}
else state = OCTDIGIT;
continue;
case HEXDIGIT_START:
if (t==e) {state = REJECT;}
else if (std::isxdigit(*t)) {++t; state = HEXDIGIT;}
else state = REJECT;
continue;
case HEXDIGIT:
if (t==e) {tokType = T_NUMERIC_EXACT; state = ACCEPT_NOINC;}
else if (*t=='l' || *t=='L') {tokType = T_NUMERIC_EXACT; state = ACCEPT_INC;}
else if (std::isxdigit(*t) || *t=='_') {++t; state = HEXDIGIT;}
else if (*t=='p' || *t=='P') {++t; state = EXPONENT_SIGN;}
else {tokType = T_NUMERIC_EXACT; state = ACCEPT_NOINC;}
continue;
case BINDIGIT_START:
if (t==e) {state = REJECT;}
else if (*t=='0' || *t=='1') {++t; state = BINDIGIT;}
else state = REJECT;
continue;
case BINDIGIT:
if (t==e) {tokType = T_NUMERIC_EXACT; state = ACCEPT_NOINC;}
else if (*t=='l' || *t=='L') {tokType = T_NUMERIC_EXACT; state = ACCEPT_INC;}
else if (*t=='0' || *t=='1' || *t=='_') {++t; state = BINDIGIT;}
else {tokType = T_NUMERIC_EXACT; state = ACCEPT_NOINC;}
continue;
case OCTDIGIT:
if (t==e) {tokType = T_NUMERIC_EXACT; state = ACCEPT_NOINC;}
else if (*t=='l' || *t=='L') {tokType = T_NUMERIC_EXACT; state = ACCEPT_INC;}
else if ((std::isdigit(*t) && *t<'8') || *t=='_') {++t; state = OCTDIGIT;}
else {tokType = T_NUMERIC_EXACT; state = ACCEPT_NOINC;}
continue;
case DIGIT:
if (t==e) {tokType = T_NUMERIC_EXACT; state = ACCEPT_NOINC;}
else if (*t=='l' || *t=='L') {tokType = T_NUMERIC_EXACT; state = ACCEPT_INC;}
else if (*t=='f' || *t=='F' || *t=='d' || *t=='D') {tokType = T_NUMERIC_APPROX; state = ACCEPT_INC;}
else if (std::isdigit(*t) || *t=='_') {++t; state = DIGIT;}
else if (*t=='.') {++t; state = DECIMAL;}
else if (*t=='e' || *t=='E') {++t; state = EXPONENT_SIGN;}
else {tokType = T_NUMERIC_EXACT; state = ACCEPT_NOINC;}
continue;
case DECIMAL:
if (t==e) {tokType = T_NUMERIC_APPROX; state = ACCEPT_NOINC;}
else if (std::isdigit(*t) || *t=='_') {++t; state = DECIMAL;}
else if (*t=='e' || *t=='E') {++t; state = EXPONENT_SIGN;}
else if (*t=='f' || *t=='F' || *t=='d' || *t=='D') {tokType = T_NUMERIC_APPROX; state = ACCEPT_INC;}
else {tokType = T_NUMERIC_APPROX; state = ACCEPT_NOINC;}
continue;
case EXPONENT:
if (t==e) {tokType = T_NUMERIC_APPROX; state = ACCEPT_NOINC;}
else if (std::isdigit(*t)) {++t; state = EXPONENT;}
else if (*t=='f' || *t=='F' || *t=='d' || *t=='D') {tokType = T_NUMERIC_APPROX; state = ACCEPT_INC;}
else {tokType = T_NUMERIC_APPROX; state = ACCEPT_NOINC;}
continue;
case ACCEPT_INC:
++t;
case ACCEPT_NOINC:
tok = Token(tokType, s, t);
s = t;
return true;
case ACCEPT_IDENTIFIER:
tok = Token(T_IDENTIFIER, s, t);
s = t;
tokeniseReservedWord(tok);
return true;
case REJECT:
return false;
};
}
