//-----------------------------------------------------------------------------
bool SqlTokenizer::nextToken()
{
sqlTokenStartM = sqlTokenEndM;
if (sqlTokenEndM == 0 || *sqlTokenEndM == 0)
{
sqlTokenTypeM = tkEOF;
return false;
}
// use QChar* member to scan
QChar c = *sqlTokenEndM;
if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'))
keywordIdentifierToken();
else if (c == '"')
quotedIdentifierToken();
else if (c == '\'')
stringToken();
else if (c == '(')
symbolToken(tkPARENOPEN);
else if (c == ')')
symbolToken(tkPARENCLOSE);
else if (c == '=')
symbolToken(tkEQUALS);
else if (c == ',')
symbolToken(tkCOMMA);
else if (c == '/' && *(sqlTokenEndM + 1) == '*')
multilineCommentToken();
else if (c == '-' && *(sqlTokenEndM + 1) == '-')
singleLineCommentToken();
else if (c.isSpace())
whitespaceToken();
else
defaultToken();
return true;
}
static bool nextToken(Token &x) {
x = Token();
const char *p = save();
if (space(x)) goto success;
restore(p); if (docStartLine(x)) goto success;
restore(p); if (docStartBlock(x)) goto success;
restore(p); if (lineComment(x)) goto success;
restore(p); if (blockComment(x)) goto success;
restore(p); if (staticIndex(x)) goto success;
restore(p); if (opIdentifier(x)) goto success;
restore(p); if (symbol(x)) goto success;
restore(p); if (op(x)) goto success;
restore(p); if (llvmToken(x)) goto success;
restore(p); if (keywordIdentifier(x)) goto success;
restore(p); if (charToken(x)) goto success;
restore(p); if (stringToken(x)) goto success;
restore(p); if (floatToken(x)) goto success;
restore(p); if (intToken(x)) goto success;
if (p != end) {
pushLocation(locationFor(p));
error("invalid token");
}
return false;
success :
assert(x.tokenKind != T_NONE);
if (!x.location.ok())
x.location = locationFor(p);
return true;
}
void xmlProcessObject(struct XmlNode *parent, char xml[])
{
int pos = 0;
char *block = stringToken(&(xml[pos]), " \t\n");
while (block != NULL)
{
if (block == '\0')
{
pos++;
continue;
}
pos += stringLength(block);
if (block[0] == '<')
{
(*parent).name = &(block[1]);
}
block = stringToken(NULL, " \t\n");
}
}
PrologToken getToken(const std::string& str, unsigned int& idx,
const std::string& fileName, int& line, int& col) {
top:
const char normalOperatorChars[] = "+-*/\\^<>=`~:.?@#&";
if (idx == str.size()) { // End of string
return endOfFile(fileName, line, col);
}
int oldLine = line, oldCol = col;
#define STAR_P (str[idx])
#define P_SUB_1 (idx + 1 == str.size() ? 0 : str[idx + 1])
#define INC_INDEX do { \
if (STAR_P == '\n') { \
++line; \
col = 1; \
} else { \
++col; \
} \
++idx; \
} while (false)
if (islower(STAR_P)) { // Normal atoms
unsigned int oldIdx = idx;
do {INC_INDEX;} while (isalnum(STAR_P) || STAR_P == '_');
bool startOfTerm = false;
if (STAR_P == '(') {startOfTerm = true; INC_INDEX;}
return atom(str.substr(oldIdx, idx - oldIdx - startOfTerm), startOfTerm,
fileName, oldLine, oldCol);
} else if (STAR_P == '_' && !isalnum(P_SUB_1)) { // Wildcard
INC_INDEX;
return wildcard(fileName, oldLine, oldCol);
} else if (isupper(STAR_P) || STAR_P == '_' || STAR_P == '$') { // Variables
unsigned int oldIdx = idx;
do {INC_INDEX;} while (isalnum(STAR_P) || STAR_P == '_' || STAR_P == '$');
return variable(str.substr(oldIdx, idx - oldIdx), fileName, oldLine, oldCol);
} else if (isdigit(STAR_P) || STAR_P == '-' && isdigit(P_SUB_1)) { // Numbers
int isNegative = 1;
if (STAR_P == '-') {isNegative = -1; INC_INDEX;}
unsigned int oldIdx = idx;
do {INC_INDEX;} while (isdigit(STAR_P));
std::string tokenString;
if (STAR_P == '.' && isdigit(P_SUB_1)) { // Floating point
INC_INDEX;
while (isdigit(STAR_P)) INC_INDEX;
if (STAR_P == 'e' || STAR_P == 'E') {
INC_INDEX;
if (STAR_P == '-') INC_INDEX;
while (isdigit(STAR_P)) INC_INDEX;
}
return floatingPoint(isNegative *
stringToDouble(str.substr(oldIdx, idx - oldIdx),
fileName, oldLine, oldCol),
fileName, oldLine, oldCol);
} else if (STAR_P == '\'') { // Base or character definition
int base = stringToInt(str.substr(oldIdx, idx - oldIdx), 10);
if (base == 0 && P_SUB_1) { // Character
INC_INDEX;
return integer(isNegative * (STAR_P), fileName, oldLine, oldCol);
} else if (base >= 2 && base <= 36) {
INC_INDEX;
oldIdx = idx;
while (isalnum(STAR_P)) INC_INDEX;
return integer(isNegative *
stringToInt(str.substr(oldIdx, idx - oldIdx), base),
fileName, oldLine, oldCol);
} else {
std::cerr << "Invalid number base " << base << std::endl;
abort();
}
} else { // Normal integer
return integer(isNegative *
stringToInt(str.substr(oldIdx, idx - oldIdx), 10),
fileName, oldLine, oldCol);
}
} else if (STAR_P == '.' && P_SUB_1 && !strchr(normalOperatorChars, P_SUB_1)) {
INC_INDEX;
return period(fileName, oldLine, oldCol);
} else if (strchr(normalOperatorChars, STAR_P)) { // Operators
unsigned int oldIdx = idx;
do {INC_INDEX;} while (strchr(normalOperatorChars, STAR_P));
bool startOfTerm = false;
if (STAR_P == '(') {startOfTerm = true; INC_INDEX;}
return atom(str.substr(oldIdx, idx - oldIdx - startOfTerm), startOfTerm,
fileName, oldLine, oldCol);
} else if (STAR_P == '!' || STAR_P == ';') { // Single-character operators
char c = STAR_P;
INC_INDEX;
bool startOfTerm = false;
if (STAR_P == '(') {startOfTerm = true; INC_INDEX;}
return atom(std::string(1, c), startOfTerm, fileName, oldLine, oldCol);
} else if (STAR_P == '\'') { // Quoted atoms
std::string result;
INC_INDEX;
unsigned int oldIdx = idx;
get_more_chars_atom:
while (STAR_P != '\'') INC_INDEX;
result += str.substr(oldIdx, idx - oldIdx);
if (P_SUB_1 == '\'') { // Escaped single quote
result += "'";
INC_INDEX; INC_INDEX;
goto get_more_chars_atom;
} else { // End of atom
INC_INDEX;
bool startOfTerm = false;
if (STAR_P == '(') {startOfTerm = true; INC_INDEX;}
return atom(result, startOfTerm, fileName, oldLine, oldCol);
}
} else if (STAR_P == '"') { // Strings
std::string result;
INC_INDEX;
unsigned int oldIdx = idx;
get_more_chars_string:
while (STAR_P && STAR_P != '"' && STAR_P != '\\') INC_INDEX;
result += str.substr(oldIdx, idx - oldIdx);
if (STAR_P == '"') {
if (P_SUB_1 == '"') { // Escaped double quote
result += "\"";
INC_INDEX; INC_INDEX;
oldIdx = idx;
goto get_more_chars_string;
} else { // End of string
INC_INDEX;
return stringToken(result, fileName, oldLine, oldCol);
}
} else if (STAR_P == '\\') {
std::cerr << "Cannot handle backslash escapes in strings yet" << std::endl;
abort();
} else if (!STAR_P) {
std::cerr << "Unterminated string constant" << std::endl;
abort();
}
} else if (STAR_P == '(') {
INC_INDEX;
return leftParen(fileName, oldLine, oldCol);
} else if (STAR_P == ')') {
INC_INDEX;
return rightParen(fileName, oldLine, oldCol);
} else if (STAR_P == '[') {
if (P_SUB_1 == ']') {
INC_INDEX; INC_INDEX;
bool startOfTerm = false;
if (STAR_P == '(') {startOfTerm = true; INC_INDEX;}
return atom("[]", startOfTerm, fileName, oldLine, oldCol);
} else {
INC_INDEX;
return leftBracket(fileName, oldLine, oldCol);
}
} else if (STAR_P == ']') {
INC_INDEX;
return rightBracket(fileName, oldLine, oldCol);
} else if (STAR_P == '|') {
INC_INDEX;
return verticalBar(fileName, oldLine, oldCol);
} else if (STAR_P == ',') {
INC_INDEX;
return comma(fileName, oldLine, oldCol);
} else if (isspace(STAR_P)) { // White space
INC_INDEX;
goto top;
} else if (STAR_P == '%') { // Comment
do {INC_INDEX;} while (STAR_P && STAR_P != '\n' && STAR_P != '\r');
if (!STAR_P) {
std::cerr << "Unterminated comment" << std::endl;
abort();
}
goto top;
} else {
std::cerr << "Bad character '" << STAR_P << "'" << std::endl;
abort();
}
#undef STAR_P
#undef P_SUB_1
#undef INC_INDEX
std::cerr << "Should not get here" << std::endl;
abort();
return endOfFile(fileName, line, col); // Should not get here
}
