const char* cPasTokenizer::HandleBlockComment(const char* strLine, bool bSkipComments)
{
tToken token;
int nOffset = 1;
const char* strEnd = strchr(strLine, '}');
const char* strEnd2 = strstr(strLine, "*)");
if (((strEnd == NULL && strEnd2 != NULL)) || ((strEnd2 != NULL) && (strEnd2 < strEnd)))
{
strLine++;
strEnd = strEnd2;
nOffset = 2;
}
if (strEnd == NULL)
{
m_strBuffer = strLine;
m_bBlockComment = true;
return NULL;
}
else if(strLine == strEnd)
{
if (!bSkipComments)
PushToken(TOKEN_COMMENT, "", 0);
return strEnd + nOffset;
}
else
{
if (!bSkipComments)
PushToken(TOKEN_COMMENT, strLine, strEnd - strLine);
m_bBlockComment = false;
m_strBuffer.clear();
return strEnd + nOffset;
}
}
const char* cPPTokenizer::HandleMessage(const char* strLine)
{
int iLen = strlen(strLine);
if (m_strMessage.empty())
{
m_strMessage.clear();
while (*strLine == ' ' || *strLine == '\t')
{
strLine++;
}
}
m_strMessage.append(strLine);
if (strLine[iLen-1] != '\\')
{
PushToken(TOKEN_TEXT, m_strMessage.c_str());
m_strMessage.clear();
m_bMessage = false;
m_bPreProcMode = false;
PushToken(TOKEN_OPERATOR, PP_OP_PREPROC_END);
}
else
{
m_strMessage.append("\n");
}
return NULL;
}
uint32_t Tokenizer::StringTokenize(char_t *str, const char_t *delim)
{
uint32_t delim_len = ecila_strlen(delim);
if(delim_len == 0) {
ECILA_TRACE();
return 0;
}
if(delim_len == 1) {
return CTokenize(str, delim);
}
bool_t ret = FALSE;
char_t *next = str;
char_t *token = 0;
token = strstr(next, delim);
if(!token) {
return count_;
}
while(token != 0)
{
if(next == token)
{
next = token + delim_len;
token = strstr(next, delim);
continue;
}
*token = '\0';
ret = PushToken(next);
if(!ret) {
ECILA_TRACE();
return 0;
}
next = token + delim_len;
token = strstr(next, delim);
}
ret = PushToken(next);
if(!ret) {
ECILA_TRACE();
return 0;
}
return count_;
}
void cPasTokenizer::PushKeyword(int nKeyword)
{
switch (nKeyword)
{
case PAS_KW_BEGIN:
PushToken(TOKEN_BLOCK_BEGIN);
break;
case PAS_KW_END:
PushToken(TOKEN_BLOCK_END);
break;
default:
cTokenizer::PushKeyword(nKeyword);
break;
}
}
void cTokenizer::PushToken(int nTokenType, char cChar)
{
tToken token;
token.m_Token = nTokenType;
token.m_cChar = cChar;
PushToken(token);
}
Scanner::IndentMarker* Scanner::PushIndentTo(int column,
IndentMarker::INDENT_TYPE type) {
// are we in flow?
if (InFlowContext()) {
return 0;
}
std::unique_ptr<IndentMarker> pIndent(new IndentMarker(column, type));
IndentMarker& indent = *pIndent;
const IndentMarker& lastIndent = *m_indents.top();
// is this actually an indentation?
if (indent.column < lastIndent.column) {
return 0;
}
if (indent.column == lastIndent.column &&
!(indent.type == IndentMarker::SEQ &&
lastIndent.type == IndentMarker::MAP)) {
return 0;
}
// push a start token
indent.pStartToken = PushToken(GetStartTokenFor(type));
// and then the indent
m_indents.push(&indent);
m_indentRefs.push_back(std::move(pIndent));
return &m_indentRefs.back();
}
void cTokenizer::PushToken(int nTokenType, int nOpType)
{
tToken token;
token.m_Token = nTokenType;
token.m_Type = nOpType;
PushToken(token);
}
const char* cPasTokenizer::HandleString(const char* strLine)
{
static char strBuffer[256];
const char* strEnd = NULL;
const char* strCrsr = strLine;
strBuffer[0] = '\0';
while (true)
{
strEnd = strchr(strCrsr+1, '\'');
if (strEnd == NULL)
{
std::stringstream strError;
strError << "ERROR: Missing end of string character " << *strLine << std::endl;
GetTokenHandler()->HandleError(strError.str().c_str(), GetLine());
return NULL;
}
strncat(strBuffer, strCrsr, strEnd-strCrsr+1);
if (*(strEnd+1) == '\'')
{
strEnd++;
}
else
{
break;
}
strCrsr = strEnd;
};
PushToken(TOKEN_STRING, strBuffer, 0);
return strEnd+1;
}
const char* cTokenizer::ParseLabel(const char* strLine)
{
std::string strBuffer;
const char* strCrsr = strLine+1;
while(char c = *strCrsr++)
{
if(!isalpha(c) && !isdigit(c) && c != '_')
break;
}
int iLen = strCrsr - strLine - 1;
strBuffer.assign(strLine, iLen);
int kw = IsKeyword(strBuffer.c_str());
if (kw != m_nUnkownKeyword)
{
PushKeyword(kw);
}
else
{
PushToken(TOKEN_LABEL, strLine, iLen);
}
return strCrsr-1;
}
void cPPTokenizer::PushTokenIfPreProcMode(int nToken, int nType, const char* strText)
{
if (m_bPreProcMode)
{
PushToken(nToken, nType);
}
else
{
if (strlen(strText) > 1)
{
PushToken(TOKEN_TEXT, strText);
}
else
{
PushToken(TOKEN_CHAR, strText[0]);
}
}
}
const char* cPPTokenizer::HandleWhiteSpace(const char* strLine, bool bLineStart)
{
const char* strCrsr = strLine;
char c = *strCrsr;
while(c && (c == ' ' || c == '\t'))
{
c = *(++strCrsr);
}
if (strCrsr[0] != '\0')
{
if (bLineStart)
{
PushToken(TOKEN_WHITESPACE, strLine, strCrsr - strLine);
}
else
{
PushToken(TOKEN_WHITESPACE, " ");
}
}
return strCrsr;
}
const char* cTokenizer::HandleWhiteSpace(const char* strLine, bool bSkipWhiteSpaces)
{
const char* strCrsr = strLine;
char c = *strCrsr;
while(c && (c == ' ' || c == '\t'))
{
c = *(++strCrsr);
}
if (!bSkipWhiteSpaces)
PushToken(TOKEN_WHITESPACE, strLine, strCrsr - strLine);
return strCrsr;
}
void cTokenizer::PushToken(int nTokenType, const char* strName, int iLen)
{
tToken token;
token.m_Token = nTokenType;
if (iLen == 0)
{
token.SetName(strName);
}
else
{
token.SetName(strName, iLen);
}
PushToken(token);
}
const char* cPasTokenizer::AppendBlockComment(const char* strLine, bool bSkipComments)
{
m_strBuffer.append("\n");
const char* strEnd = strchr(strLine, '}');
if (strEnd == NULL)
{
m_strBuffer.append(strLine);
return NULL;
}
m_strBuffer.append(strLine, strEnd - strLine);
if (!bSkipComments)
PushToken(TOKEN_COMMENT, m_strBuffer.c_str());
m_bBlockComment = false;
m_strBuffer.clear();
return strEnd + 1;
}
uint32_t Tokenizer::CharacterTokenize(char_t *str, const char_t *delim)
{
bool_t ret = FALSE;
char_t *next = 0;
char_t *token = 0;
token = ecila_strtok(str, delim, &next);
while(token != 0)
{
ret = PushToken(token);
if(!ret) {
ECILA_TRACE();
return 0;
}
token = ecila_strtok(0, delim, &next);
}
return count_;
}
bool cPPTokenizer::PushPreProcEnd()
{
// inside a block comment or strings, the state of the preprocessor mode is not changed!
if (m_bBlockComment || m_bMultiLineString)
{
return true;
}
if (m_bPreProcMode)
{
m_bPreProcMode = false;
m_bInclude = false;
PushToken(TOKEN_OPERATOR, PP_OP_PREPROC_END);
}
if (m_bStop)
{
m_bStop = false;
return false;
}
return true;
}
const char* cPPTokenizer::HandleString(const char* strLine, char cDelimiter, int nToken)
{
std::stringstream strError;
const char* strEnd = NULL;
const char* strCrsr = strLine;
if (*strLine == 'l' || *strLine == 'L')
strCrsr++;
do
{
strEnd = strchr(strCrsr+1, cDelimiter);
if (strEnd == NULL)
{
int iLen = strlen(strLine)-1;
if (nToken == TOKEN_STRING)
{
switch (strLine[iLen])
{
case '\\':
m_strBuffer.assign(strLine, iLen);
m_bMultiLineString = true;
return strLine + iLen + 1;
default: // continue
break;
}
}
strError << "ERROR: Missing end of string character " << *strLine << std::endl;
GetTokenHandler()->HandleError(strError.str().c_str(), GetLine());
return NULL;
}
strCrsr = strEnd;
} while((*(strEnd-1) == '\\') && (*(strEnd-2) != '\\'));
int iLen = strEnd - strLine + 1;
PushToken(nToken, strLine, iLen);
m_bMultiLineString = false;
return strLine + iLen;
}
void cPPTokenizer::PushKeyword(int nKeyword)
{
if (!m_bExpectKeyword)
{
PushToken(TOKEN_LABEL, GetKeywordString(nKeyword));
}
else
{
m_bInclude = false;
m_bExpectKeyword = false;
switch (nKeyword)
{
case PP_KW_INCLUDE:
m_bInclude = true;
// after includes, preprocmode must be false, since the included file
// shouldn't start with this mode activated
m_bPreProcMode = false;
break;
case PP_KW_ERROR:
case PP_KW_WARNING:
m_bMessage = true;
break;
case PP_KW_PRAGMA:
m_bPragma = true;
break;
case PP_KW_IF:
case PP_KW_ELSE:
case PP_KW_ELIF:
case PP_KW_DEFINED:
m_bExpectKeyword = true;
break;
default:
break;
}
cTokenizer::PushKeyword(nKeyword);
}
}
const char* cPPTokenizer::AppendString(const char* strLine)
{
const char* strEnd = strstr(strLine, "\"");
if (strEnd == NULL)
{
int nLen = strlen(strLine);
if (strLine[nLen-1] == '\\')
{
m_strBuffer.append(strLine, nLen-1);
}
else
{
GetTokenHandler()->HandleError("Newline in constant", GetLine());
}
return NULL;
}
m_strBuffer.append(strLine, strEnd - strLine + 1);
PushToken(TOKEN_STRING, m_strBuffer.c_str());
m_bMultiLineString = false;
m_strBuffer.clear();
return strEnd + 1;
}
bool cPasTokenizer::Parse(const char* strLine, bool bSkipWhiteSpaces, bool bSkipComments)
{
if (!GetTokenHandler())
return false;
IncLine();
LogLine(strLine);
if (m_bBlockComment)
{
strLine = AppendBlockComment(strLine, bSkipComments);
}
else if (GetLine() > 1)
{
if (!bSkipWhiteSpaces)
PushToken(TOKEN_NEWLINE);
}
if (strLine == NULL)
return false;
while(char c = *strLine++)
{
tToken token;
switch(c)
{
case ' ':
case '\t':
strLine = HandleWhiteSpace(strLine-1, bSkipWhiteSpaces);
break;
case '{':
strLine = HandleBlockComment(strLine, bSkipComments);
if (strLine == NULL)
{
return true;
}
break;
case '+':
PushToken(TOKEN_OPERATOR, PAS_OP_ADDITION);
break;
case '-':
PushToken(TOKEN_OPERATOR, PAS_OP_SUBTRACTION);
break;
case '*':
PushToken(TOKEN_OPERATOR, PAS_OP_MULTIPLICATION);
break;
case '/':
switch(*strLine)
{
case '/':
PushToken(TOKEN_LINECOMMENT, strLine+1);
return true;
default:
PushToken(TOKEN_OPERATOR, PAS_OP_DIVISION);
break;
}
break;
case '=':
PushToken(TOKEN_OPERATOR, PAS_OP_EQUAL);
break;
case '>':
switch(*strLine)
{
case '=':
PushToken(TOKEN_OPERATOR, PAS_OP_BIGGER_OR_EQUAL);
strLine++;
break;
default:
PushToken(TOKEN_OPERATOR, PAS_OP_BIGGER);
break;
};
break;
case '<':
switch(*strLine)
{
case '=':
PushToken(TOKEN_OPERATOR, PAS_OP_SMALLER_OR_EQUAL);
strLine++;
break;
case '>':
PushToken(TOKEN_OPERATOR, PAS_OP_NOT_EQUAL);
strLine++;
break;
default:
PushToken(TOKEN_OPERATOR, PAS_OP_SMALLER);
break;
};
break;
case ':':
switch(*strLine)
{
case '=':
PushToken(TOKEN_OPERATOR, PAS_OP_ASSIGNMENT);
strLine++;
break;
default:
PushToken(TOKEN_OPERATOR, PAS_OP_ISTYPE);
break;
};
break;
case '^':
PushToken(TOKEN_OPERATOR, PAS_OP_DEREFERENCE);
break;
case '@':
PushToken(TOKEN_OPERATOR, PAS_OP_ADDRESS);
break;
case '&':
PushToken(TOKEN_OPERATOR, PAS_OP_AMPERSAND);
break;
case ',':
PushToken(TOKEN_OPERATOR, PAS_OP_LIST);
break;
case ';':
PushToken(TOKEN_OPERATOR, PAS_OP_COMMAND_SEPARATOR);
break;
case '#':
strLine = ParseLiteral(strLine-1, TOKEN_CHAR);
if (strLine == NULL)
return false;
break;
case '$':
strLine = ParseLiteral(strLine-1, TOKEN_LITERAL);
if (strLine == NULL)
return false;
break;
case '.':
switch(*strLine)
{
case '$':
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
strLine = ParseLiteral(strLine-1, TOKEN_LITERAL);
if (strLine == NULL)
return false;
break;
case '.':
PushToken(TOKEN_OPERATOR, PAS_OP_RANGE);
strLine++;
break;
case ')':
PushToken(TOKEN_OPERATOR, PAS_OP_INDEX_CLOSE);
strLine++;
break;
default:
PushToken(TOKEN_OPERATOR, PAS_OP_MEMBER_ACCESS);
break;
}
break;
case '\'':
strLine = HandleString(strLine-1);
if (strLine == NULL)
return true;
break;
case '(':
switch(*strLine)
{
case '.':
PushToken(TOKEN_OPERATOR, PAS_OP_INDEX_OPEN);
strLine++;
break;
case '*':
strLine = HandleBlockComment(strLine, bSkipComments);
if (strLine == NULL)
{
return true;
}
break;
default:
PushToken(TOKEN_OPERATOR, PAS_OP_BRACKET_OPEN);
break;
}
break;
case ')':
PushToken(TOKEN_OPERATOR, PAS_OP_BRACKET_CLOSE);
break;
case '[':
PushToken(TOKEN_OPERATOR, PAS_OP_INDEX_OPEN);
break;
case ']':
PushToken(TOKEN_OPERATOR, PAS_OP_INDEX_CLOSE);
break;
default:
if (isalpha(c) || c == '_')
{
strLine = ParseLabel(strLine-1);
}
else if (isdigit(c))
{
strLine = ParseLiteral(strLine-1, TOKEN_LITERAL);
}
else
{
std::stringstream strLog;
strLog << "unknown character " << c;
GetTokenHandler()->HandleError(strLog.str().c_str(), GetLine());
}
if (strLine == NULL)
return false;
break;
}
}
return false;
}
const char* cPasTokenizer::ParseLiteral(const char* strLine, int nToken)
{
const char* strCrsr = strLine;
int pos = 0;
bool bContinue = true;
bool bExponent = false;
bool bFloating = false;
bool bExpectSign = false;
bool bHex = false;
while(bContinue && *strCrsr)
{
char c = *strCrsr++;
switch(c)
{
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
break;
case '$':
if ((pos == 0) || ((pos == 1) && (*strLine == '#'))) // $ only at position 0 or position 1 if first char was #
{
bHex = true;
}
else
{
strCrsr--;
bContinue = false;
bFloating = true; // no . in hex numbers
}
break;
case '#':
if (pos == 0) // $ only at position 0
{
bFloating = true; // no . in char codes
}
else
{
strCrsr--;
bContinue = false;
}
break;
case 'e':
case 'E':
if (!bHex && !bExponent)
{
bExponent = true;
bFloating = true; // no . after e
bExpectSign = true;
pos++;
continue;
}
else
{
if (!bHex)
{
strCrsr--;
bContinue = false;
}
}
break;
case 'a':
case 'b':
case 'c':
case 'd':
case 'f':
case 'A':
case 'B':
case 'C':
case 'D':
case 'F':
if (!bHex)
{
strCrsr--;
bContinue = false;
}
break;
case '.':
if (!bFloating)
bFloating = true;
else
{
strCrsr--;
bContinue = false;
}
break;
case '-':
case '+':
if (!bExpectSign)
{
strCrsr--;
bContinue = false;
}
break;
default:
strCrsr--;
bContinue = false;
break;
}
bExpectSign = false;
pos++;
}
PushToken(nToken, strLine, strCrsr - strLine);
return strCrsr;
}
bool cPPTokenizer::Parse(const char* strLine, bool bSkipWhiteSpaces, bool bSkipComments)
{
LOG("strLine: %s", strLine);
LOG("Stop: %s", m_bStop ? "true" : "false");
bool bSlashFound = false;
char c;
if (!GetTokenHandler())
return false;
IncLine();
LogLine(strLine);
if (m_bBlockComment)
{
strLine = AppendBlockComment(strLine);
if (strLine == NULL)
return true;
}
else if (m_bLineComment)
{
HandleLineComment(strLine);
return true;
}
else if (m_bMultiLineString)
{
strLine = AppendString(strLine);
if (strLine == NULL)
return true;
}
else if (m_bMessage)
{
HandleMessage(strLine);
return true;
}
else if (GetLine() > 1)
{
if (!bSkipWhiteSpaces)
PushToken(TOKEN_NEWLINE);
}
if (m_bStop)
{
m_bStop = false;
return false;
}
bool bLineStart = true;
while(c = *strLine++)
{
tToken token;
switch(c)
{
case ' ':
case '\t':
strLine = HandleWhiteSpace(strLine-1, bLineStart);
break;
case '#':
if (m_bPreProcMode)
{
switch(*strLine)
{
case '#': PushToken(TOKEN_OPERATOR, PP_OP_CONCATENATION); strLine++; break;
default:
PushToken(TOKEN_OPERATOR, PP_OP_STRINGIFICATION);
break;
}
}
else
{
PushToken(TOKEN_OPERATOR, PP_OP_PREPROC);
m_bPreProcMode = true;
m_bExpectKeyword = true;
}
break;
// ',', '(', ')' are needed to interpret parameter lists in macro calls
case ',': PushToken(TOKEN_OPERATOR, PP_OP_COMMA); break;
case '(': PushToken(TOKEN_OPERATOR, PP_OP_BRACKET_OPEN); break;
case ')': PushToken(TOKEN_OPERATOR, PP_OP_BRACKET_CLOSE); break;
case '+': PushTokenIfPreProcMode(TOKEN_OPERATOR, PP_OP_ADDITION, "+"); break;
case '-': PushTokenIfPreProcMode(TOKEN_OPERATOR, PP_OP_SUBTRACTION, "-"); break;
case '*': PushTokenIfPreProcMode(TOKEN_OPERATOR, PP_OP_MULTIPLICATION, "*"); break;
case '%':
switch(*strLine)
{
case '>': PushToken(TOKEN_CHAR, '}'); strLine++; break;
case ':': PushToken(TOKEN_CHAR, '#'); strLine++; break;
default: PushTokenIfPreProcMode(TOKEN_OPERATOR, PP_OP_MODULUS, "%"); break;
}
break;
case '~': PushTokenIfPreProcMode(TOKEN_OPERATOR, PP_OP_BITWISE_NOT, "~"); break;
case '^': PushTokenIfPreProcMode(TOKEN_OPERATOR, PP_OP_BITWISE_XOR, "^"); break;
case '!':
switch(*strLine)
{
case '=': PushTokenIfPreProcMode(TOKEN_OPERATOR, PP_OP_NOT_EQUAL, "!="); strLine++; break;
default: PushTokenIfPreProcMode(TOKEN_OPERATOR, PP_OP_LOGICAL_NOT, "!"); break;
}
break;
case '&':
switch(*strLine)
{
case '&': PushTokenIfPreProcMode(TOKEN_OPERATOR, PP_OP_LOGICAL_AND, "&&"); strLine++; break;
default: PushTokenIfPreProcMode(TOKEN_OPERATOR, PP_OP_BITWISE_AND, "&"); break;
}
break;
case '|':
switch(*strLine)
{
case '|': PushTokenIfPreProcMode(TOKEN_OPERATOR, PP_OP_LOGICAL_OR, "||"); strLine++; break;
default: PushTokenIfPreProcMode(TOKEN_OPERATOR, PP_OP_BITWISE_OR, "|"); break;
}
break;
case '=':
switch(*strLine)
{
case '=': PushTokenIfPreProcMode(TOKEN_OPERATOR, PP_OP_EQUAL, "=="); strLine++; break;
default: PushToken(TOKEN_CHAR, '='); break;
}
break;
case '.':
if (strLine[0] == '.' && strLine[1] == '.')
{
strLine += 2;
PushTokenIfPreProcMode(TOKEN_OPERATOR, PP_OP_ELLIPSIS, "...");
}
else
{
switch(*strLine)
{
case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9':
strLine = ParseLiteral(strLine-1);
if (strLine == NULL)
return PushPreProcEnd();
break;
default:
PushToken(TOKEN_CHAR, '.');
break;
}
}
break;
case ':':
switch(*strLine)
{
case '>': PushToken(TOKEN_CHAR, ']'); strLine++; break;
default: PushTokenIfPreProcMode(TOKEN_OPERATOR, PP_OP_CONDITIONAL_ELSE, ":"); break;
}
break;
case '/':
switch(*strLine)
{
case '/':
// line comments are removed
HandleLineComment(strLine);
return PushPreProcEnd();
case '*':
strLine = HandleBlockComment(strLine-2);
if (strLine == NULL)
return PushPreProcEnd();
break;
default: PushTokenIfPreProcMode(TOKEN_OPERATOR, PP_OP_DIVISION, "/"); break;
}
break;
case '\'':
strLine = HandleString(strLine-1, '\'', TOKEN_STRING);
if (strLine == NULL)
return PushPreProcEnd();
break;
case '\"':
strLine = HandleString(strLine-1, '\"', TOKEN_STRING);
if (strLine == NULL)
return PushPreProcEnd();
break;
case '>':
switch(*strLine)
{
case '>': PushTokenIfPreProcMode(TOKEN_OPERATOR, PP_OP_SHR, ">>"); strLine++; break;
case '=': PushTokenIfPreProcMode(TOKEN_OPERATOR, PP_OP_BIGGER_OR_EQUAL, ">="); strLine++; break;
default: PushTokenIfPreProcMode(TOKEN_OPERATOR, PP_OP_BIGGER, ">"); break;
}
break;
case '<':
if (m_bInclude)
{
strLine = HandleString(strLine-1, '>', TOKEN_STRING);
m_bInclude = false;
if (strLine == NULL)
return PushPreProcEnd();
}
else
{
switch(*strLine)
{
case '<': PushTokenIfPreProcMode(TOKEN_OPERATOR, PP_OP_SHL, "<<"); strLine++; break;
case '=': PushTokenIfPreProcMode(TOKEN_OPERATOR, PP_OP_SMALLER_OR_EQUAL, "<="); strLine++; break;
case '%': PushToken(TOKEN_CHAR, '{'); strLine++; break;
case ':': PushToken(TOKEN_CHAR, '['); strLine++; break;
default: PushTokenIfPreProcMode(TOKEN_OPERATOR, PP_OP_SMALLER, "<"); break;
}
break;
}
break;
case '\\':
// check whether a line concatenation takes place
if(*strLine == '\0')
{
return true;
}
break;
case '?':
switch(*strLine)
{
case '?':
strLine++;
switch(*strLine)
{
case '<': PushToken(TOKEN_CHAR, '{'); strLine++; break;
case '>': PushToken(TOKEN_CHAR, '}'); strLine++; break;
case '(': PushToken(TOKEN_CHAR, '['); strLine++; break;
case ')': PushToken(TOKEN_CHAR, ']'); strLine++; break;
case '\'': PushToken(TOKEN_CHAR, '^'); strLine++; break;
case '!': PushToken(TOKEN_CHAR, '|'); strLine++; break;
case '-': PushToken(TOKEN_CHAR, '~'); strLine++; break;
case '=': PushToken(TOKEN_CHAR, '#'); strLine++; break;
case '/': PushToken(TOKEN_CHAR, '\\'); strLine++; break;
default: PushToken(TOKEN_TEXT, "??"); break;
}
break;
default: PushTokenIfPreProcMode(TOKEN_OPERATOR, PP_OP_CONDITIONAL, "?"); break;
}
break;
default:
if (isalpha(c) || c == '_')
{
strLine = ParseLabel(strLine-1);
if (m_bMessage)
{
HandleMessage(strLine);
m_bInclude = false;
return true;
}
}
else if (isdigit(c))
{
strLine = ParseLiteral(strLine-1);
}
else
{
PushToken(TOKEN_CHAR, c);
}
break;
}
bLineStart = false;
}
return PushPreProcEnd();
}
const char* cPPTokenizer::ParseLiteral(const char* strLine)
{
const char* strCrsr = strLine;
int pos = 0;
bool bContinue = true;
bool bExponent = false;
bool bFloating = false;
bool bExpectSign = true;
bool bHex = false;
while(bContinue && *strCrsr)
{
char c = *strCrsr++;
switch(c)
{
case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9':
break;
case 'x': case 'X':
if (pos != 1) // x only at position 1
{
bContinue = false;
bFloating = true; // no . in hex numbers
}
else
{
if (*strLine != '0')
{
strCrsr--;
bContinue = false;
}
bHex = true;
}
break;
case 'e':
case 'E':
if (!bHex && !bExponent)
{
bExponent = true;
bFloating = true; // no . after e
bExpectSign = true;
pos++; // bExpectedSign is resetted at the end of the loop!
continue;
}
else
{
if (!bHex)
{
strCrsr--;
bContinue = false;
}
}
break;
case 'a': case 'b': case 'c': case 'd': case 'f':
case 'A': case 'B': case 'C': case 'D': case 'F':
if (!bHex)
{
strCrsr--;
bContinue = false;
}
break;
case '.':
if (!bFloating)
bFloating = true;
else
{
strCrsr--;
bContinue = false;
}
break;
case '-':
case '+':
if (bExpectSign)
bExpectSign = false;
else
{
strCrsr--;
bContinue = false;
}
break;
default:
strCrsr--;
bContinue = false;
break;
}
pos++;
bExpectSign = false;
}
switch(*strCrsr)
{
case 'F': case 'f':
strCrsr++;
break;
case 'L': case 'l':
case 'U': case 'u':
strCrsr++;
switch (*strCrsr)
{
case 'l':
case 'L':
strCrsr++;
switch (*strCrsr) // c++11: ll & ull
{
case 'l':
case 'L':
strCrsr++;
break;
}
break;
}
break;
}
PushToken(TOKEN_LITERAL, strLine, strCrsr - strLine);
return strCrsr;
}
void cTokenizer::PushToken(int nToken)
{
tToken token;
token.m_Token = nToken;
PushToken(token);
}
void cTokenizer::PushKeyword(int nKeyword)
{
PushToken(TOKEN_KEYWORD, nKeyword);
}
