CToken* CTokenizer::HandleDecimal(bool thesign)
{
byte theByte;
if (!NextChar(theByte))
{
if (thesign)
{
PutBackChar('.');
return HandleSymbol('-');
}
HandleSymbol('.');
}
PutBackChar(theByte);
if ((theByte <= '9') && (theByte >= '0'))
{
return HandleFloat(thesign);
}
if (thesign)
{
PutBackChar('.');
theByte = '-';
}
else
{
theByte = '.';
}
return HandleSymbol(theByte);
}
CToken* CTokenizer::HandleQuote()
{
byte theByte;
if (!NextChar(theByte))
{
Error(TKERR_EXPECTED_CHAR);
return NULL;
}
if (theByte == '\\')
{
theByte = Escapement();
}
byte dummy;
if (!NextChar(dummy))
{
Error(TKERR_EXPECTED_CHAR);
return NULL;
}
if (dummy != '\'')
{
PutBackChar(dummy);
PutBackChar(theByte);
Error(TKERR_EXPECTED_CHAR);
return NULL;
}
return CCharToken::Create(theByte);
}
CToken* CTokenizer::HandleNumeric(byte theByte)
{
bool thesign = theByte == '-';
if (thesign)
{
if (!NextChar(theByte))
{
return HandleSymbol('-');
}
if (theByte == '.')
{
return HandleDecimal(thesign);
}
if ((theByte < '0') || (theByte > '9'))
{
PutBackChar(theByte);
return HandleSymbol('-');
}
}
if (theByte == '0')
{
return HandleOctal(thesign);
}
long value = 0;
bool digithit = false;
while((theByte >= '0') && (theByte <= '9'))
{
value = (value * 10) + (theByte - '0');
if (!NextChar(theByte))
{
if (thesign)
{
value = -value;
}
return CIntToken::Create(value);
}
digithit = true;
}
if (theByte == '.')
{
if (digithit)
{
return HandleFloat(thesign, value);
}
if (thesign)
{
PutBackChar(theByte);
theByte = '-';
}
return HandleSymbol(theByte);
}
PutBackChar(theByte);
if (thesign)
{
value = -value;
}
return CIntToken::Create(value);
}
CToken* CTokenizer::HandleIdentifier(byte theByte)
{
char theString[MAX_IDENTIFIER_LENGTH];
theString[0] = theByte;
for (int i = 1; i < MAX_IDENTIFIER_LENGTH; i++)
{
if (NextChar((byte&)theString[i]))
{
if (((theString[i] != '_') || ((m_flags & TKF_NOUNDERSCOREINIDENTIFIER) == 0)) &&
((theString[i] != '-') || ((m_flags & TKF_NODASHINIDENTIFIER) == 0)))
{
if (((theString[i] >= 'A') && (theString[i] <= 'Z'))
|| ((theString[i] >= 'a') && (theString[i] <= 'z'))
|| ((theString[i] >= '0') && (theString[i] <= '9'))
|| (theString[i] == '_') || (theString[i] == '-'))
{
continue;
}
}
PutBackChar(theString[i]);
}
theString[i] = '\0';
return TokenFromName(theString);
}
Error(TKERR_IDENTIFIERLENGTHEXCEEDED);
return NULL;
}
void
ConfigManager::FinishTag(std::istream& in)
{
char c = '\0';
for(int i = 0; !in.eof() && ((c = ReadChar(in)) != '>'); ++i);
if(c != '>')
PutBackChar(c);
}
CToken* CTokenizer::HandleHex(bool thesign)
{
int value = 0;
while (true)
{
byte theByte;
if (!NextChar(theByte))
{
if (thesign)
{
value = -value;
}
return CIntToken::Create(value);
}
switch (theByte)
{
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
theByte = theByte - '0';
break;
case 'A':
case 'B':
case 'C':
case 'D':
case 'E':
case 'F':
theByte = theByte - 'A' + 10;
break;
case 'a':
case 'b':
case 'c':
case 'd':
case 'e':
case 'f':
theByte = theByte - 'a' + 10;
break;
default:
PutBackChar(theByte);
if (thesign)
{
value = -value;
}
return CIntToken::Create(value);
}
value = (value * 16) + theByte;
}
}
CToken* CTokenizer::HandleOctal(bool thesign)
{
byte theByte;
int value = 0;
if (!NextChar(theByte))
{
return CIntToken::Create(value);
}
if (theByte == '.')
{
return HandleDecimal(thesign);
}
if ((theByte == 'x') || (theByte == 'X'))
{
return HandleHex(thesign);
}
while(true)
{
if((theByte >= '0') && (theByte <='7'))
{
value = (value * 8) + (theByte - '0');
}
else
{
PutBackChar(theByte);
if (thesign)
{
value = -value;
}
return CIntToken::Create(value);
}
if (!NextChar(theByte))
{
if (thesign)
{
value = -value;
}
return CIntToken::Create(value);
}
}
}
CToken* CTokenizer::HandleFloat(bool thesign, long value)
{
float lower = 1.0;
float newValue = (float)value;
byte theByte;
while(NextChar(theByte))
{
if ((theByte >= '0') && (theByte <= '9'))
{
lower = lower / 10;
newValue = newValue + ((theByte - '0') * lower);
continue;
}
PutBackChar(theByte);
break;
}
if (thesign)
{
newValue = -newValue;
}
return CFloatToken::Create(newValue);
}
byte CTokenizer::Escapement()
{
byte theByte;
if (NextChar(theByte))
{
switch(theByte)
{
case 'n':
return '\n';
case '\\':
return '\\';
case '\'':
return '\'';
case '?':
return '\?';
case 'a':
return '\a';
case 'b':
return '\b';
case 'f':
return '\f';
case 'r':
return '\r';
case 't':
return '\t';
case 'v':
return '\v';
case '0':
return '\0';
// support for octal or hex sequences?? \000 or \xhhh
default:
PutBackChar(theByte);
}
}
return '\\';
}
CToken* CTokenizer::HandleSymbol(byte theByte)
{
char symbolString[128];
int curStrLen = 0;
symbolString[0] = '\0';
bool consumed = false;
CSymbolLookup* curLookup;
if ((m_flags & TKF_RAWSYMBOLSONLY) == 0)
{
curLookup = m_symbolLookup;
}
else
{
curLookup = NULL;
}
CSymbolLookup* lastLookup = NULL;
while(curLookup != NULL)
{
if (curLookup->GetByte() == theByte)
{
symbolString[curStrLen++] = theByte;
symbolString[curStrLen] = '\0';
lastLookup = curLookup;
consumed = true;
if (curLookup->GetChild() == NULL)
{
break;
}
if (!NextChar(theByte))
{
PutBackToken(CToken::Create());
break;
}
consumed = false;
curLookup = curLookup->GetChild();
continue;
}
curLookup = curLookup->GetNext();
}
if ((!consumed) && (lastLookup != NULL))
{
PutBackChar(theByte);
}
while ((lastLookup != NULL) && (lastLookup->GetValue() == -1))
{
curStrLen--;
symbolString[curStrLen] = '\0';
// symbolString = symbolString.Left(symbolString.GetLength() - 1);
if (lastLookup->GetParent() == NULL)
{
if ((m_flags & TKF_WANTUNDEFINED) == 0)
{
Error(TKERR_UNRECOGNIZEDSYMBOL);
}
}
else
{
PutBackChar(lastLookup->GetByte());
}
lastLookup = lastLookup->GetParent();
}
if (lastLookup == NULL)
{
if ((m_flags & TKF_WANTUNDEFINED) == 0)
{
return NULL;
}
curStrLen = 0;
symbolString[curStrLen++] = char(theByte);
symbolString[curStrLen] = '\0';
if ((m_flags & TKF_WIDEUNDEFINEDSYMBOLS) != 0)
{
while (true)
{
if (!NextChar(theByte))
{
PutBackToken(CToken::Create());
break;
}
if (theByte == ' ')
{
break;
}
if (((theByte >= 'a') && (theByte <= 'z')) || ((theByte >= 'A') && (theByte <= 'Z')) ||
((theByte >= '0') && (theByte <= '9')))
{
PutBackChar(theByte);
break;
}
if (theByte < ' ')
{
if ((theByte == '\n') && ((TKF_USES_EOL & m_flags) != 0))
{
PutBackToken(CUserToken::Create(TK_EOL, "-EOLN-"));
break;
}
continue;
}
symbolString[curStrLen++] = theByte;
symbolString[curStrLen] = '\0';
}
}
return CUndefinedToken::Create(symbolString);
}
return CUserToken::Create(lastLookup->GetValue(), symbolString);
}
CToken* CTokenizer::HandleSlash()
{
byte theByte;
if (!NextChar(theByte))
{
return NULL;
}
if (theByte == '/')
{
if (m_flags & TKF_COMMENTTOKENS)
{
return GetToEndOfLine(TK_COMMENT);
}
SkipToLineEnd();
return NULL;
}
if (theByte == '*')
{
if (m_flags & TKF_COMMENTTOKENS)
{
char theString[MAX_IDENTIFIER_LENGTH + 1];
theString[0] = ' ';
while (theString[0] == ' ')
{
if (!NextChar((byte&)theString[0]))
{
return NULL;
}
}
for (int i = 1; i < MAX_IDENTIFIER_LENGTH; i++)
{
if (NextChar((byte&)theString[i]))
{
if (theString[i] != '*')
{
continue;
}
i++;
if (NextChar((byte&)theString[i]))
{
if (theString[i] == '/')
{
i--;
theString[i] = '\0';
return CCommentToken::Create(theString);
}
}
}
}
Error(TKERR_IDENTIFIERLENGTHEXCEEDED);
return NULL;
}
while(NextChar(theByte))
{
while(theByte == '*')
{
if (!NextChar(theByte))
{
break;
}
if (theByte == '/')
{
return NULL;
}
}
}
return NULL;
}
PutBackChar(theByte);
return HandleSymbol('/');
}
CToken* CTokenizer::GetToEndOfLine(int tokenType)
{
// update, if you just want the whole line returned as a string, then allow a much bigger size than
// the default string size of only 128 chars...
//
if (tokenType == TK_STRING)
{
#define iRETURN_STRING_SIZE 2048
char theString[iRETURN_STRING_SIZE];
theString[0] = ' ';
for (int i = 1; i < iRETURN_STRING_SIZE; i++)
{
if (NextChar((byte&)theString[i]))
{
if (theString[i] != '\n')
{
continue;
}
PutBackChar(theString[i]);
}
theString[i] = '\0';
return CStringToken::Create(theString);
}
// line would maks a string too big to fit in buffer...
//
Error(TKERR_STRINGLENGTHEXCEEDED);
}
else
{
char theString[MAX_IDENTIFIER_LENGTH];
theString[0] = ' ';
while (theString[0] == ' ')
{
if (!NextChar((byte&)theString[0]))
{
return NULL;
}
}
for (int i = 1; i < MAX_IDENTIFIER_LENGTH; i++)
{
if (NextChar((byte&)theString[i]))
{
if (theString[i] != '\n')
{
continue;
}
PutBackChar(theString[i]);
}
theString[i] = '\0';
switch(tokenType)
{
case TK_COMMENT:
return CCommentToken::Create(theString);
case TK_IDENTIFIER:
default:
return CIdentifierToken::Create(theString);
}
}
Error(TKERR_IDENTIFIERLENGTHEXCEEDED);
}
return NULL;
}
