void EBNFScanner::nextToken()
{
if(currentToken > -1 && currentToken < tokens.size() - 1)
{
currentToken++;
return;
}
while (hasNextCharacter())
{
nextCharacter();
// Scan the Identifier type. [a-zA-Z\-_]
if(TokenScannerHelper::isAlpha(currentCharacter))
{
Token tok = scanIdentifier();
tokens.append(std::move(tok));
currentToken++;
return;
}
// Scan a literal
else if (TokenScannerHelper::isQuote(currentCharacter))
{
Token tok = scanLiteral();
tokens.append(std::move(tok));
currentToken++;
return;
}
// Scan an operator Token.
else if (TokenScannerHelper::isOperator(currentCharacter))
{
if (currentCharacter == '(' && peekCharacter() == '*') {
Token tok = scanComment();
tokens.append(std::move(tok));
currentToken++;
return;
}
else {
Token tok = scanOperator();
tokens.append(std::move(tok));
currentToken++;
return;
}
}
else if (TokenScannerHelper::isWhiteSpace(currentCharacter))
{
// Ignore white spaces.
}
else
{
raiseError (String("Invalid Character."));
}
checkNewLine();
}
tokens.append(Token(Token::TypeEof, String(""), createPosition()));
currentToken++;
}
/*!
* \brief Get the next token in the stream
*/
HllTokenizer::Token HllTokenizer::getNext()
{
Token next;
// Start out by moving past any whitespace
skipCharacters(whitespace);
// Check if we've reached the end of the file
if(!fillBuffer(1)) {
next = createToken(Token::TypeEnd, "");
return next;
}
// Scan through the list of literals and see if any match
if(scanLiteral(literals, next)) {
return next;
}
// If no literals matched, see if an identifier can be constructed
if(std::isalpha(buffer()[0]) || buffer()[0] == '_') {
size_t len = 0;
while(std::isalpha(buffer()[len]) || buffer()[len] == '_') {
len++;
if(!fillBuffer(len + 1)) {
break;
}
}
TokenType type = TypeIdentifier;
std::string string = buffer().substr(0, len);
// Check if the string is a keyword
for(std::string &keyword : keywords) {
if(string == keyword) {
type = TypeLiteral;
break;
}
}
// Construct a token out of the characters found
next = createToken(type, string);
emptyBuffer(len);
return next;
}
// If an identifier couldn't be found, check for a number
if(std::isdigit(buffer()[0])) {
size_t len = 0;
while(std::isdigit(buffer()[len])) {
len++;
if(!fillBuffer(len + 1)) {
break;
}
}
// Construct a token out of the characters found
next = createToken(TypeNumber, buffer().substr(0, len));
emptyBuffer(len);
return next;
}
if(buffer()[0] == '\"') {
size_t len = 1;
while(true) {
if(!fillBuffer(len + 1)) {
setError("Unterminated string literal");
return next;
}
if(buffer()[len] == '\"') {
break;
}
len++;
}
// Construct a token out of the characters found
std::string text = buffer().substr(1, len - 1);
emptyBuffer(len + 1);
if(evaluateEscapes(text)) {
next = createToken(TypeString, text);
}
return next;
}
if(buffer()[0] == '\'') {
size_t len = 1;
while(true) {
if(!fillBuffer(len + 1)) {
setError("Unterminated character literal");
return next;
}
if(buffer()[len] == '\'') {
break;
}
len++;
}
// Construct a token out of the characters found
std::string text = buffer().substr(1, len - 1);
emptyBuffer(len + 1);
if(evaluateEscapes(text)) {
if(text.size() == 1) {
next = createToken(TypeChar, text);
} else {
setError("Invalid character literal");
}
}
return next;
}
// Nothing matched, log an error
std::stringstream ss;
ss << "Illegal symbol '" << buffer()[0] << "'";
setError(ss.str());
return next;
}