AstNode* Parser::parseStatement() {
if (isKeyword(currentTokenValue())) {
if (currentTokenValue() == "function") {
return parseFunction();
} else if (currentTokenValue() == "for") {
return parseFor();
} else if (currentTokenValue() == "if") {
return parseIf();
} else if (currentTokenValue() == "while") {
return parseWhile();
} else if (currentTokenValue() == "print") {
return parsePrint();
} else if (currentTokenValue() == "int") {
return parseDeclaration(VT_INT);
} else if (currentTokenValue() == "double") {
return parseDeclaration(VT_DOUBLE);
} else if (currentTokenValue() == "string") {
return parseDeclaration(VT_STRING);
} else if (currentTokenValue() == "return") {
return parseReturn();
} else {
cout << "unhandled keyword " << currentTokenValue() << endl;
assert(false);
return 0;
}
}
if (currentToken() == tLBRACE) {
return parseBlock(true);
}
if ((currentToken() == tIDENT) && isAssignment(lookaheadToken(1))) {
return parseAssignment();
}
return parseExpression();
}
bool CVarConsole::parseInput(char* input) {
if(strlen(input) == 0) return true;
if(parseAssignment(input)) return true;
if(parseCommand(input)) return true;
if(parseIdentifier(input)) return true;
return false;
}
StringMath::Token StringMath::parseExpression() {
// Expression ::= Assignment | Conditional
Token value = parseAssignment();
if(value.isEmpty())
value = parseConditional();
return value;
}
AstNode* Parser::parseDeclaration(VarType type) {
// Skip type.
ensureToken(tIDENT);
const string& var = currentTokenValue();
if (!_currentScope->declareVariable(var, type)) {
error("Variable %s already declared", var.c_str());
}
if (lookaheadToken(1) == tASSIGN) {
return parseAssignment();
}
ensureToken(tIDENT);
ensureToken(tSEMICOLON);
return 0;
}
bool CVarConsole::parseInput(const char* input_) {
if(strlen(input_) == 1 && input_[0] == '.') {
return parseInput(last_command.c_str());
} else {
last_command = input_;
}
char input[1024];
if(!luaMode) {
strcpy(input, input_);
} else {
sprintf(input, "lua %s", input_);
}
if(strlen(input) == 0) return true;
if(parseCommand(input)) return true;
if(parseAssignment(input)) return true;
if(parseIdentifier(input)) return true;
return false;
}
static PSmmAstNode parseExpressionStmt(PSmmParser parser) {
PSmmAstNode lval;
struct SmmFilePos fpos = parser->curToken->filePos;
parser->curToken->canBeNewSymbol = true;
lval = parseExpression(parser);
if (lval == &errorNode) {
if (findToken(parser, ';')) getNextToken(parser);
return &errorNode;
}
if (!lval->isIdent && (parser->curToken->kind == ':' || parser->curToken->kind == '=')) {
smmPostMessage(parser->msgs, errSmmOperandMustBeLVal, fpos);
if (findToken(parser, ';')) getNextToken(parser);
return &errorNode;
}
if (parser->curToken->kind == ':') {
lval = parseDeclaration(parser, lval);
} else if (parser->curToken->kind == '=') {
lval = parseAssignment(parser, lval);
}
if (lval) {
bool isJustIdent = lval->isIdent && (lval->kind != nkSmmCall) && (lval->kind != nkSmmError);
bool isAnyBinOpExceptLogical = lval->isBinOp && lval->kind != nkSmmAndOp && lval->kind != nkSmmOrOp;
if (isJustIdent || isAnyBinOpExceptLogical) {
smmPostMessage(parser->msgs, wrnSmmNoEffectStmt, lval->token->filePos);
if (isJustIdent) lval = NULL;
}
}
if (parser->prevToken->kind != '}' && (lval != &errorNode || parser->curToken->kind == ';')) {
expect(parser, ';');
}
return lval;
}
Node* Parser::parseLetExpr()
{
expect(Token::TYPE::LET);
nextToken();
return parseAssignment();
}
static bool
parseTestSource(const std::string &path, TestFile &tests)
{
auto file = std::ifstream { path };
std::string name, lhs, rhs;
Target target;
Value value;
TestData *data = nullptr;
for (std::string line; std::getline(file, line); ) {
line = trim(line);
if (!line.size()) {
continue;
}
// Decode test name
if (line.back() == ':') {
name = line;
name.erase(name.end() - 1);
data = &tests.tests[name];
}
// Decode input
if (line.find("# in ") == 0) {
line.erase(0, std::strlen("# in "));
if (!parseAssignment(line, lhs, rhs)) {
return false;
}
if (!decodeLHS(lhs, target)) {
return false;
}
if (!decodeValue(rhs, target, value)) {
return false;
}
data->fields[target].setInput(value);
}
// Decode output
if (line.find("# out ") == 0) {
line.erase(0, std::strlen("# out "));
if (!parseAssignment(line, lhs, rhs)) {
return false;
}
if (!decodeLHS(lhs, target)) {
return false;
}
if (!decodeValue(rhs, target, value)) {
return false;
}
data->fields[target].setOutput(value);
}
}
return true;
}
// ----------------------------------------------------------------------------
// ParseTreeNode::parse
//
// Parses formatted text data. Current valid formatting is:
// (type) child = value;
// (type) child = value1, value2, ...;
// (type) child = { value1, value2, ... }
// (type) child { grandchild = value; etc... }
// (type) child : inherited { ... }
//
// All values are read as strings, but can be retrieved as string, int, bool
// or float.
// ----------------------------------------------------------------------------
bool ParseTreeNode::parse(Tokenizer& tz)
{
// Keep parsing until final } is reached (or end of file)
string name, type;
while (!tz.atEnd() && tz.current() != '}')
{
// Check for preprocessor stuff
if (parser_ && tz.current()[0] == '#')
{
if (!parsePreprocessor(tz))
return false;
tz.advToNextLine();
continue;
}
// If it's a special character (ie not a valid name), parsing fails
if (tz.isSpecialCharacter(tz.current().text[0]))
{
logError(tz, S_FMT("Unexpected special character '%s'", CHR(tz.current().text)));
return false;
}
// So we have either a node or property name
name = tz.current().text;
type.Empty();
if (name.empty())
{
logError(tz, "Unexpected empty string");
return false;
}
// Check for type+name pair
if (tz.peek() != '=' && tz.peek() != '{' && tz.peek() != ';' && tz.peek() != ':')
{
type = name;
name = tz.next().text;
if (name.empty())
{
logError(tz, "Unexpected empty string");
return false;
}
}
//Log::debug(S_FMT("%s \"%s\", op %s", CHR(type), CHR(name), CHR(tz.current().text)));
// Assignment
if (tz.advIfNext('=', 2))
{
if (!parseAssignment(tz, addChildPTN(name, type)))
return false;
}
// Child node
else if (tz.advIfNext('{', 2))
{
// Parse child node
if (!addChildPTN(name, type)->parse(tz))
return false;
}
// Child node (with no values/children)
else if (tz.advIfNext(';', 2))
{
// Add child node
addChildPTN(name, type);
continue;
}
// Child node + inheritance
else if (tz.advIfNext(':', 2))
{
// Check for opening brace
if (tz.checkNext('{'))
{
// Add child node
auto child = addChildPTN(name, type);
child->inherit_ = tz.current().text;
// Skip {
tz.adv(2);
// Parse child node
if (!child->parse(tz))
return false;
}
else if (tz.checkNext(';')) // Empty child node
{
// Add child node
auto child = addChildPTN(name, type);
child->inherit_ = tz.current().text;
// Skip ;
tz.adv(2);
continue;
}
else
{
logError(tz, S_FMT("Expecting \"{\" or \";\", got \"%s\"", CHR(tz.next().text)));
return false;
}
}
// Unexpected token
else
{
logError(tz, S_FMT("Unexpected token \"%s\"", CHR(tz.next().text)));
return false;
}
// Continue parsing
tz.adv();
}
// Success
return true;
}
static bool parseStatement(MemoryStack* stack, LexerCarriage* carriage, Array<SyntaxError>* errors, ASTNode* result)
{
if (parseVariableDefinition(stack, carriage, errors, result)) return true;
if (parseFunctionCall(stack, carriage, errors, result)) return true;
if (parseAssignment(stack, carriage, errors, result)) return true;
if (carriage->topToken.type == TOK_IDENTIFIER)
{
Token identifier = carriage->topToken;
parseNextToken(carriage);
if (carriage->topToken.type == TOK_DOUBLE_DOT)
{
parseNextToken(carriage);
if (carriage->topToken.type == TOK_OPEN_BRACKET) // function definition
{
parseNextToken(carriage);
result->nodeType = AST_STATEMENT_FUNCTION_DEFINITION;
result->functionDefinition.lineNumber = carriage->currentLineNumber;
result->functionDefinition.identifier = identifier.identifierValue;
ASTNode arg;
LinkedList<ASTNode> args = {};
if (carriage->topToken.type != TOK_CLOSE_BRACKET)
{
while (true)
{
if (parseVariableDefinition(stack, carriage, errors, &arg))
{
*pushElement(&args, stack) = arg;
if (carriage->topToken.type != TOK_COMMA)
break;
else
parseNextToken(carriage); // pop ,
}
else
{
pushBack(errors, SyntaxError{ SET_EXPRESSION_EXPECTED });
break;
}
};
}
expectAndEat(carriage, TOK_CLOSE_BRACKET, errors);
result->functionDefinition.arguments = args;
if (carriage->topToken.type != TOK_VOID)
{
result->functionDefinition.returnType = pushMemory<ASTNode>(stack);
if (!parseTypeIdentifier(stack, carriage, errors, result->functionDefinition.returnType))
{
// TODO: change this
SyntaxError error = SyntaxError{ SET_UNEXPECTED_TOKEN };
error.unexpectedToken.token = carriage->topToken;
error.unexpectedToken.expecting = TOK_IDENTIFIER;
pushBack(errors, error);
}
}
else
{
parseNextToken(carriage);
result->functionDefinition.returnType = nullptr;
}
result->functionDefinition.body = nullptr;
if (carriage->topToken.type == TOK_ASSIGN)
{
parseNextToken(carriage);
result->functionDefinition.body = pushMemory<ASTNode>(stack);
expectAndEatStatement(stack, carriage, errors, result->functionDefinition.body);
}
return true;
}
else if (carriage->topToken.type == TOK_STRUCT)
{
parseNextToken(carriage);
expectAndEat(carriage, TOK_ASSIGN, errors);
int bracesOpenedAt = carriage->currentLineNumber;
expectAndEat(carriage, TOK_OPEN_BRACES, errors);
result->nodeType = AST_STATEMENT_TYPE_DEFINITION;
result->typeDefinition.lineNumber = carriage->currentLineNumber;
result->typeDefinition.identifier = identifier.identifierValue;
result->typeDefinition.definitions = {};
ASTNode definition;
do
{
if (parseVariableDefinition(stack, carriage, errors, &definition))
*pushElement(&result->typeDefinition.definitions, stack) = definition;
else
break;
} while (true);
if (carriage->topToken.type == TOK_CLOSE_BRACES)
parseNextToken(carriage);
else
{
SyntaxError error = { SET_UNCLOSED_BRACKETS, bracesOpenedAt };
error.unclosedBrackets.closedAt = carriage->currentLineNumber;
error.unclosedBrackets.token = carriage->topToken;
pushBack(errors, error);
}
return true;
}
else
{
// TODO: error
}
}
else
{
// TODO: error
}
}
else if (carriage->topToken.type == TOK_OPEN_BRACES)
{
int openedAt = carriage->currentLineNumber;
parseNextToken(carriage);
result->nodeType = AST_STATEMENTS_BLOCK;
result->statementsBlock.statements = { };
parseStatements(stack, carriage, errors, &result->statementsBlock.statements);
if (carriage->topToken.type == TOK_CLOSE_BRACES)
parseNextToken(carriage);
else
{
SyntaxError error = SyntaxError{ SET_UNCLOSED_BRACKETS, openedAt };
error.unclosedBrackets.closedAt = carriage->currentLineNumber;
error.unclosedBrackets.token = carriage->topToken;
pushBack(errors, error);
}
return true;
}
else if (carriage->topToken.type == TOK_IF)
{
parseNextToken(carriage);
result->nodeType = AST_STATEMENT_IF;
result->ifStatement.condition = pushMemory<ASTNode>(stack);
result->ifStatement.ifCase = pushMemory<ASTNode>(stack);
expectAndEatExpression(stack, carriage, errors, result->ifStatement.condition);
expectAndEatStatement(stack, carriage, errors, result->ifStatement.ifCase);
if (carriage->topToken.type == TOK_ELSE)
{
parseNextToken(carriage);
result->ifStatement.elseCase = pushMemory<ASTNode>(stack);
expectAndEatStatement(stack, carriage, errors, result->ifStatement.elseCase);
}
else
result->ifStatement.elseCase = nullptr;
return true;
}
else if (carriage->topToken.type == TOK_WHILE)
{
parseNextToken(carriage);
result->nodeType = AST_STATEMENT_WHILE;
result->whileStatement.condition = pushMemory<ASTNode>(stack);
result->whileStatement.body = pushMemory<ASTNode>(stack);
expectAndEatExpression(stack, carriage, errors, result->whileStatement.condition);
expectAndEatStatement(stack, carriage, errors, result->whileStatement.body);
return true;
}
return false;
}
// -----------------------------------------------------------------------------
// Parses an assignment operation at [tz]'s current token to [child]
// -----------------------------------------------------------------------------
bool ParseTreeNode::parseAssignment(Tokenizer& tz, ParseTreeNode* child) const
{
// Check type of assignment list
char list_end = ';';
if (tz.current() == '{' && !tz.current().quoted_string)
{
list_end = '}';
tz.adv();
}
// Parse until ; or }
while (true)
{
auto& token = tz.current();
// Check for list end
if (token == list_end && !token.quoted_string)
break;
// Setup value
Property value;
// Detect value type
if (token.quoted_string) // Quoted string
value = token.text;
else if (token == "true") // Boolean (true)
value = true;
else if (token == "false") // Boolean (false)
value = false;
else if (token.isInteger()) // Integer
value = token.asInt();
else if (token.isHex()) // Hex (0xXXXXXX)
value = token.asInt();
else if (token.isFloat()) // Floating point
value = token.asFloat();
else // Unknown, just treat as string
value = token.text;
// Add value
child->values_.push_back(value);
// Check for ,
if (tz.peek() == ',')
tz.adv(); // Skip it
else if (tz.peek() != list_end)
{
logError(tz, fmt::format(R"(Expected "," or "{}", got "{}")", list_end, tz.peek().text));
return false;
}
tz.adv();
}
return true;
}
// -----------------------------------------------------------------------------
// Parses formatted text data. Current valid formatting is:
// (type) child = value;
// (type) child = value1, value2, ...;
// (type) child = { value1, value2, ... }
// (type) child { grandchild = value; etc... }
// (type) child : inherited { ... }
//
// All values are read as strings, but can be retrieved as string, int, bool
// or float.
// -----------------------------------------------------------------------------
bool ParseTreeNode::parse(Tokenizer& tz)
{
// Keep parsing until final } is reached (or end of file)
string name, type;
while (!tz.atEnd() && tz.current() != '}')
{
// Check for preprocessor stuff
if (parser_ && tz.current()[0] == '#')
{
if (!parsePreprocessor(tz))
return false;
tz.advToNextLine();
continue;
}
// If it's a special character (ie not a valid name), parsing fails
if (tz.isSpecialCharacter(tz.current().text[0]))
{
logError(tz, fmt::format("Unexpected special character '{}'", tz.current().text));
return false;
}
// So we have either a node or property name
name = tz.current().text;
type.clear();
if (name.empty())
{
logError(tz, "Unexpected empty string");
return false;
}
// Check for type+name pair
if (tz.peek() != '=' && tz.peek() != '{' && tz.peek() != ';' && tz.peek() != ':')
{
type = name;
name = tz.next().text;
if (name.empty())
{
logError(tz, "Unexpected empty string");
return false;
}
}
// Log::debug(wxString::Format("%s \"%s\", op %s", CHR(type), CHR(name), CHR(tz.current().text)));
// Assignment
if (tz.advIfNext('=', 2))
{
if (!parseAssignment(tz, addChildPTN(name, type)))
return false;
}
// Child node
else if (tz.advIfNext('{', 2))
{
// Parse child node
if (!addChildPTN(name, type)->parse(tz))
return false;
}
// Child node (with no values/children)
else if (tz.advIfNext(';', 2))
{
// Add child node
addChildPTN(name, type);
continue;
}
// Child node + inheritance
else if (tz.advIfNext(':', 2))
{
// Check for opening brace
if (tz.checkNext('{'))
{
// Add child node
auto child = addChildPTN(name, type);
child->inherit_ = tz.current().text;
// Skip {
tz.adv(2);
// Parse child node
if (!child->parse(tz))
return false;
}
else if (tz.checkNext(';')) // Empty child node
{
// Add child node
auto child = addChildPTN(name, type);
child->inherit_ = tz.current().text;
// Skip ;
tz.adv(2);
continue;
}
else
{
logError(tz, fmt::format(R"(Expecting "{{" or ";", got "{}")", tz.next().text));
return false;
}
}
// Unexpected token
else
{
logError(tz, fmt::format("Unexpected token \"{}\"", tz.next().text));
return false;
}
// Continue parsing
tz.adv();
}
// Success
return true;
}
static PSmmAstNode parseDeclaration(PSmmParser parser, PSmmAstNode lval) {
PSmmToken declToken = parser->curToken;
assert(parser->curToken->kind == ':');
getNextToken(parser);
if (parser->curToken->kind == tkSmmIdent) {
//Type is given in declaration so use it
lval->type = parseType(parser);
} else if (parser->curToken->kind == ';') {
// In case of a statement like 'a :;'
smmPostMessage(parser->msgs, errSmmGotUnexpectedToken, parser->curToken->filePos, "type", "';'");
lval->type = &builtInTypes[tiSmmUnknown];
}
PSmmAstDeclNode spareNode = NULL;
PSmmAstNode expr = NULL;
if (parser->curToken->kind == '=') {
expr = parseAssignment(parser, lval);
} else if (parser->curToken->kind == ':') {
lval->kind = nkSmmConst;
lval->isConst = true;
expr = parseAssignment(parser, lval);
if (expr->kind == nkSmmParamDefinition) {
expr->kind = nkSmmParam;
if (parser->curScope->level > 0) {
smmPostMessage(parser->msgs, errSmmFuncUnderScope, lval->token->filePos, lval->token->repr);
}
lval->kind = nkSmmFunc;
lval->asFunc.params = (PSmmAstParamNode)expr;
if (lval->asFunc.params->count == 0) {
spareNode = &expr->asDecl;
spareNode->kind = nkSmmDecl;
lval->asFunc.params = NULL;
}
lval = parseFunction(parser, &lval->asFunc);
if (parser->curScope->level > 0) {
lval = &errorNode;
}
expr = lval;
}
} else if (parser->curToken->kind != ';') {
expr = &errorNode;
if (parser->curToken->kind != tkSmmErr) {
char gotBuf[4];
const char* got = smmTokenToString(parser->curToken, gotBuf);
smmPostMessage(parser->msgs, errSmmGotUnexpectedToken, parser->curToken->filePos, "':', '=' or type", got);
}
findToken(parser, ';');
}
if (lval == &errorNode) {
return &errorNode;
}
PSmmAstNode existing = ibsDictGet(parser->idents, lval->token->repr);
if (existing && existing->asIdent.level == lval->asIdent.level && lval->kind != nkSmmFunc) {
assert(existing->kind == nkSmmFunc);
smmPostMessage(parser->msgs, errSmmRedefinition, lval->token->filePos, lval->token->repr);
return &errorNode;
}
ibsDictPush(parser->idents, lval->token->repr, lval);
PSmmAstDeclNode decl = spareNode ? spareNode : smmNewAstNode(nkSmmDecl, parser->a);
decl->token = declToken;
if (expr == &errorNode) expr = NULL;
parser->curScope->lastDecl->nextDecl = decl;
parser->curScope->lastDecl = decl;
if (lval->kind == nkSmmFunc) {
decl->left = lval;
return NULL;
}
if (!expr) {
expr = smmNewAstNode(nkSmmAssignment, parser->a);
expr->left = lval;
expr->right = smmGetZeroValNode(parser->curToken->filePos, lval->type, parser->a);
expr->type = lval->type;
expr->token = ibsAlloc(parser->a, sizeof(struct SmmToken));
if (lval->isConst) {
expr->token->repr = ":";
} else {
expr->token->repr = "=";
}
expr->token->kind = expr->token->repr[0];
expr->token->filePos = parser->curToken->filePos;
}
decl->left = expr;
if (lval->isConst) return NULL;
return (PSmmAstNode)decl;
}