bool PHPSourceFile::ReadVariableInitialization(PHPEntityBase::Ptr_t var) { phpLexerToken token; if(!NextToken(token)) { return false; } if(token.type != '=') { // restore the token UngetToken(token); return false; } wxString expr; if(!ReadExpression(expr)) { return false; // EOF } // Optimize 'new ClassName(..)' expression if(expr.StartsWith("new")) { expr = expr.Mid(3); expr.Trim().Trim(false); expr = expr.BeforeFirst('('); expr.Trim().Trim(false); var->Cast<PHPEntityVariable>()->SetTypeHint(MakeIdentifierAbsolute(expr)); } else { // keep the expression var->Cast<PHPEntityVariable>()->SetExpressionHint(expr); } return true; }
/// <summary>Reads a literal or sub-expression</summary> /// <param name="pos">Position of literal or first token of sub-expression</param> /// <returns>Expression tree</returns> /// <exception cref="Logic::AlgorithmException">Attempted to read incorrect type of Token</exception> /// <exception cref="Logic::ExpressionParserException">Syntax error</exception> /// <remarks>Advances the iterator to beyond the end of the literal or sub-expression</remarks> ExpressionParser::ExpressionTree ExpressionParser::ReadValue(TokenIterator& pos) { ExpressionTree expr = nullptr; // Rule: Value = Literal / '(' Expression ')' // Match: Literal if (MatchLiteral(pos)) return ExpressionTree( new LiteralValue(ReadLiteral(pos)) ); // Read: Bracket [nothrow] if (!MatchOperator(pos, L"(")) { // Failed: Unexpected EOF if (pos >= InputEnd) throw ExpressionParserException(HERE, --pos, L"Missing operand"); // Failed: Unexpected token throw ExpressionParserException(HERE, pos, VString(L"Unexpected '%s'", pos->Text.c_str())); } // Read: Expression (may throw) const ScriptToken& open = ReadOperator(pos); expr = ReadExpression(pos); // Adv. then match // Read: Bracket [nothrow] if (MatchOperator(pos, L")")) return ExpressionTree( new BracketedExpression(open, expr, ReadOperator(pos)) ); // Failure: Missing closing bracket if (pos >= InputEnd) throw ExpressionParserException(HERE, --pos, L"Missing closing bracket"); else throw ExpressionParserException(HERE, pos, L"Expected closing bracket"); }
void ParsedObject::GetOtherSide(LispInt aNrArgsToCombine, LispInt depth) { const LispString* theOperator = iLookAhead; MatchToken(iLookAhead); ReadExpression(depth); InsertAtom(theOperator); Combine(aNrArgsToCombine); }
void ParsedObject::Parse() { ReadToken(); if (iEndOfFile) { iResult = (iParser.iEnvironment.iEndOfFile->Copy()); return; } ReadExpression(KMaxPrecedence); // least precedence if (iLookAhead != iParser.iEnvironment.iEndStatement->String()) Fail(); }
ExpressionSptr Parser::Next() { assert(_lexer); if (_lexer->Next()) { return ReadExpression(); } auto lx = _lexer->GetLexeme(); if (lx) { AddLexerError(lx); } return ExpressionSptr(); }
/// <summary>Parses the expression, ensures it is correct and produces infix/postfix tokens.</summary> /// <exception cref="Logic::AlgorithmException">Error in parsing algorithm</exception> /// <exception cref="Logic::ExpressionParserException">Syntax error in expression</exception> void ExpressionParser::Parse(TokenIterator& start) { TokenIterator pos = InputBegin; // Produce parse tree ExpressionTree tree = ReadExpression(pos); // Ensure all tokens parsed if (pos != InputEnd) throw ExpressionParserException(HERE, pos, L"Unexpected token"); // Extract tokens tree->getTokenArray(Traversal::InOrder, InfixParams); tree->getTokenArray(Traversal::PostOrder, PostfixParams); #ifdef PRINT_DEBUG Console << L"Output: " << tree->debugPrint() << ENDL; //Console << L"Infix: " << tree->debugPrintTraversal(Traversal::InOrder) << ENDL; //Console << L"Postfix: " << tree->debugPrintTraversal(Traversal::PostOrder) << ENDL; #endif }
ExpressionSptr Parser::ReadList() { assert(_lexer); auto list = Expression::List(); for (;;) { if (!_lexer->Next()) { auto lx = _lexer->GetLexeme(); if (lx) { AddLexerError(lx); return ExpressionSptr(); } AddParserError(lx, ParserErrorCode::UnexpectedEof); return ExpressionSptr(); } auto lx = _lexer->GetLexeme(); if (lx->Type == LexemeType::RightBrace) { return list; } auto expr = ReadExpression(); if (!expr) { return ExpressionSptr(); } list->ListPushBack(expr); } }
void PHPSourceFile::OnVariable(const phpLexerToken& tok) { phpLexerToken token; // Read until we find the ';' std::vector<phpLexerToken> tokens; PHPEntityBase::Ptr_t var(new PHPEntityVariable()); var->SetFullName(tok.text); var->SetFilename(m_filename.GetFullPath()); var->SetLine(tok.lineNumber); if(!CurrentScope()->FindChild(var->GetFullName(), true)) { CurrentScope()->AddChild(var); } if(!NextToken(token)) return; if(token.type != '=') { m_lookBackTokens.clear(); return; } wxString expr; if(!ReadExpression(expr)) return; // EOF // Optimize 'new ClassName(..)' expression if(expr.StartsWith("new")) { expr = expr.Mid(3); expr.Trim().Trim(false); expr = expr.BeforeFirst('('); expr.Trim().Trim(false); var->Cast<PHPEntityVariable>()->SetTypeHint(MakeIdentifierAbsolute(expr)); } else { // keep the expression var->Cast<PHPEntityVariable>()->SetExpressionHint(expr); } }
void ParsedObject::ReadAtom() { LispOperators::const_iterator opi = iParser.iPrefixOperators.find(iLookAhead); if (opi != iParser.iPrefixOperators.end()) { const LispString* theOperator = iLookAhead; MatchToken(iLookAhead); { ReadExpression(opi->second.iPrecedence); InsertAtom(theOperator); Combine(1); } } // Else parse brackets else if (iLookAhead == iParser.iEnvironment.iBracketOpen->String()) { MatchToken(iLookAhead); ReadExpression(KMaxPrecedence); // least precedence MatchToken(iParser.iEnvironment.iBracketClose->String()); } //Parse lists else if (iLookAhead == iParser.iEnvironment.iListOpen->String()) { LispInt nrargs = 0; MatchToken(iLookAhead); while (iLookAhead != iParser.iEnvironment.iListClose->String()) { ReadExpression(KMaxPrecedence); // least precedence nrargs++; if (iLookAhead == iParser.iEnvironment.iComma->String()) { MatchToken(iLookAhead); } else if (iLookAhead != iParser.iEnvironment.iListClose->String()) { throw LispErrGeneric(std::string("Expecting a } close bracket for program block, but got ") + *iLookAhead + std::string(" instead")); } } MatchToken(iLookAhead); const LispString* theOperator = iParser.iEnvironment.iList->String(); InsertAtom(theOperator); Combine(nrargs); } // Parse prog bodies else if (iLookAhead == iParser.iEnvironment.iProgOpen->String()) { LispInt nrargs = 0; MatchToken(iLookAhead); while (iLookAhead != iParser.iEnvironment.iProgClose->String()) { ReadExpression(KMaxPrecedence); // least precedence nrargs++; if (iLookAhead == iParser.iEnvironment.iEndStatement->String()) { MatchToken(iLookAhead); } else { throw LispErrGeneric(std::string("Expecting ; end of statement in program block, but got ") + *iLookAhead + std::string(" instead")); } } MatchToken(iLookAhead); const LispString* theOperator = iParser.iEnvironment.iProg->String(); InsertAtom(theOperator); Combine(nrargs); } // Else we have an atom. else { const LispString* theOperator = iLookAhead; MatchToken(iLookAhead); LispInt nrargs = -1; if (iLookAhead == iParser.iEnvironment.iBracketOpen->String()) { nrargs = 0; MatchToken(iLookAhead); while (iLookAhead != iParser.iEnvironment.iBracketClose->String()) { ReadExpression(KMaxPrecedence); // least precedence nrargs++; if (iLookAhead == iParser.iEnvironment.iComma->String()) { MatchToken(iLookAhead); } else if (iLookAhead != iParser.iEnvironment.iBracketClose->String()) { throw LispErrGeneric(std::string("Expecting a ) closing bracket for sub-expression, but got ") + *iLookAhead + std::string(" instead")); } } MatchToken(iLookAhead); opi = iParser.iBodiedOperators.find(theOperator); if (opi != iParser.iBodiedOperators.end()) { ReadExpression(opi->second.iPrecedence); // KMaxPrecedence nrargs++; } } InsertAtom(theOperator); if (nrargs >= 0) Combine(nrargs); } // Parse postfix operators while (iParser.iPostfixOperators.find(iLookAhead) != iParser.iPostfixOperators.end()) { InsertAtom(iLookAhead); MatchToken(iLookAhead); Combine(1); } }
void ParsedObject::ReadExpression(LispInt depth) { ReadAtom(); for (;;) { //Handle special case: a[b]. a is matched with lowest precedence!! if (iLookAhead == iParser.iEnvironment.iProgOpen->String()) { // Match opening bracket MatchToken(iLookAhead); // Read "index" argument ReadExpression(KMaxPrecedence); // Match closing bracket if (iLookAhead != iParser.iEnvironment.iProgClose->String()) throw LispErrGeneric(std::string("Expecting a ] close bracket for program block, but got ") + *iLookAhead + std::string(" instead")); MatchToken(iLookAhead); // Build into Ntn(...) const LispString* theOperator = iParser.iEnvironment.iNth->String(); InsertAtom(theOperator); Combine(2); } else { LispOperators::const_iterator opi = iParser.iInfixOperators.find(iLookAhead); if (opi == iParser.iInfixOperators.end()) { if (!IsSymbolic((*iLookAhead)[0])) return; const std::size_t origlen = iLookAhead->size(); std::size_t len = origlen; while (len > 1) { len -= 1; const LispString* lookUp = iParser.iEnvironment.HashTable().LookUp(iLookAhead->substr(0, len)); opi = iParser.iInfixOperators.find(lookUp); if (opi != iParser.iInfixOperators.end()) { const LispString* lookUpRight = iParser.iEnvironment.HashTable().LookUp(iLookAhead->substr(len, origlen - len)); if (iParser.iPrefixOperators.find(lookUpRight) != iParser.iPrefixOperators.end()) { iLookAhead = lookUp; LispInput& input = iParser.iInput; LispInt newPos = input.Position() - (origlen - len); input.SetPosition(newPos); break; } opi = iParser.iInfixOperators.end(); } } if (opi == iParser.iInfixOperators.end()) return; } if (depth < opi->second.iPrecedence) return; LispInt upper = opi->second.iPrecedence; if (!opi->second.iRightAssociative) upper--; GetOtherSide(2, upper); } } }
void PHPSourceFile::Parse(int exitDepth) { int retDepth = exitDepth; phpLexerToken token; while(NextToken(token)) { switch(token.type) { case '=': m_lookBackTokens.clear(); break; case '{': m_lookBackTokens.clear(); break; case '}': m_lookBackTokens.clear(); if(m_depth == retDepth) { return; } break; case ';': m_lookBackTokens.clear(); break; case kPHP_T_VARIABLE: if(!CurrentScope()->Is(kEntityTypeClass)) { // A global variable OnVariable(token); } break; case kPHP_T_CATCH: // found 'catch (...)' OnCatch(); break; case kPHP_T_PUBLIC: case kPHP_T_PRIVATE: case kPHP_T_PROTECTED: { int visibility = token.type; PHPEntityClass* cls = CurrentScope()->Cast<PHPEntityClass>(); if(cls) { /// keep the current token m_lookBackTokens.push_back(token); // Now we have a small problem here: // public can be a start for a member or a function // we let the lexer run forward until it finds kPHP_T_VARIABLE (for variable) // or kPHP_T_IDENTIFIER int what = ReadUntilFoundOneOf(kPHP_T_VARIABLE, kPHP_T_FUNCTION, token); if(what == kPHP_T_VARIABLE) { // A variable PHPEntityBase::Ptr_t member(new PHPEntityVariable()); member->SetFilename(m_filename.GetFullPath()); member->Cast<PHPEntityVariable>()->SetVisibility(visibility); member->Cast<PHPEntityVariable>()->SetFullName(token.text); size_t flags = LookBackForVariablesFlags(); member->Cast<PHPEntityVariable>()->SetFlag(kVar_Member); member->Cast<PHPEntityVariable>()->SetFlag(kVar_Const, flags & kVar_Const); member->Cast<PHPEntityVariable>()->SetFlag(kVar_Static, flags & kVar_Static); member->Cast<PHPEntityVariable>()->SetLine(token.lineNumber); CurrentScope()->AddChild(member); // Handle member assignment // public $memberVar = new Something(); // for such cases, assign $memberVar type of Something() phpLexerToken t; if(!NextToken(t)) { // EOF return; } if(t.type == '=') { // assignment wxString expr; if(!ReadExpression(expr)) { return; } // Optimize 'new ClassName(..)' expression if(expr.StartsWith("new")) { expr = expr.Mid(3); expr.Trim().Trim(false); expr = expr.BeforeFirst('('); expr.Trim().Trim(false); member->Cast<PHPEntityVariable>()->SetTypeHint(MakeIdentifierAbsolute(expr)); } else { // keep the expression member->Cast<PHPEntityVariable>()->SetExpressionHint(expr); } } else { // restore the token UngetToken(t); if(!ConsumeUntil(';')) return; } } else if(what == kPHP_T_FUNCTION) { // A function... OnFunction(); m_lookBackTokens.clear(); } } break; } case kPHP_T_DEFINE: // Define statement OnDefine(token); break; case kPHP_T_CONST: OnConstant(token); break; case kPHP_T_REQUIRE: case kPHP_T_REQUIRE_ONCE: case kPHP_T_INCLUDE: case kPHP_T_INCLUDE_ONCE: // Handle include files m_lookBackTokens.clear(); break; case kPHP_T_FOREACH: // found "foreach" statement OnForEach(); m_lookBackTokens.clear(); break; case kPHP_T_USE: // Found outer 'use' statement - construct the alias table if(Class()) { // inside a class, this means that this is a 'use <trait>;' OnUseTrait(); } else { // alias table OnUse(); } m_lookBackTokens.clear(); break; case kPHP_T_CLASS: case kPHP_T_INTERFACE: case kPHP_T_TRAIT: // Found class OnClass(token); m_lookBackTokens.clear(); break; case kPHP_T_NAMESPACE: // Found a namespace OnNamespace(); m_lookBackTokens.clear(); break; case kPHP_T_FUNCTION: // Found function OnFunction(); m_lookBackTokens.clear(); break; default: // Keep the token break; } } PhaseTwo(); }
void PHPSourceFile::ParseFunctionBody() { m_lookBackTokens.clear(); // when we reach the current depth-1 -> leave int exitDepth = m_depth - 1; phpLexerToken token; PHPEntityBase::Ptr_t var(NULL); while(NextToken(token)) { switch(token.type) { case '{': m_lookBackTokens.clear(); break; case '}': m_lookBackTokens.clear(); if(m_depth == exitDepth) { return; } break; case ';': m_lookBackTokens.clear(); break; case kPHP_T_CATCH: OnCatch(); break; case kPHP_T_VARIABLE: { var.Reset(new PHPEntityVariable()); var->SetFullName(token.text); var->SetFilename(m_filename.GetFullPath()); var->SetLine(token.lineNumber); CurrentScope()->AddChild(var); // Peek at the next token if(!NextToken(token)) return; // EOF if(token.type != '=') { m_lookBackTokens.clear(); var.Reset(NULL); UngetToken(token); } else { wxString expr; if(!ReadExpression(expr)) return; // EOF // Optimize 'new ClassName(..)' expression if(expr.StartsWith("new")) { expr = expr.Mid(3); expr.Trim().Trim(false); expr = expr.BeforeFirst('('); expr.Trim().Trim(false); var->Cast<PHPEntityVariable>()->SetTypeHint(MakeIdentifierAbsolute(expr)); } else { // keep the expression var->Cast<PHPEntityVariable>()->SetExpressionHint(expr); } } } break; default: break; } } }