int Scanner::getNextToken(ScannerToken &t, Location &l) { int tokid; bool la = !m_lookahead.empty(); tokid = fetchToken(t, l); if (LIKELY(tokid != T_UNRESOLVED_LT)) { // In the common case, we don't have to perform any resolution // and we can just return the token if (UNLIKELY(la)) { // If we pulled a lookahead token, we need to remove it from // the lookahead store m_lookahead.popFront(); } return tokid; } // We encountered a '<' character that needs to be resolved. if (!la) { // If this token didn't come from the lookahead store, we // need to stash it there TokenStore::iterator it = m_lookahead.appendNew(); LookaheadToken ltd = { t, l, tokid }; *it = ltd; } // Look at subsequent tokens to determine if the '<' character // is the start of a type list TokenStore::iterator pos = m_lookahead.begin(); TokenStore::iterator ltPos = pos; nextLookahead(pos); ++m_lookaheadLtDepth; bool isTypeList = tryParseTypeList(pos); --m_lookaheadLtDepth; if (!isTypeList || pos->t != '>') { ltPos->t = '<'; } else { ltPos->t = T_TYPELIST_LT; pos->t = T_TYPELIST_GT; } tokid = fetchToken(t, l); // We pulled a lookahead token, we need to remove it from the // lookahead store m_lookahead.popFront(); return tokid; }
int Scanner::getNextToken(ScannerToken &t, Location &l) { int tokid; bool la = !m_lookahead.empty(); tokid = fetchToken(t, l); if (LIKELY(!isUnresolved(tokid))) { // In the common case, we don't have to perform any resolution // and we can just return the token if (UNLIKELY(la)) { // If we pulled a lookahead token, we need to remove it from // the lookahead store m_lookahead.popFront(); } return tokid; } if (!la) { // If this token didn't come from the lookahead store, we // need to stash it there TokenStore::iterator it = m_lookahead.appendNew(); LookaheadToken ltd = { t, l, tokid }; *it = ltd; } switch (tokid) { case T_UNRESOLVED_NEWTYPE: case T_UNRESOLVED_TYPE: { auto pos = m_lookahead.begin(); auto typePos = pos; nextLookahead(pos); if (isValidClassConstantName(pos->t)) { typePos->t = tokid == T_UNRESOLVED_TYPE ? T_TYPE : T_NEWTYPE; } else { typePos->t = T_STRING; } break; } case T_UNRESOLVED_LT: { // Look at subsequent tokens to determine if the '<' character // is the start of a type list auto pos = m_lookahead.begin(); auto ltPos = pos; nextLookahead(pos); ++m_lookaheadLtDepth; bool isTypeList = tryParseTypeList(pos); --m_lookaheadLtDepth; if (isTypeList && pos->t == '>') { ltPos->t = T_TYPELIST_LT; pos->t = T_TYPELIST_GT; } else { ltPos->t = '<'; } break; } case T_UNRESOLVED_OP: { // Look at subsequent tokens to determine if the '(' character // is the start of a lambda expression auto pos = m_lookahead.begin(); auto opPos = pos; nextLookahead(pos); if (pos->t != ')' && pos->t != T_LAMBDA_CP) { if (!tryParseNonEmptyLambdaParams(pos) || pos->t != ')') { opPos->t = '('; break; } } auto cpPos = pos; nextLookahead(pos); if (pos->t == ':') { nextLookahead(pos); if (!tryParseNSType(pos)) { opPos->t = '('; break; } } if (pos->t == T_LAMBDA_ARROW) { opPos->t = T_LAMBDA_OP; cpPos->t = T_LAMBDA_CP; } else { opPos->t = '('; } break; } default: always_assert(0); } tokid = fetchToken(t, l); // We pulled a lookahead token, we need to remove it from the // lookahead store m_lookahead.popFront(); return tokid; }
bool Scanner::tryParseNSType(TokenStore::iterator& pos) { if (pos->t == '@') { nextLookahead(pos); } if (pos->t == '?') { nextLookahead(pos); } if (pos->t == '(' || pos->t == T_UNRESOLVED_OP) { nextLookahead(pos); if (pos->t == T_FUNCTION) { nextLookahead(pos); if (pos->t != '(') return false; nextLookahead(pos); if (pos->t != ')') { if (!tryParseFuncTypeList(pos)) return false; if (pos->t != ')') return false; } nextLookahead(pos); if (pos->t == ')') { nextLookahead(pos); return true; } if (pos->t != ':') return false; nextLookahead(pos); if (!tryParseNSType(pos)) return false; if (pos->t != ')') return false; nextLookahead(pos); return true; } if (!tryParseTypeList(pos)) return false; if (pos->t != ')') return false; nextLookahead(pos); return true; } if (pos->t == T_NAMESPACE) { nextLookahead(pos); if (pos->t != T_NS_SEPARATOR) return false; nextLookahead(pos); } else if (pos->t == T_NS_SEPARATOR) { nextLookahead(pos); } for (;;) { switch (pos->t) { case T_STRING: case T_SUPER: case T_XHP_ATTRIBUTE: case T_XHP_CATEGORY: case T_XHP_CHILDREN: case T_XHP_REQUIRED: case T_ENUM: case T_ARRAY: case T_CALLABLE: case T_UNRESOLVED_TYPE: case T_UNRESOLVED_NEWTYPE: nextLookahead(pos); break; case T_SHAPE: return tryParseShapeType(pos); case T_XHP_LABEL: nextLookahead(pos); return true; default: return false; } if (pos->t == T_UNRESOLVED_LT) { TokenStore::iterator ltPos = pos; nextLookahead(pos); ++m_lookaheadLtDepth; bool isTypeList = tryParseTypeList(pos); --m_lookaheadLtDepth; if (!isTypeList || pos->t != '>') { ltPos->t = '<'; return false; } ltPos->t = T_TYPELIST_LT; pos->t = T_TYPELIST_GT; nextLookahead(pos); return true; } if (pos->t != T_NS_SEPARATOR && pos->t != T_DOUBLE_COLON) { return true; } nextLookahead(pos); } }
void Scanner::parseApproxParamDefVal(TokenStore::iterator& pos) { int64_t opNum = 0; // counts nesting for ( and T_UNRESOLVED_OP int64_t obNum = 0; // counts nesting for [ int64_t ocNum = 0; // counts nesting for { int64_t ltNum = 0; // counts nesting for T_TYPELIST_LT for (;; nextLookahead(pos)) { switch (pos->t) { case ',': if (!opNum && !obNum && !ocNum && !ltNum) return; break; case '(': case T_UNRESOLVED_OP: ++opNum; break; case ')': if (!opNum) return; --opNum; break; case '[': ++obNum; break; case ']': if (!obNum) return; --obNum; break; case '{': ++ocNum; break; case '}': if (!ocNum) return; --ocNum; break; case T_TYPELIST_LT: ++ltNum; break; case T_UNRESOLVED_LT: { auto endPos = pos; nextLookahead(endPos); if (tryParseTypeList(endPos) && endPos->t == '>') { pos->t = T_TYPELIST_LT; endPos->t = T_TYPELIST_GT; } else { pos->t = '<'; } ++ltNum; break; } case T_TYPELIST_GT: if (!ltNum) return; --ltNum; break; case T_LNUMBER: case T_DNUMBER: case T_ONUMBER: case T_CONSTANT_ENCAPSED_STRING: case T_START_HEREDOC: case T_ENCAPSED_AND_WHITESPACE: case T_END_HEREDOC: case T_LINE: case T_FILE: case T_DIR: case T_CLASS_C: case T_TRAIT_C: case T_METHOD_C: case T_FUNC_C: case T_NS_C: case T_COMPILER_HALT_OFFSET: case T_STRING: case T_ENUM: case T_XHP_LABEL: case T_XHP_ATTRIBUTE: case T_XHP_CATEGORY: case T_XHP_CHILDREN: case T_XHP_REQUIRED: case T_NS_SEPARATOR: case T_NAMESPACE: case T_SHAPE: case T_ARRAY: case T_FUNCTION: case T_DOUBLE_ARROW: case T_DOUBLE_COLON: case '+': case '-': case ':': case '?': case '@': break; default: return; } } }