// ScanNextToken // . The main scanning function; here we branch out and // scan whatever the next token should be. void Scanner::ScanNextToken() { if(m_endedStream) return; if(!m_startedStream) return StartStream(); // get rid of whitespace, etc. (in between tokens it should be irrelevent) ScanToNextToken(); // check the latest simple key VerifySimpleKey(); // maybe need to end some blocks PopIndentTo(INPUT.column); // ***** // And now branch based on the next few characters! // ***** // end of stream if(INPUT.peek() == EOF) return EndStream(); if(INPUT.column == 0 && INPUT.peek() == Keys::Directive) return ScanDirective(); // document token if(INPUT.column == 0 && Exp::DocStart.Matches(INPUT)) return ScanDocStart(); if(INPUT.column == 0 && Exp::DocEnd.Matches(INPUT)) return ScanDocEnd(); // flow start/end/entry if(INPUT.peek() == Keys::FlowSeqStart || INPUT.peek() == Keys::FlowMapStart) return ScanFlowStart(); if(INPUT.peek() == Keys::FlowSeqEnd || INPUT.peek() == Keys::FlowMapEnd) return ScanFlowEnd(); if(INPUT.peek() == Keys::FlowEntry) return ScanFlowEntry(); // block/map stuff if(Exp::BlockEntry.Matches(INPUT)) return ScanBlockEntry(); if((m_flowLevel == 0 ? Exp::Key : Exp::KeyInFlow).Matches(INPUT)) return ScanKey(); if((m_flowLevel == 0 ? Exp::Value : Exp::ValueInFlow).Matches(INPUT)) return ScanValue(); // alias/anchor if(INPUT.peek() == Keys::Alias || INPUT.peek() == Keys::Anchor) return ScanAnchorOrAlias(); // tag if(INPUT.peek() == Keys::Tag) return ScanTag(); // special scalars if(m_flowLevel == 0 && (INPUT.peek() == Keys::LiteralScalar || INPUT.peek() == Keys::FoldedScalar)) return ScanBlockScalar(); if(INPUT.peek() == '\'' || INPUT.peek() == '\"') return ScanQuotedScalar(); // plain scalars if((m_flowLevel == 0 ? Exp::PlainScalar : Exp::PlainScalarInFlow).Matches(INPUT)) return ScanPlainScalar(); // don't know what it is! throw ParserException(INPUT.line, INPUT.column, ErrorMsg::UNKNOWN_TOKEN); }
void Scanner::ScanNextToken() { if (m_endedStream) { return; } if (!m_startedStream) { return StartStream(); } // get rid of whitespace, etc. (in between tokens it should be irrelevent) ScanToNextToken(); // maybe need to end some blocks PopIndentToHere(); // ***** // And now branch based on the next few characters! // ***** // end of stream if (!INPUT) { return EndStream(); } if (INPUT.column() == 0 && INPUT.peek() == Keys::Directive) { return ScanDirective(); } // document token if (INPUT.column() == 0 && Exp::DocStart().Matches(INPUT)) { return ScanDocStart(); } if (INPUT.column() == 0 && Exp::DocEnd().Matches(INPUT)) { return ScanDocEnd(); } // flow start/end/entry if (INPUT.peek() == Keys::FlowSeqStart || INPUT.peek() == Keys::FlowMapStart) { return ScanFlowStart(); } if (INPUT.peek() == Keys::FlowSeqEnd || INPUT.peek() == Keys::FlowMapEnd) { return ScanFlowEnd(); } if (INPUT.peek() == Keys::FlowEntry) { return ScanFlowEntry(); } // block/map stuff if (Exp::BlockEntry().Matches(INPUT)) { return ScanBlockEntry(); } if ((InBlockContext() ? Exp::Key() : Exp::KeyInFlow()).Matches(INPUT)) { return ScanKey(); } if (GetValueRegex().Matches(INPUT)) { return ScanValue(); } // alias/anchor if (INPUT.peek() == Keys::Alias || INPUT.peek() == Keys::Anchor) { return ScanAnchorOrAlias(); } // tag if (INPUT.peek() == Keys::Tag) { return ScanTag(); } // special scalars if (InBlockContext() && (INPUT.peek() == Keys::LiteralScalar || INPUT.peek() == Keys::FoldedScalar)) { return ScanBlockScalar(); } if (INPUT.peek() == '\'' || INPUT.peek() == '\"') { return ScanQuotedScalar(); } // plain scalars if ((InBlockContext() ? Exp::PlainScalar() : Exp::PlainScalarInFlow()) .Matches(INPUT)) { return ScanPlainScalar(); } // don't know what it is! throw ParserException(INPUT.mark(), ErrorMsg::UNKNOWN_TOKEN); }
TokenPtr SLScanner::ScanToken() { std::string spell; /* Scan directive (beginning with '#') */ if (Is('#')) return ScanDirective(); /* Scan identifier */ if (std::isalpha(UChr()) || Is('_')) return ScanIdentifier(); /* Scan number */ if (Is('.')) return ScanNumberOrDot(); if (std::isdigit(UChr())) return ScanNumber(); /* Scan string literal */ if (Is('\"')) return ScanStringLiteral(); /* Scan operators */ if (Is('=')) { spell += TakeIt(); if (Is('=')) return Make(Tokens::BinaryOp, spell, true); return Make(Tokens::AssignOp, spell); } if (Is('~')) return Make(Tokens::UnaryOp, spell, true); if (Is('!')) { spell += TakeIt(); if (Is('=')) return Make(Tokens::BinaryOp, spell, true); return Make(Tokens::UnaryOp, spell); } if (Is('%')) { spell += TakeIt(); if (Is('=')) return Make(Tokens::AssignOp, spell, true); return Make(Tokens::BinaryOp, spell); } if (Is('*')) { spell += TakeIt(); if (Is('=')) return Make(Tokens::AssignOp, spell, true); return Make(Tokens::BinaryOp, spell); } if (Is('^')) { spell += TakeIt(); if (Is('=')) return Make(Tokens::AssignOp, spell, true); return Make(Tokens::BinaryOp, spell); } if (Is('+')) return ScanPlusOp(); if (Is('-')) return ScanMinusOp(); if (Is('<') || Is('>')) return ScanAssignShiftRelationOp(Chr()); if (Is('&')) { spell += TakeIt(); if (Is('=')) return Make(Tokens::AssignOp, spell, true); if (Is('&')) return Make(Tokens::BinaryOp, spell, true); return Make(Tokens::BinaryOp, spell); } if (Is('|')) { spell += TakeIt(); if (Is('=')) return Make(Tokens::AssignOp, spell, true); if (Is('|')) return Make(Tokens::BinaryOp, spell, true); return Make(Tokens::BinaryOp, spell); } if (Is(':')) { spell += TakeIt(); if (Is(':')) return Make(Tokens::DColon, spell, true); return Make(Tokens::Colon, spell); } /* Scan punctuation, special characters and brackets */ switch (Chr()) { case ';': return Make(Tokens::Semicolon, true); break; case ',': return Make(Tokens::Comma, true); break; case '?': return Make(Tokens::TernaryOp, true); break; case '(': return Make(Tokens::LBracket, true); break; case ')': return Make(Tokens::RBracket, true); break; case '{': return Make(Tokens::LCurly, true); break; case '}': return Make(Tokens::RCurly, true); break; case '[': return Make(Tokens::LParen, true); break; case ']': return Make(Tokens::RParen, true); break; } ErrorUnexpected(); return nullptr; }