/* TokenizeAll() - Get to the point we have a set of input tokens ready to parse. * * This function will, if it encounters no errors, completely tokenize all of the * input of the designated input file, as well as all files it %include's. * It will throw an exception on the first bad token (if any), but only after having * tokenized all of the input, %include files as well. ****************************************************************************************/ static void TokenizeAll(unique_ref<TTokens> Tokens, string GrammarFilename, int IncludeToken) { TFileTokens FileTokens; TToken Token; Tokens->Tokenize(GrammarFilename, IncludeToken); FileTokens = Tokens->GetFile(GrammarFilename); assert(FileTokens.Begin >= 0); for(auto iToken=FileTokens.Begin; ; ++iToken) { Token = Tokens->Get(iToken); if(Token.Type == TToken::TKEOF) break; if(Token.Type == TToken::INCLUDE) { Token = Tokens->Get(++iToken); if(Token.Type != TToken::WHITESPACE) Error("EXPECTING_WHITESPACE_AFTER_INCLUDE", "Token", iToken); Token = Tokens->Get(++iToken); if(Token.Type != TToken::QUOTED) Error("EXPECTING_QUOTED_STRING_AFTER_INCLUDE", "Token", iToken); auto Filename = Token.Unquote(); TokenizeAll(Tokens, Filename.get(), iToken); } } }
/* ??? modify to handle includes */ TParseToken TParser::GetNextToken() { CurrentToken = Tokens->Get(TokenCursor++); while( CurrentToken.Type == TToken::WHITESPACE || CurrentToken.Type == TToken::COMMENT || CurrentToken.Type == TToken::MCOMMENT || (IgnoreNewlines && CurrentToken.Type == TToken::NEWLINE)) CurrentToken = Tokens->Get(TokenCursor++); CurrentToken.Index = TokenCursor-1; if(CurrentToken.Type == TToken::TKEOF) --TokenCursor; return CurrentToken; }