int TokensTree::insert(int loc, Token* newToken) { if(!newToken) return -1; return AddToken(newToken, loc); }
static void AddLiteral(const char* zLit, int nChars, struct Parse *pParse) { TemplateToken *pToken; if (!nChars) return; pToken = (TemplateToken*)malloc(sizeof(TemplateToken) + (nChars+1)*sizeof(char)); assert(pToken); memset(pToken, 0, sizeof(TemplateToken)); if ( (nChars==1 && *zLit=='\n') || (nChars==2 && *zLit=='\n' && *zLit=='\r')) pToken->nType = Newline; else { if (nChars>2 && *zLit=='\n' && *zLit=='\r') { AddLiteral(zLit, 2, pParse); // break tokens zLit+=2; nChars-=2; } else if (nChars>1 && *zLit=='\n') { AddLiteral(zLit, 1, pParse); // break tokens zLit++; nChars--; } pToken->nType = Literal; } pToken->nChars = nChars; AddToken(pToken, pParse); pToken++; strncpy((char*)pToken, zLit, nChars); ((char*)pToken)[nChars]='\0'; }
static void AddCommand(const char *zCmd, struct Parse *pParse) { TemplateToken *pToken; int nChars = strlen(zCmd); char **pzVar; if (!nChars) return; pToken = (TemplateToken*)malloc(sizeof(TemplateToken)+(nChars+1)*sizeof(char)); assert(pToken); memset(pToken, 0, sizeof(TemplateToken)); if ( strcmp(zCmd, "other")==0 ) pToken->nType = Alternative; else if (strcmp(zCmd, "common")==0) pToken->nType = Common; else { pToken->nType = Variable; pToken->nChars = nChars; for (pzVar = VariableStr; *pzVar; pzVar++) if (strcmp(*pzVar, zCmd)==0) { pToken->nType = InternalVariable; pToken->nVariable = pzVar-VariableStr; } } AddToken(pToken, pParse); pToken++; strncpy((char*)pToken, zCmd, nChars); ((char*)pToken)[nChars]='\0'; }
/** * This method is called just after a known text char has * been consumed and we should read a text run. Note: we actually ignore the * first character of the text run so that we can consume invalid markup * as text. * * @param aToken The OUT parameter that holds our resulting token. * @param aScanner Our source of data * @return Error result. */ nsresult nsHTMLTokenizer::ConsumeText(CToken*& aToken, nsScanner& aScanner) { nsresult result = NS_OK; nsTokenAllocator* theAllocator = this->GetTokenAllocator(); CTextToken* theToken = (CTextToken*)theAllocator->CreateTokenOfType(eToken_text, eHTMLTag_text); if (theToken) { PRUnichar ch = '\0'; result = theToken->Consume(ch, aScanner, mFlags); if (NS_FAILED(result)) { if (0 == theToken->GetTextLength()) { IF_FREE(aToken, mTokenAllocator); aToken = nullptr; } else { result = NS_OK; } } aToken = theToken; AddToken(aToken, result, &mTokenDeque, theAllocator); } return result; }
void Tokenizer::TokenizeByPreidentified(bool enclosed, const TokenRange& range) { std::vector<TokenRange> preidentified_tokens; keyword_manager.Peek(filename_, range, elements_, preidentified_tokens); size_t offset = range.offset; TokenRange subrange(range.offset, 0); while (offset < range.offset + range.size) { for (const auto& preidentified_token : preidentified_tokens) { if (offset == preidentified_token.offset) { if (subrange.size > 0) TokenizeByDelimiters(enclosed, subrange); AddToken(kIdentifier, enclosed, preidentified_token); subrange.offset = preidentified_token.offset + preidentified_token.size; offset = subrange.offset - 1; // It's going to be incremented below break; } } subrange.size = ++offset - subrange.offset; } // Either there was no preidentified token range, or we're now about to // process the tail of our current range. if (subrange.size > 0) TokenizeByDelimiters(enclosed, subrange); }
void CParser::checkFinished() { if( IsCorrect() ) { state = &CParser::parsingInitial; } else if( IsWrong() ) { state = &CParser::parsingIgnoreLine; AddToken(); } }
void Driver2Part1Test2() { std::vector<Token> tokens; AddToken(tokens, "class", TokenType::Class); AddToken(tokens, "Enemy", TokenType::Identifier); AddToken(tokens, "{", TokenType::OpenCurley); AddToken(tokens, "var", TokenType::Var); AddToken(tokens, "Lives", TokenType::Identifier); AddToken(tokens, ":", TokenType::Colon); AddToken(tokens, "Integer", TokenType::Identifier); AddToken(tokens, ";", TokenType::Semicolon); AddToken(tokens, "}", TokenType::CloseCurley); RunTest(1, 2, tokens); }
static inline void AddCustomToken(GLToken* TokenHash, const char* Value) { GLToken Token; GLString String; String.Chars = (char*)Value; String.Length = (unsigned int)strlen(Value); Token.Hash = GetStringHash(String); AddToken(TokenHash, Token); }
void CParser::BoolExpr() { if (mLookahead == '!') { Match('!'); BoolExpr(); AddToken(opNOT); } else { Expr(); if (mLookahead == RELOP) { int savedRelop = mRelop; Match(RELOP); Expr(); AddToken(savedRelop); } } } // CParser::BoolExpr
void AddTokenCommand(const CString& sLine) { CString sToken = sLine.Token(1).Left(DEFAULT_TOKEN_ID_LEN); if (sToken.length() != 12) { PutModule("Invalid token ID"); return; } AddToken(sToken); PutModule(sToken + " added"); }
void Tokenizer::TokenizeByBrackets() { static const std::vector<std::pair<char_t, char_t>> brackets{ {L'(', L')'}, // U+0028-U+0029 Parenthesis {L'[', L']'}, // U+005B-U+005D Square bracket {L'{', L'}'}, // U+007B-U+007D Curly bracket {L'\u300C', L'\u300D'}, // Corner bracket {L'\u300E', L'\u300F'}, // White corner bracket {L'\u3010', L'\u3011'}, // Black lenticular bracket {L'\uFF08', L'\uFF09'}, // Fullwidth parenthesis }; bool is_bracket_open = false; char_t matching_bracket = L'\0'; auto char_begin = filename_.begin(); const auto char_end = filename_.end(); // This is basically std::find_first_of() customized to our needs auto find_first_bracket = [&]() -> string_t::const_iterator { for (auto it = char_begin; it != char_end; ++it) { for (const auto& bracket_pair : brackets) { if (*it == bracket_pair.first) { matching_bracket = bracket_pair.second; return it; } } } return char_end; }; auto current_char = char_begin; while (current_char != char_end && char_begin != char_end) { if (!is_bracket_open) { current_char = find_first_bracket(); } else { // Looking for the matching bracket allows us to better handle some rare // cases with nested brackets. current_char = std::find(char_begin, char_end, matching_bracket); } const TokenRange range(std::distance(filename_.begin(), char_begin), std::distance(char_begin, current_char)); if (range.size > 0) // Found unknown token TokenizeByPreidentified(is_bracket_open, range); if (current_char != char_end) { // Found bracket AddToken(kBracket, true, TokenRange(range.offset + range.size, 1)); is_bracket_open = !is_bracket_open; char_begin = ++current_char; } } }
/** * This method consumes an end tag and any "attributes" that may come after it. * * @param aChar The last character read from the scanner. * @param aToken The OUT parameter that holds our resulting token. * @param aScanner Our source of data * @return Error result */ nsresult nsHTMLTokenizer::ConsumeEndTag(PRUnichar aChar, CToken*& aToken, nsScanner& aScanner) { // Get the "/" (we've already seen it with a Peek) aScanner.GetChar(aChar); nsTokenAllocator* theAllocator = this->GetTokenAllocator(); aToken = theAllocator->CreateTokenOfType(eToken_end, eHTMLTag_unknown); NS_ENSURE_TRUE(aToken, NS_ERROR_OUT_OF_MEMORY); // Remember this for later in case you have to unwind... int32_t theDequeSize = mTokenDeque.GetSize(); nsresult result = NS_OK; // Tell the new token to finish consuming text... result = aToken->Consume(aChar, aScanner, mFlags); AddToken(aToken, result, &mTokenDeque, theAllocator); if (NS_FAILED(result)) { // Note that this early-return here is safe because we have not yet // added any of our tokens to the queue (AddToken only adds the token if // result is a success), so we don't need to fall through. return result; } result = aScanner.Peek(aChar); if (NS_FAILED(result)) { aToken->SetInError(true); // Note: We know here that the scanner is not incremental since if // this peek fails, then we've already masked over a kEOF coming from // the Consume() call above. return NS_OK; } if (kGreaterThan != aChar) { result = ConsumeAttributes(aChar, aToken, aScanner); } else { aScanner.GetChar(aChar); } // Do the same thing as we do in ConsumeStartTag. Basically, if we've run // out of room in this *section* of the document, pop all of the tokens // we've consumed this round and wait for more data. if (NS_FAILED(result)) { while (mTokenDeque.GetSize() > theDequeSize) { CToken* theToken = (CToken*)mTokenDeque.Pop(); IF_FREE(theToken, mTokenAllocator); } } return result; }
static inline int ParseFile(char* Filename, GLArbToken* ArbHash, GLToken* FunctionsHash, unsigned int* FunctionCount, GLToken* DefinesHash, unsigned int* DefinesCount, GLSettings* Settings) { char* Data = ReadEntireFile(Filename); int Success = 0; if (Data) { GLTokenizer Tokenizer; Tokenizer.At = Data; while(*Tokenizer.At) { GLToken Token = ParseToken(&Tokenizer); if (StartsWith(Token.Value, "gl") && IsUpperCase(Token.Value.Chars[2])) { if (!Contains(FunctionsHash, Token) && IsKnownOrIgnoredToken(ArbHash, &Token, Settings)) { AddToken(FunctionsHash, Token); *FunctionCount += 1; } } if (StartsWith(Token.Value, "GL_")) { if (!Contains(DefinesHash, Token) && IsKnownOrIgnoredToken(ArbHash, &Token, Settings)) { AddToken(DefinesHash, Token); *DefinesCount += 1; } } } free(Data); Success = 1; } else { fprintf(stderr, "Couldn't open file %s", Filename); } return Success; }
void createCheckbox(const char *title,NxI32 token,const char *cmd,bool state) { AddToken(cmd,token); CPARSER.Parse("TuiCheckboxBegin %s", cmd); CPARSER.Parse("TuiName \"%s\"",title); if ( state ) CPARSER.Parse("TuiArg true"); else CPARSER.Parse("TuiArg false"); CPARSER.Parse("TuiScript %s %%1",cmd); CPARSER.Parse("TuiCheckboxEnd"); }
void AccumulationEncoder::DoEncoding(int token) { AddToken(token); // now normalize the vector! // TODO: what do we want here? A Unit vector? Max of 1? (atm it doesn't decay...) double _max = 0.0; for (int i = 0; i < dimensions; i++) _max = std::max(_max, myEncoder[i]); if (_max > 0) for (int i = 0; i<dimensions; i++) { myEncoder[i] /= _max; } }
void CParser::parsingWordBlankS() { if( token.type == TT_Blank ) { token.Swap( savedToken1 ); CQualifierParser::StartNamedQualifier(); state = IsWrong() ? &CParser::parsingIgnoreLine : &CParser::parsingQualifier; token.Swap( savedToken1 ); } else { token.Swap( savedToken1 ); if( !CRuleParser::BeginFunction() ) { // action state = &CParser::parsingIgnoreLine; return; } token.Swap( savedToken1 ); state = &CParser::parsingRule; savedToken2.Swap( token ); AddToken(); // QualifierTag savedToken2.Move( token ); AddToken(); // current token } }
void createSlider(const char *title,NxI32 token,const char *cmd,NxF32 smin,NxF32 smax,NxF32 sdefault,bool isint) { AddToken(cmd,token); CPARSER.Parse("TuiSliderBegin %s", cmd); CPARSER.Parse("TuiName \"%s\"",title); CPARSER.Parse("TuiSliderRange %f %f",smin,smax); CPARSER.Parse("TuiArg %f",sdefault); if ( isint ) { CPARSER.Parse("TuiInt true"); } CPARSER.Parse("TuiScript %s %%1",cmd); CPARSER.Parse("TuiSliderEnd"); }
void createButton(const char *title,NxI32 token,const char *cmd) { char scratch[512]; strcpy(scratch,cmd); char *space = strchr(scratch,' '); if ( space ) *space = 0; AddToken(scratch,token); CPARSER.Parse("TuiButtonBegin %s", scratch); CPARSER.Parse("TuiName \"%s\"",title); CPARSER.Parse("TuiScript %s",cmd); CPARSER.Parse("TuiExecuteOk false"); CPARSER.Parse("TuiCheckboxEnd"); }
void CParser::parsingBlank() { if( token.type == TT_Word && CDirectiveParser::StartParseIfDirective() ) { state = &CParser::parsingDirective; } else if( token.type == TT_LineFeed ) { state = &CParser::parsingInitial; } else { if( !CRuleParser::BeginRule() ) { state = &CParser::parsingIgnoreLine; return; } state = &CParser::parsingRule; AddToken(); } }
/** * This method is called just after a "<!" has been consumed. * NOTE: Here we might consume DOCTYPE and "special" markups. * * @param aChar The last character read from the scanner. * @param aToken The OUT parameter that holds our resulting token. * @param aScanner Our source of data * @return Error result. */ nsresult nsHTMLTokenizer::ConsumeSpecialMarkup(PRUnichar aChar, CToken*& aToken, nsScanner& aScanner) { // Get the "!" aScanner.GetChar(aChar); nsresult result = NS_OK; nsAutoString theBufCopy; aScanner.Peek(theBufCopy, 20); ToUpperCase(theBufCopy); int32_t theIndex = theBufCopy.Find("DOCTYPE", false, 0, 0); nsTokenAllocator* theAllocator = this->GetTokenAllocator(); if (theIndex == kNotFound) { if ('[' == theBufCopy.CharAt(0)) { aToken = theAllocator->CreateTokenOfType(eToken_cdatasection, eHTMLTag_comment); } else if (StringBeginsWith(theBufCopy, NS_LITERAL_STRING("ELEMENT")) || StringBeginsWith(theBufCopy, NS_LITERAL_STRING("ATTLIST")) || StringBeginsWith(theBufCopy, NS_LITERAL_STRING("ENTITY")) || StringBeginsWith(theBufCopy, NS_LITERAL_STRING("NOTATION"))) { aToken = theAllocator->CreateTokenOfType(eToken_markupDecl, eHTMLTag_markupDecl); } else { aToken = theAllocator->CreateTokenOfType(eToken_comment, eHTMLTag_comment); } } else { aToken = theAllocator->CreateTokenOfType(eToken_doctypeDecl, eHTMLTag_doctypeDecl); } if (aToken) { result = aToken->Consume(aChar, aScanner, mFlags); AddToken(aToken, result, &mTokenDeque, theAllocator); } if (result == kNotAComment) { result = ConsumeText(aToken, aScanner); } return result; }
/** * This method is called just after a newline has been consumed. * * @param aChar The last character read from the scanner. * @param aToken The OUT parameter that holds our resulting token. * @param aScanner Our source of data * @return Error result. */ nsresult nsHTMLTokenizer::ConsumeNewline(PRUnichar aChar, CToken*& aToken, nsScanner& aScanner) { // Get the newline character aScanner.GetChar(aChar); nsTokenAllocator* theAllocator = this->GetTokenAllocator(); aToken = theAllocator->CreateTokenOfType(eToken_newline, eHTMLTag_newline); nsresult result = NS_OK; if (aToken) { result = aToken->Consume(aChar, aScanner, mFlags); AddToken(aToken, result, &mTokenDeque, theAllocator); } return result; }
/** * This method is called just after a <? has been consumed. * * @param aChar The last character read from the scanner. * @param aToken The OUT parameter that holds our resulting token. * @param aScanner Our source of data * @return Error result. */ nsresult nsHTMLTokenizer::ConsumeProcessingInstruction(PRUnichar aChar, CToken*& aToken, nsScanner& aScanner) { // Get the "?" aScanner.GetChar(aChar); nsTokenAllocator* theAllocator = this->GetTokenAllocator(); aToken = theAllocator->CreateTokenOfType(eToken_instruction, eHTMLTag_unknown); nsresult result = NS_OK; if (aToken) { result = aToken->Consume(aChar, aScanner, mFlags); AddToken(aToken, result, &mTokenDeque, theAllocator); } return result; }
void CParser::parsingWord() { if( token.type == TT_LineFeed ) { token.Swap( savedToken1 ); CRuleParser::BeginFunction(); // action state = &CParser::parsingInitial; } else if( token.type == TT_Blank ) { state = &CParser::parsingWordBlank; } else { token.Swap( savedToken1 ); if( !CRuleParser::BeginFunction() ) { // action state = &CParser::parsingIgnoreLine; return; } token.Swap( savedToken1 ); state = &CParser::parsingRule; AddToken(); } }
void Tokenizer::TokenizeByBrackets() { static const string_t kOpeningBrackets = L"[({"; static const string_t kClosingBrackets = L"])}"; bool bracket_open = false; size_t last_bracket_index = 0; TokenRange range; for (size_t offset = 0; offset < filename_.size(); offset++) { const auto& brackets = bracket_open ? kClosingBrackets : kOpeningBrackets; const size_t index = brackets.find(filename_.at(offset)); // Character is a bracket if (index != string_t::npos) { // Check if it matches last open bracket if (bracket_open) { if (index != last_bracket_index) continue; } else { last_bracket_index = index; } // Add unknown token if (range.offset < offset) { range.size = offset - range.offset; TokenizeByDelimiter(bracket_open, range); } // Add bracket AddToken(kBracket, true, TokenRange(offset, 1)); bracket_open = !bracket_open; range.offset = offset + 1; // Character is not a bracket, and the loop reached the end } else if (offset == filename_.size() - 1) { // Add last unknown token range.size = offset - range.offset + 1; TokenizeByDelimiter(false, range); } } }
void CParser::ParamList() { int args = 0; mIsFormula = true; while (mLookahead != ')') { if (mLookahead != LIST) RelExpr(); else AddToken(valNil); if (mLookahead != ')') Match(LIST); args++; } mArgCnt = args; } // CParser::ParamList
bool CParser::Parse(const char *inString, cell inLocation) { // fprintf( stderr, "CParser::Parse( \"%s\", [ %hd , %hd ] )\n", // inString , inLocation.h , inLocation.v ) ; mLoc = inLocation; mExpr = mExprStart = inString; mOffset = 0; bool result = true; try { mTokenStart = mExpr; mLookahead = GetNextToken(true); if (mLookahead == RELOP && mRelop == opEQ) { Match(RELOP); mIsFormula = true; } else mIsFormula = false; RelExpr(); Match(END); AddToken(opEnd); } catch (CErr& e) { if (mIsFormula) throw; result = false; } return result; } // CParser::Parse
/** * This method is called just after a "&" has been consumed * and we know we're at the start of an entity. * * @param aChar The last character read from the scanner. * @param aToken The OUT parameter that holds our resulting token. * @param aScanner Our source of data * @return Error result. */ nsresult nsHTMLTokenizer::ConsumeEntity(PRUnichar aChar, CToken*& aToken, nsScanner& aScanner) { PRUnichar theChar; nsresult result = aScanner.Peek(theChar, 1); nsTokenAllocator* theAllocator = this->GetTokenAllocator(); if (NS_SUCCEEDED(result)) { if (nsCRT::IsAsciiAlpha(theChar) || theChar == kHashsign) { aToken = theAllocator->CreateTokenOfType(eToken_entity, eHTMLTag_entity); NS_ENSURE_TRUE(aToken, NS_ERROR_OUT_OF_MEMORY); result = aToken->Consume(theChar, aScanner, mFlags); if (result == NS_HTMLTOKENS_NOT_AN_ENTITY) { IF_FREE(aToken, mTokenAllocator); } else { if (result == kEOF && !aScanner.IsIncremental()) { result = NS_OK; // Use as much of the entity as you can get. } AddToken(aToken, result, &mTokenDeque, theAllocator); return result; } } // Oops, we're actually looking at plain text... result = ConsumeText(aToken, aScanner); } else if (result == kEOF && !aScanner.IsIncremental()) { // If the last character in the file is an &, consume it as text. result = ConsumeText(aToken, aScanner); if (aToken) { aToken->SetInError(true); } } return result; }
/** * This method is called just after a "<!" has been consumed * and we know we're at the start of a comment. * * @param aChar The last character read from the scanner. * @param aToken The OUT parameter that holds our resulting token. * @param aScanner Our source of data * @return Error result. */ nsresult nsHTMLTokenizer::ConsumeComment(PRUnichar aChar, CToken*& aToken, nsScanner& aScanner) { // Get the "!" aScanner.GetChar(aChar); nsTokenAllocator* theAllocator = this->GetTokenAllocator(); aToken = theAllocator->CreateTokenOfType(eToken_comment, eHTMLTag_comment); nsresult result = NS_OK; if (aToken) { result = aToken->Consume(aChar, aScanner, mFlags); AddToken(aToken, result, &mTokenDeque, theAllocator); } if (kNotAComment == result) { // AddToken has IF_FREE()'d our token, so... result = ConsumeText(aToken, aScanner); } return result; }
CppColorizer() { text::ColorEntry entry=win7::Win7GetTextBoxTextColor(); SetDefaultColor(entry); entry.normal.text=Color(128, 0, 255); AddToken(L"/d+(./d*)?([eE][+/-]?/d+)?", entry); entry.normal.text=Color(163, 21, 21); AddToken(L"\"([^\\\\\"]|\\\\/.)*\"", entry); entry.normal.text=Color(0, 128, 0); AddToken(L"////[^\r\n]*", entry); AddToken(L"///*(//|[*]*[^*//])*/*+//", entry); // debug this: L"//[*]([^*]|[*]+[^//])*[*]+//" entry.normal.text=Color(0, 0, 255); AddToken(L"#[a-zA-Z0-9_]*", entry); AddToken(CppKeywords, entry); AddToken(L"[a-zA-Z0-9_]+", GetDefaultColor()); Setup(); }
int VP8RecordCoeffTokens(int ctx, int coeff_type, int first, int last, const int16_t* const coeffs, VP8TBuffer* const tokens) { int n = first; uint32_t base_id = TOKEN_ID(coeff_type, n, ctx, 0); if (!AddToken(tokens, last >= 0, base_id + 0)) { return 0; } while (n < 16) { const int c = coeffs[n++]; const int sign = c < 0; int v = sign ? -c : c; if (!AddToken(tokens, v != 0, base_id + 1)) { ctx = 0; base_id = TOKEN_ID(coeff_type, VP8EncBands[n], ctx, 0); continue; } if (!AddToken(tokens, v > 1, base_id + 2)) { ctx = 1; } else { if (!AddToken(tokens, v > 4, base_id + 3)) { if (AddToken(tokens, v != 2, base_id + 4)) AddToken(tokens, v == 4, base_id + 5); } else if (!AddToken(tokens, v > 10, base_id + 6)) { if (!AddToken(tokens, v > 6, base_id + 7)) { AddConstantToken(tokens, v == 6, 159); } else { AddConstantToken(tokens, v >= 9, 165); AddConstantToken(tokens, !(v & 1), 145); } } else { int mask; const uint8_t* tab; if (v < 3 + (8 << 1)) { // VP8Cat3 (3b) AddToken(tokens, 0, base_id + 8); AddToken(tokens, 0, base_id + 9); v -= 3 + (8 << 0); mask = 1 << 2; tab = VP8Cat3; } else if (v < 3 + (8 << 2)) { // VP8Cat4 (4b) AddToken(tokens, 0, base_id + 8); AddToken(tokens, 1, base_id + 9); v -= 3 + (8 << 1); mask = 1 << 3; tab = VP8Cat4; } else if (v < 3 + (8 << 3)) { // VP8Cat5 (5b) AddToken(tokens, 1, base_id + 8); AddToken(tokens, 0, base_id + 10); v -= 3 + (8 << 2); mask = 1 << 4; tab = VP8Cat5; } else { // VP8Cat6 (11b) AddToken(tokens, 1, base_id + 8); AddToken(tokens, 1, base_id + 10); v -= 3 + (8 << 3); mask = 1 << 10; tab = VP8Cat6; } while (mask) { AddConstantToken(tokens, !!(v & mask), *tab++); mask >>= 1; } } ctx = 2; } AddConstantToken(tokens, sign, 128); base_id = TOKEN_ID(coeff_type, VP8EncBands[n], ctx, 0); if (n == 16 || !AddToken(tokens, n <= last, base_id + 0)) { return 1; // EOB } } return 1; }