wxString Tokenizer::PeekToken() { if (!m_PeekAvailable) { m_PeekAvailable = true; unsigned int savedTokenIndex = m_TokenIndex; unsigned int savedLineNumber = m_LineNumber; unsigned int savedNestLevel = m_NestLevel; if (SkipUnwanted()) m_PeekToken = DoGetToken(); else m_PeekToken.Clear(); m_PeekTokenIndex = m_TokenIndex; m_PeekLineNumber = m_LineNumber; m_PeekNestLevel = m_NestLevel; m_TokenIndex = savedTokenIndex; m_LineNumber = savedLineNumber; m_NestLevel = savedNestLevel; } return m_PeekToken; }
wxString Tokenizer::GetToken() { m_UndoTokenIndex = m_TokenIndex; m_UndoLineNumber = m_LineNumber; m_UndoNestLevel = m_NestLevel; if (m_PeekAvailable) { m_TokenIndex = m_PeekTokenIndex; m_LineNumber = m_PeekLineNumber; m_NestLevel = m_PeekNestLevel; m_Token = m_PeekToken; } else { if (SkipUnwanted()) m_Token = DoGetToken(); else m_Token.Clear(); } m_PeekAvailable = false; return m_Token; }
bool Tokenizer::SkipUnwanted() { while (SkipWhiteSpace() || SkipComment()) ; wxChar c = CurrentChar(); const unsigned int startIndex = m_TokenIndex; if (c == _T('#')) { const PreprocessorType type = GetPreprocessorType(); if (type != ptOthers) { HandleConditionPreprocessor(type); c = CurrentChar(); } } // skip [XXX][YYY] if (m_State & tsSkipSubScrip) { while (c == _T('[') ) { SkipBlock('['); SkipWhiteSpace(); if (IsEOF()) return false; c = CurrentChar(); } } // skip the following = or ? if (m_State & tsSkipEqual) { if (c == _T('=')) { if (!SkipToOneOfChars(_T(",;}"), true, true, false)) return false; } } else if (m_State & tsSkipQuestion) { if (c == _T('?')) { if (!SkipToOneOfChars(_T(";}"), false, true)) return false; } } // skip the following white space and comments while (SkipWhiteSpace() || SkipComment()) ; if (startIndex != m_TokenIndex && CurrentChar() == _T('#')) return SkipUnwanted(); return NotEOF(); }
void TextCutter::GetChunk(wxString& chunk, bool& isWord) { isWord = false; chunk = wxEmptyString; if (IsEOF()) return; unsigned int start = m_CurIdx; if (isalpha(CurrentChar()) || CurrentChar() == '_') { while (!IsEOF() && (isalnum(CurrentChar()) || CurrentChar() == '_')) MoveToNextChar(); chunk = m_Text.Mid(start, m_CurIdx - start); isWord = true; } else { SkipWhiteSpace(); SkipUnwanted(); if (start != m_CurIdx) { chunk = m_Text.Mid(start, m_CurIdx - start); return; } if (isdigit(CurrentChar())) { // numbers while (!IsEOF() && CharInString(CurrentChar(), "0123456789.abcdefABCDEFXxLl")) MoveToNextChar(); } else if (CurrentChar() == '"' || CurrentChar() == '\'') { // string, char, etc. wxChar match = CurrentChar(); MoveToNextChar(); // skip starting ' or " SkipToChar(match); MoveToNextChar(); // skip ending ' or " } else { MoveToNextChar(); } chunk = m_Text.Mid(start, m_CurIdx - start); } return; }
void Tokenizer::MacroReplace(wxString& str) { if (m_IsReplaceParsing) { const int id = m_TokensTree->TokenExists(str, -1, tkPreprocessor); if (id != -1) { Token* tk = m_TokensTree->at(id); if (tk) { bool replaced = false; if (!tk->m_Args.IsEmpty()) replaced = ReplaceMacroActualContext(tk, false); else if (tk->m_Type != tk->m_Name) replaced = ReplaceBufferForReparse(tk->m_Type, false); if (replaced || tk->m_Type.IsEmpty()) { SkipUnwanted(); str = DoGetToken(); } } } } wxStringHashMap::const_iterator it = s_Replacements.find(str); if (it == s_Replacements.end()) return; TRACE(_T("MacroReplace() : Replacing '%s' with '%s' (file='%s', line='%d')."), it->first.wx_str(), it->second.wx_str(), m_Filename.wx_str(), m_LineNumber); if (it->second.IsEmpty()) { SkipUnwanted(); str = DoGetToken(); } else if (it->second[0] == _T('+')) { while (SkipWhiteSpace() || SkipComment()) ; DoGetToken(); // eat (...) wxString target = (const wxChar*)it->second + 1; if (target.IsEmpty()) { while (SkipWhiteSpace() || SkipComment()) ; str = DoGetToken(); } else if (target != str && ReplaceBufferForReparse(target, false)) str = DoGetToken(); } else if (it->second[0] == _T('-')) { wxString end((const wxChar*)it->second + 1); if (end.IsEmpty()) return; while (NotEOF()) { while (SkipComment() && SkipWhiteSpace()) ; if (CurrentChar() == end[0]) { if (DoGetToken() == end) break; } else MoveToNextChar(); } // eat () SkipUnwanted(); str = DoGetToken(); if (str[0] == _T('(')) { SkipUnwanted(); str = DoGetToken(); } } else { if (it->second != str && ReplaceBufferForReparse(it->second, false)) str = DoGetToken(); } }
//vfc add bGetValue wxString Tokenizer::DoGetToken(bool bGetValue, bool bTemplate) { if (IsEOF()) return wxEmptyString; if (!SkipWhiteSpace()) return wxEmptyString; if (m_SkipUnwantedTokens && !SkipUnwanted(bGetValue)) return wxEmptyString; // if m_SkipUnwantedTokens is false, we need to handle comments here too if (!m_SkipUnwantedTokens) SkipComment(); int start = m_TokenIndex; wxString m_Str; wxChar c = CurrentChar(); if (c == '_' || wxIsalpha(c)) { // keywords, identifiers, etc. // operator== is cheaper than wxIsalnum, also MoveToNextChar already includes IsEOF while ( ( CurrentChar() == '_' || wxIsalnum(CurrentChar()) ) && MoveToNextChar() ) ; if (IsEOF()) return wxEmptyString; m_Str = m_Buffer.Mid(start, m_TokenIndex - start); m_IsOperator = m_Str.IsSameAs(TokenizerConsts::operator_str); } #ifdef __WXMSW__ // This is a Windows only bug! else if (c == 178 || c == 179 || c == 185) // fetch ?and ? { m_Str = c; MoveToNextChar(); } #endif else if (wxIsdigit(CurrentChar())) { // numbers while (NotEOF() && CharInString(CurrentChar(), _T("0123456789.abcdefABCDEFXxLl"))) MoveToNextChar(); if (IsEOF()) return wxEmptyString; m_Str = m_Buffer.Mid(start, m_TokenIndex - start); m_IsOperator = false; } else if (CurrentChar() == '"' || CurrentChar() == '\'') { // string, char, etc. wxChar match = CurrentChar(); MoveToNextChar(); // skip starting ' or " if (!SkipToChar(match)) return wxEmptyString; MoveToNextChar(); // skip ending ' or " m_Str = m_Buffer.Mid(start, m_TokenIndex - start); } else if (CurrentChar() == ':') { if (NextChar() == ':') { MoveToNextChar(); MoveToNextChar(); m_Str.assign(TokenizerConsts::colon_colon); // this only copies a pointer, but operator= allocates memory and does a memcpy! } else { MoveToNextChar(); m_Str.assign(TokenizerConsts::colon); } } else if (CurrentChar() == '<' && bTemplate) { wxChar match = _T('>'); MoveToNextChar(); if (!SkipToOneOfChars(_T(">\r\n")),false) return wxEmptyString; MoveToNextChar(); wxString tmp = m_Buffer.Mid(start+1,m_TokenIndex-start-2); tmp.Trim(); m_Str = _T("<"); m_Str += tmp; m_Str += _T(">");//m_Buffer.Mid(start, m_TokenIndex - start); } else if (CurrentChar() == '(') { m_IsOperator = false; // skip blocks () [] if (!SkipBlock(CurrentChar())) return wxEmptyString; wxString tmp = m_Buffer.Mid(start, m_TokenIndex - start); // tmp.Replace(_T("\t"), _T(" ")); // replace tabs with spaces // tmp.Replace(_T("\n"), _T(" ")); // replace LF with spaces // tmp.Replace(_T("\r"), _T(" ")); // replace CR with spaces { // this is much faster: size_t i; while((i = tmp.find_first_of(TokenizerConsts::tabcrlf)) != wxString::npos) //tmp[i] = _T(' '); tmp.SetAt(i,_T(' ')); } // fix-up arguments (remove excessive spaces/tabs/newlines) for (unsigned int i = 0; i < tmp.Length() - 1; ++i) { //skip spaces before '=' and ',' if (tmp.GetChar(i) == ' ' && (tmp.GetChar(i + 1) == ',' || tmp.GetChar(i + 1) == '=')) continue; if (tmp.GetChar(i) == '/' && tmp.GetChar(i + 1) == '*') { // skip C comments i += 2; while (i < tmp.Length() - 1) { if (tmp.GetChar(i) == '*' && tmp.GetChar(i + 1) == '/') break; ++i; } if (i >= tmp.Length() - 1 || tmp.GetChar(i + 1) != '/') continue; // we failed... i += 2; } else if (tmp.GetChar(i) == '=') { // skip default assignments ++i; int level = 0; // nesting parenthesis while (i < tmp.Length()) { if (tmp.GetChar(i) == '(') ++level; else if (tmp.GetChar(i) == ')') --level; if ((tmp.GetChar(i) == ',' && level == 0) || (tmp.GetChar(i) == ')' && level < 0)) break; ++i; } if (i < tmp.Length() && tmp.GetChar(i) == ',') --i; continue; // we are done here } if (i < tmp.Length() - 1) { if ((tmp.GetChar(i) == ' ') && (tmp.GetChar(i + 1) == ' ')) continue; // skip excessive spaces // in case of c-style comments "i" might already be tmp.Length() // thus do only add the current char otherwise. // otherwise the following statement: // m_Str << _T(')'); // below would add another closing bracket. m_Str << tmp.GetChar(i); } } m_Str << _T(')'); // add closing parenthesis (see "i < tmp.Length() - 1" in previous "for") // m_Str.Replace(_T(" "), _T(" ")); // replace two-spaces with single-space (introduced if it skipped comments or assignments) // m_Str.Replace(_T("( "), _T("(")); // m_Str.Replace(_T(" )"), _T(")")); //Str.Replace is massive overkill here since it has to allocate one new block per replacement CompactSpaces(m_Str); } else { if (CurrentChar() == '{') ++m_NestLevel; else if (CurrentChar() == '}') --m_NestLevel; m_Str = CurrentChar(); MoveToNextChar(); } if (m_LastWasPreprocessor && !m_Str.IsSameAs(_T("#")) && !m_LastPreprocessor.IsSameAs(_T("#"))) { if (!m_LastPreprocessor.IsSameAs(TokenizerConsts::include_str)) { // except for #include and #if[[n]def], all other preprocessor directives need only // one word exactly after the directive, e.g. #define THIS_WORD SkipToEOL(); } m_LastPreprocessor.Clear(); } if (m_LastWasPreprocessor) m_LastPreprocessor << m_Str; m_LastWasPreprocessor = false; return m_Str; }