LLTemplateTokenizer::LLTemplateTokenizer(const std::string & contents) : mStarted(false), mTokens() { boost::char_separator<char> newline("\r\n", "", boost::keep_empty_tokens); boost::char_separator<char> spaces(" \t"); U32 line_counter = 1; tokenizer line_tokens(contents, newline); for(tokenizer::iterator line_iter = line_tokens.begin(); line_iter != line_tokens.end(); ++line_iter, ++line_counter) { tokenizer word_tokens(*line_iter, spaces); for(tokenizer::iterator word_iter = word_tokens.begin(); word_iter != word_tokens.end(); ++word_iter) { if((*word_iter)[0] == '/') { break; // skip to end of line on comments } positioned_token pt;// = new positioned_token(); pt.str = std::string(*word_iter); pt.line = line_counter; mTokens.push_back(pt); } } mCurrent = mTokens.begin(); }
BOOL LLKeywords::loadFromFile( const std::string& filename ) { mLoaded = FALSE; //////////////////////////////////////////////////////////// // File header const S32 BUFFER_SIZE = 1024; char buffer[BUFFER_SIZE]; /* Flawfinder: ignore */ llifstream file; file.open(filename); /* Flawfinder: ignore */ if( file.fail() ) { llinfos << "LLKeywords::loadFromFile() Unable to open file: " << filename << llendl; return mLoaded; } // Identifying string file >> buffer; if( strcmp( buffer, "llkeywords" ) ) { llinfos << filename << " does not appear to be a keyword file" << llendl; return mLoaded; } // Check file version file >> buffer; U32 version_num; file >> version_num; if( strcmp(buffer, "version") || version_num != (U32)KEYWORD_FILE_CURRENT_VERSION ) { llinfos << filename << " does not appear to be a version " << KEYWORD_FILE_CURRENT_VERSION << " keyword file" << llendl; return mLoaded; } // start of line (SOL) std::string SOL_COMMENT("#"); std::string SOL_WORD("[word "); std::string SOL_LINE("[line "); std::string SOL_ONE_SIDED_DELIMITER("[one_sided_delimiter "); std::string SOL_TWO_SIDED_DELIMITER("[two_sided_delimiter "); LLColor3 cur_color( 1, 0, 0 ); LLKeywordToken::TOKEN_TYPE cur_type = LLKeywordToken::WORD; while (!file.eof()) { buffer[0] = 0; file.getline( buffer, BUFFER_SIZE ); std::string line(buffer); if( line.find(SOL_COMMENT) == 0 ) { continue; } else if( line.find(SOL_WORD) == 0 ) { cur_color = readColor( line.substr(SOL_WORD.size()) ); cur_type = LLKeywordToken::WORD; continue; } else if( line.find(SOL_LINE) == 0 ) { cur_color = readColor( line.substr(SOL_LINE.size()) ); cur_type = LLKeywordToken::LINE; continue; } else if( line.find(SOL_TWO_SIDED_DELIMITER) == 0 ) { cur_color = readColor( line.substr(SOL_TWO_SIDED_DELIMITER.size()) ); cur_type = LLKeywordToken::TWO_SIDED_DELIMITER; continue; } else if( line.find(SOL_ONE_SIDED_DELIMITER) == 0 ) { cur_color = readColor( line.substr(SOL_ONE_SIDED_DELIMITER.size()) ); cur_type = LLKeywordToken::ONE_SIDED_DELIMITER; continue; } std::string token_buffer( line ); LLStringUtil::trim(token_buffer); typedef boost::tokenizer<boost::char_separator<char> > tokenizer; boost::char_separator<char> sep_word("", " \t"); tokenizer word_tokens(token_buffer, sep_word); tokenizer::iterator token_word_iter = word_tokens.begin(); if( !token_buffer.empty() && token_word_iter != word_tokens.end() ) { // first word is keyword std::string keyword = (*token_word_iter); LLStringUtil::trim(keyword); // following words are tooltip std::string tool_tip; while (++token_word_iter != word_tokens.end()) { tool_tip += (*token_word_iter); } LLStringUtil::trim(tool_tip); if( !tool_tip.empty() ) { // Replace : with \n for multi-line tool tips. LLStringUtil::replaceChar( tool_tip, ':', '\n' ); addToken(cur_type, keyword, cur_color, tool_tip ); } else { addToken(cur_type, keyword, cur_color, LLStringUtil::null ); } } } file.close(); mLoaded = TRUE; return mLoaded; }