C++ (Cpp) TStr::SplitOnAllAnyCh示例

///////////////////////////////
// Tokenizer-Simple
void TTokenizerSimple::GetTokens(const PSIn& SIn, TStrV& TokenV) const {
	TStr LineStr; TStrV WordStrV;
	while (SIn->GetNextLn(LineStr)) {
		WordStrV.Clr(false);
		LineStr.SplitOnAllAnyCh(" .,!?\n\r()+=-{}[]%$#@\\/", WordStrV, true);
		for (int WordStrN = 0; WordStrN < WordStrV.Len(); WordStrN++) {
			const TStr& WordStr = WordStrV[WordStrN];
			const TStr UcStr = WordStr.GetUc();
			if (SwSet.Empty() || (!SwSet->IsIn(UcStr))) {
				TStr TokenStr = ToUcP ? UcStr : WordStr;
				if (!Stemmer.Empty()) {	
					TokenStr = Stemmer->GetStem(TokenStr); }
				TokenV.Add(TokenStr);
			}
		}
	}
}

示例#2

显示文件

文件： InOut.cpp 项目： KronGen/KronTest

void GetMtxFromSepLine(const TStr& line, const TStr& separator, TFltV& matrix){
    TStrV strVals;
    line.SplitOnAllAnyCh(separator, strVals);
    for (int i = 0; i < strVals.Len(); i++) 
        matrix.Add(strVals[i].GetFlt());
}