///////////////////////////////
// Tokenizer-Html-Unicode
void TTokenizerHtmlUnicode::GetTokens(const PSIn& SIn, TStrV& TokenV) const {
TStr LineStr; TStrV WordStrV;
while (SIn->GetNextLn(LineStr)) {
TStr SimpleText = TUStr(LineStr).GetStarterLowerCaseStr();
TTokenizerHtml::GetTokens(TStrIn::New(SimpleText), TokenV);
}
}
TUStr TUStr::GetSubValV(const int& BChN, const int& EChN){
//size_t Len = EChN - BChN + 1;
TIntV UniChV;
int capacity = BChN - EChN + 1;
if(capacity < this->UniChV.Reserved()){
UniChV.Reserve(BChN - EChN + 1, 0);
}
this->UniChV.GetSubValV(BChN, EChN, UniChV);
return TUStr(UniChV);
}
