bool TCpDoc::FNextCpd(const PSIn& SIn, PCpDoc& CpDoc){
if (SIn->Eof()){
CpDoc=NULL; return false;
} else {
CpDoc=TCpDoc::Load(*SIn); return true;
}
}
///////////////////////////////
// Tokenizer-Utils
void TTokenizerUtil::Sentencize(const PSIn& SIn, TStrV& Sentences, const bool& SplitNewLineP) {
TChA SentenceBuf;
int c;
while (!SIn->Eof()) {
c = SIn->GetCh();
switch (c) {
case '\r':
case '\n': {
if (!SplitNewLineP) {
SentenceBuf += ' ';
break;
}
}
case '"' :
case '.' :
case '!' :
case ':' :
case ';' :
case '?' :
case '\t': {
if (SentenceBuf.Len() > 2) {
Sentences.Add(SentenceBuf);
SentenceBuf.Clr();
}
break;
}
default:
SentenceBuf += c;
break;
}
}
if (SentenceBuf.Len() > 0) {
Sentences.Add(SentenceBuf);
}
}
char THttpLx::GetFirstCh(){
if (SIn->Eof()){
if (AtEof){throw THttpEx(heUnexpectedEof);}
AtEof=true; return 0;
} else {
Ch=SIn->GetCh(); return Ch;
}
}
char THttpLx::GetCh(){
if (EofChPrS.Empty()){
if (SIn->Eof()){
if (AtEof){throw THttpEx(heUnexpectedEof);}
AtEof=true; SfMem+=Ch; Ch=TCh::NullCh; return Ch;
} else {
SfMem+=Ch; Ch=SIn->GetCh(); return Ch;
}
} else {
SfMem+=Ch;
AtEof=EofChPrS.Top().Val1; Ch=EofChPrS.Top().Val2; EofChPrS.Pop();
return Ch;
}
}
bool TReplaySrv::ReplayStream(const PSIn& SIn, const PNotify& ErrorNotify)
{
while (!SIn->Eof()) {
try {
THttpReqSerInfo ReqInfo(*SIn);
PHttpRq HttpRq = ReqInfo.GetHttpRq();
ReplayHttpRq(HttpRq);
}
catch (PExcept E) {
ErrorNotify->OnNotifyFmt(ntErr, "TReplaySrv::ReplayStream. Exception while loading next request: %s", E->GetMsgStr().CStr());
}
catch (...) {
ErrorNotify->OnNotifyFmt(ntErr, "TReplaySrv::ReplayStream. General exception while loading next request.");
}
}
return true;
}
void TTokenizerUtil::Paragraphize(const PSIn& SIn, TStrV& Paragraphs) {
TChA ParagraphBuf;
int c;
bool wasSpace = false;
while (!SIn->Eof()) {
c = SIn->GetCh();
// two consecutive spaces signal a new paragraph
if (c == ' ' || c == '\t' || c == '\n') {
if (wasSpace) {
Paragraphs.Add(ParagraphBuf);
ParagraphBuf.Clr();
continue;
}
wasSpace = true;
} else {
wasSpace = false;
}
ParagraphBuf += c;
}
if (ParagraphBuf.Len() > 0) {
Paragraphs.Add(ParagraphBuf);
}
}
PBowDocBs TFtrGenBs::LoadCsv(TStr& FNm, const int& ClassId,
const TIntV& IgnoreIdV, const int& TrainLen) {
// feature generators
PFtrGenBs FtrGenBs = TFtrGenBs::New();
// CSV parsing stuff
PSIn SIn = TFIn::New(FNm);
char SsCh = ' '; TStrV FldValV;
// read the headers and initialise the feature generators
TSs::LoadTxtFldV(ssfCommaSep, SIn, SsCh, FldValV, false);
for (int FldValN = 0; FldValN < FldValV.Len(); FldValN++) {
const TStr& FldVal = FldValV[FldValN];
if (FldValN == ClassId) {
if (FldVal == "NOM") {
FtrGenBs->PutClsFtrGen(TFtrGenNominal::New());
} else if (FldVal == "MULTI-NOM") {
FtrGenBs->PutClsFtrGen(TFtrGenMultiNom::New());
} else {
TExcept::Throw("Wrong class type '" + FldVal + "', should be NOM or MULTI-NOM!");
}
} else if (!IgnoreIdV.IsIn(FldValN)) {
if (FldVal == TFtrGenNumeric::GetType()) {
FtrGenBs->AddFtrGen(TFtrGenNumeric::New());
} else if (FldVal == TFtrGenNominal::GetType()) {
FtrGenBs->AddFtrGen(TFtrGenNominal::New());
} else if (FldVal == TFtrGenToken::GetType()) {
FtrGenBs->AddFtrGen(TFtrGenToken::New(
TSwSet::New(swstNone), TStemmer::New(stmtNone)));
} else if (FldVal == TFtrGenSparseNumeric::GetType()) {
FtrGenBs->AddFtrGen(TFtrGenSparseNumeric::New());
} else if (FldVal == TFtrGenMultiNom::GetType()) {
FtrGenBs->AddFtrGen(TFtrGenMultiNom::New());
} else {
TExcept::Throw("Wrong type '" + FldVal + "'!");
}
}
}
const int Flds = FldValV.Len();
// read the lines and feed them to the feature generators
int Recs = 0;
while (!SIn->Eof()) {
if (Recs == TrainLen) { break; }
Recs++; printf("%7d\r", Recs);
TSs::LoadTxtFldV(ssfCommaSep, SIn, SsCh, FldValV, false);
// make sure line still has the same number of fields as the header
EAssertR(FldValV.Len() == Flds,
TStr::Fmt("Wrong number of fields in line %d! Found %d and expected %d!",
Recs + 1, FldValV.Len(), Flds));
// go over lines
try {
TStrV FtrValV;
for (int FldValN = 0; FldValN < FldValV.Len(); FldValN++) {
const TStr& FldVal = FldValV[FldValN];
if (FldValN == ClassId) {
FtrGenBs->UpdateCls(FldVal);
} else if (!IgnoreIdV.IsIn(FldValN)) {
FtrValV.Add(FldVal);
}
}
FtrGenBs->Update(FtrValV);
} catch (PExcept Ex) {
TExcept::Throw(TStr::Fmt("Error in line %d: '%s'!",
Recs+1, Ex->GetMsgStr().CStr()));
}
}
// read the file again and feed it to the training set
PBowDocBs BowDocBs = FtrGenBs->MakeBowDocBs();
// we read and ignore the headers since we parsed them already
SIn = TFIn::New(FNm); SsCh = ' ';
TSs::LoadTxtFldV(ssfCommaSep, SIn, SsCh, FldValV, false);
// read the lines and feed them to the training set
Recs = 0;
while (!SIn->Eof()){
Recs++; printf("%7d\r", Recs);
TSs::LoadTxtFldV(ssfCommaSep, SIn, SsCh, FldValV, false);
// make sure line still has the same number of fields as the header
EAssertR(FldValV.Len() == Flds,
TStr::Fmt("Wrong number of fields in line %s! Found %d and expected %d!",
Recs + 1, FldValV.Len(), Flds));
// go over lines and construct the sparse vector
TStrV FtrValV; TStr ClsFtrVal;
try {
for (int FldValN = 0; FldValN < FldValV.Len(); FldValN++) {
const TStr& FldVal = FldValV[FldValN];
if (FldValN == ClassId) {
ClsFtrVal = FldVal;
} else if (!IgnoreIdV.IsIn(FldValN)) {
FtrValV.Add(FldVal);
}
}
} catch (PExcept Ex) {
TExcept::Throw(TStr::Fmt("Error in line %d: '%s'!",
Recs+1, Ex->GetMsgStr().CStr()));
}
// add the feature vector to trainsets
FtrGenBs->AddBowDoc(BowDocBs, TStr::Fmt("Line-%d", Recs), FtrValV, ClsFtrVal);
}
// prepare training and testing doc ids
TIntV AllDIdV; BowDocBs->GetAllDIdV(AllDIdV); IAssert(AllDIdV.IsSorted());
TIntV TrainDIdV = AllDIdV; TrainDIdV.Trunc(TrainLen);
BowDocBs->PutTrainDIdV(TrainDIdV);
TIntV TestDIdV = AllDIdV; TestDIdV.Minus(TrainDIdV);
BowDocBs->PutTestDIdV(TestDIdV);
return BowDocBs;
}
void THttpLx::GetRest(){
while ((!SIn->Eof())&&(!EofChPrS.Empty())){GetCh();}
if (!SIn->Eof()){SfMem+=Ch;}
TMem RestMem; TMem::LoadMem(SIn, RestMem);
SfMem+=RestMem;
}
