void TEnv::GetVarNmValV(TStrV& VarNmValV) {
VarNmValV.Clr();
int VarN = 0;
while (_environ[VarN] != NULL) {
VarNmValV.Add(_environ[VarN++]);
}
}
void TStrUtil::GetStdNameV(TStr AuthorNames, TStrV& StdNameV) {
AuthorNames.ChangeChAll('\n', ' ');
AuthorNames.ToLc();
// split into author names
TStrV AuthV, TmpV, Tmp2V;
// split on 'and'
AuthorNames.SplitOnStr(" and ", TmpV);
int i;
for (i = 0; i < TmpV.Len(); i++) {
TmpV[i].SplitOnAllCh(',', Tmp2V); AuthV.AddV(Tmp2V); }
// split on '&'
TmpV = AuthV; AuthV.Clr();
for (i = 0; i < TmpV.Len(); i++) {
TmpV[i].SplitOnAllCh('&', Tmp2V); AuthV.AddV(Tmp2V); }
// split on ','
TmpV = AuthV; AuthV.Clr();
for (i = 0; i < TmpV.Len(); i++) {
TmpV[i].SplitOnAllCh(',', Tmp2V); AuthV.AddV(Tmp2V); }
// split on ';'
TmpV = AuthV; AuthV.Clr();
for (i = 0; i < TmpV.Len(); i++) {
TmpV[i].SplitOnAllCh(';', Tmp2V); AuthV.AddV(Tmp2V); }
// standardize names
StdNameV.Clr();
//printf("\n*** %s\n", AuthorNames.CStr());
for (i = 0; i < AuthV.Len(); i++) {
TStr StdName = GetStdName(AuthV[i]);
if (! StdName.Empty()) {
//printf("\t%s ==> %s\n", AuthV[i].CStr(), StdName.CStr());
StdNameV.Add(StdName);
}
}
}
void TFtrGenMultiNom::GetValV(const TStr& Str, TStrV& ValV) const {
TStrV EltV; Str.SplitOnAllCh(';', EltV); ValV.Clr();
for (int EltN = 0; EltN < EltV.Len(); EltN++) {
const TStr& Val = EltV[EltN];
TStrV SubValV; FtrGen->GetValV(Val, SubValV);
ValV.AddV(SubValV);
}
}
void TSAppSrvFun::GetFldValV(const TStrKdV& FldNmValPrV, const TStr& FldNm, TStrV& FldValV) {
FldValV.Clr();
int ValN = FldNmValPrV.SearchForw(TStrKd(FldNm, ""));
while (ValN != -1) {
FldValV.Add(FldNmValPrV[ValN].Dat);
ValN = FldNmValPrV.SearchForw(TStrKd(FldNm, ""), ValN + 1);
}
}
void TExpHelp::GetCatNmV(TStrV& CatNmV) const {
CatNmV.Clr();
for (int ObjN=0; ObjN<ObjV.Len(); ObjN++){
TStr CatNm=ObjV[ObjN]->GetCatNm();
CatNmV.AddUnique(CatNm);
}
CatNmV.Ins(0, "All");
}
void TNGramBs::GetNGramStrV(
const TStr& HtmlStr, TStrV& NGramStrV, TIntPrV& NGramBEChXPrV) const {
TIntV NGramIdV; NGramStrV.Clr(); NGramBEChXPrV.Clr();
TNGramBs::GetNGramIdV(HtmlStr, NGramIdV, NGramBEChXPrV);
NGramStrV.Gen(NGramIdV.Len(), 0);
for (int NGramIdN=0; NGramIdN<NGramIdV.Len(); NGramIdN++){
TStr NGramStr=GetNGramStr(NGramIdV[NGramIdN]);
NGramStrV.Add(NGramStr);
}
}
void TExpHelp::GetObjHdNmV(const TStr& CatNm, TStrV& ObjHdNmV) const {
ObjHdNmV.Clr();
for (int ObjN=0; ObjN<ObjV.Len(); ObjN++){
TStr ObjCatNm=ObjV[ObjN]->GetCatNm();
TStr ObjHdNm=ObjV[ObjN]->GetHdItem()->GetNm();
if ((CatNm.Empty())||(CatNm=="All")||(CatNm==ObjCatNm)){
ObjHdNmV.AddUnique(ObjHdNm);}
}
ObjHdNmV.Sort();
}
// return the list of folders containing backups for a given profile name
// folders are sorted from the oldest to the newest
void TFolderBackup::GetBackupFolders(const TStr& ProfileName, TStrV& FolderNmV) const
{
FolderNmV.Clr();
if (ProfileH.IsKey(ProfileName)) {
TBackupProfile Profile = ProfileH.GetDat(ProfileName);
const TVec<TBackupLogInfo>& LogV = Profile.GetLogs();
for (int N = 0; N < LogV.Len(); N++)
FolderNmV.Add(LogV[N].GetFolderName());
}
}
void TAmazonItemBs::GetVoidItemIdV(TStrV& VoidItemIdV) const {
VoidItemIdV.Clr();
for (int ItemIdN=0; ItemIdN<GetItems(); ItemIdN++){
PAmazonItem Item=GetItem(ItemIdN);
for (int NextItemIdN=0; NextItemIdN<Item->GetNextItemIds(); NextItemIdN++){
TStr NextItemId=Item->GetNextItemId(NextItemIdN);
if (!IsItem(NextItemId)){VoidItemIdV.AddUnique(NextItemId);}
}
}
}
void TNytNGramBs::GetNGramStrV(const TStr& HtmlStr, TStrV& NGramStrV){
NGramStrV.Clr();
// prepare html parsing
PSIn HtmlSIn=TStrIn::New(HtmlStr);
THtmlLx HtmlLx(HtmlSIn);
// process text
TStrV StrV;
while (HtmlLx.GetSym()!=hsyEof){
TStr Str=HtmlLx.ChA;
Str.ToLc();
switch (HtmlLx.Sym){
case hsyUndef:
case hsyUrl:
case hsyMTag:
case hsySSym:
StrV.Clr();
break;
case hsyStr:
case hsyNum:
NGramStrV.Add(Str);
StrV.Add(Str);
for (int NGramLen=2; NGramLen<=4; NGramLen++){
if (StrV.Len()<NGramLen){break;}
TStrV TermStrV(NGramLen, 0);
for (int StrN=StrV.Len()-NGramLen; StrN<StrV.Len(); StrN++){
TermStrV.Add(StrV[StrN]);
}
int NGramId;
if (IsNGram(TermStrV, NGramId)){
TStr NGramStr=GetNGramStr(NGramId);
NGramStrV.Add(NGramStr);
}
}
break;
case hsyBTag:
case hsyETag:
StrV.Clr();
break;
case hsyEof: break;
default: Fail;
}
}
}
void TEnv::GetVarNmV(TStrV& VarNmV) {
VarNmV.Clr();
int VarN=0;
while (_environ[VarN]!=NULL) {
TStr VarNmVal=_environ[VarN++];
TStr VarNm;
TStr VarVal;
VarNmVal.SplitOnCh(VarNm, '=', VarVal);
VarNmV.Add(VarNm);
}
}
void TFFile::GetFNmV(
const TStr& FPath, const TStrV& FExtV, const bool& RecurseP, TStrV& FNmV){
// prepare file-directory traversal
TStrV FPathV; FPathV.Add(FPath);
TFFile FFile(FPathV, FExtV, "", RecurseP); TStr FNm;
// traverse directory
FNmV.Clr();
while (FFile.Next(FNm)){
FNmV.Add(FNm);
}
}
void TTransCorpus::SaveTxt(const TStr& OutFBase, const TStr& OutOrgFNm,
const TStr& OutTransFNm, const TStr& OutRefTransFNm, TStrV& OrgFNmV,
TStrV& TransFNmV, TStrV& RefTransFNmV, const int& LinesPerFile) {
// prepare filenames
OrgFNmV.Clr(); TransFNmV.Clr(); RefTransFNmV.Clr();
if (!OutOrgFNm.Empty()) { OrgFNmV.Add(GetOutFNm(OutFBase, 0, LinesPerFile, OutOrgFNm)); }
if (!OutTransFNm.Empty()) { TransFNmV.Add(GetOutFNm(OutFBase, 0, LinesPerFile, OutTransFNm)); }
if (!OutRefTransFNm.Empty()) { RefTransFNmV.Add(GetOutFNm(OutFBase, 0, LinesPerFile, OutRefTransFNm)); }
// open files
PSOut OrgSOut = !OutOrgFNm.Empty() ? TFOut::New(OrgFNmV.Last()) : PSOut();
PSOut TransSOut = !OutTransFNm.Empty() ? TFOut::New(TransFNmV.Last()) : PSOut();
PSOut RefTransSOut = !OutRefTransFNm.Empty() ? TFOut::New(RefTransFNmV.Last()) : PSOut();
// check which are given
const bool IsOrgP = !OrgSOut.Empty() && IsOrg();
const bool IsTransP = !TransSOut.Empty() && IsTrans();
const bool IsRefTransP = !RefTransSOut.Empty() && IsRefTrans();
// print warnings
if (!IsOrgP) { printf("No original sentences!\n"); }
if (!IsTransP) { printf("No machine translation sentences!\n"); }
if (!IsRefTransP) { printf("No reference translation sentences!\n"); }
// go over all the sentences and store them in the file
TIntV SentIdV; GetSentIdV(SentIdV);
for (int SentIdN = 0; SentIdN < SentIdV.Len(); SentIdN++) {
const int SentId = SentIdV[SentIdN];
if (IsOrgP) { OrgSOut->PutStrLn(GetOrgStr(SentId)); }
if (IsTransP) { TransSOut->PutStrLn(GetTransStr(SentId)); }
if (IsRefTransP) { RefTransSOut->PutStrLn(GetRefTransStrV(SentId)[0]); }
// should we break and go to next file?
if ((LinesPerFile!=-1) && (SentIdN>0) && (SentIdN % LinesPerFile == 0)) {
// prepare filenames
if (!OutOrgFNm.Empty()) { OrgFNmV.Add(GetOutFNm(OutFBase, SentIdN, LinesPerFile, OutOrgFNm)); }
if (!OutTransFNm.Empty()) { TransFNmV.Add(GetOutFNm(OutFBase, SentIdN, LinesPerFile, OutTransFNm)); }
if (!OutRefTransFNm.Empty()) { RefTransFNmV.Add(GetOutFNm(OutFBase, SentIdN, LinesPerFile, OutRefTransFNm)); }
// open next files files
OrgSOut = !OutOrgFNm.Empty() ? TFOut::New(OrgFNmV.Last()) : PSOut();
TransSOut = !OutTransFNm.Empty() ? TFOut::New(TransFNmV.Last()) : PSOut();
RefTransSOut = !OutRefTransFNm.Empty() ? TFOut::New(RefTransFNmV.Last()) : PSOut();
}
}
}
/////////////////////////////////////////////////
// Translation-Evaluation-Scores
void TEvalScore::Tokenize(const TStr& Str, TStrV& TokenV) {
TStr LcStr = GetLc(Str);
TokenV.Clr(); TChA WordChA;
const int Chs = LcStr.Len();
for (int ChN = 0; ChN < Chs; ChN++) {
const char Ch = LcStr[ChN];
if (IsChar(Ch)) {
WordChA += Ch;
} else if (!WordChA.Empty()) {
TokenV.Add(WordChA);
WordChA.Clr();
}
}
}
int main(){
TTableContext Context;
// Case 1: Euclidean Distance
Schema BuildingS;
BuildingS.Add(TPair<TStr,TAttrType>("Building", atStr));
BuildingS.Add(TPair<TStr,TAttrType>("X", atInt));
BuildingS.Add(TPair<TStr,TAttrType>("Y", atInt));
// create table
PTable TBuildings = TTable::LoadSS("Buildings", BuildingS, "tests/buildings.txt", Context, '\t', false);
TStrV Cols;
Cols.Add("X");
Cols.Add("Y");
// Find all buildings within 5 Euc Distance of each other.
PTable BuildingJointTable = TBuildings->SelfSimJoin(Cols, "Euclidean_Distance", L2Norm, 5.0);
BuildingJointTable->SaveSS("tests/buildings.out.txt");
// Case2 : Haversine distance
Schema PlaceS;
PlaceS.Add(TPair<TStr,TAttrType>("Name", atStr));
PlaceS.Add(TPair<TStr,TAttrType>("Location", atStr));
PlaceS.Add(TPair<TStr,TAttrType>("Latitude", atFlt));
PlaceS.Add(TPair<TStr,TAttrType>("Longitude", atFlt));
// create table
PTable TPlaces = TTable::LoadSS("Places", PlaceS, "tests/places.txt", Context, '\t', false);
Cols.Clr();
Cols.Add("Latitude");
Cols.Add("Longitude");
PTable PlacesJointTable = TPlaces->SelfSimJoin(Cols, "Distance",Haversine, 1000.0);
TStrV ProjectionV;
ProjectionV.Add("Places_1.Name");
ProjectionV.Add("Places_1.Location");
ProjectionV.Add("Places_2.Name");
ProjectionV.Add("Places_2.Location");
ProjectionV.Add("Distance");
PlacesJointTable->ProjectInPlace(ProjectionV);
PlacesJointTable->SelectAtomic("Places_1.Name", "Places_2.Name", NEQ);
PlacesJointTable->SaveSS("tests/places.out.txt");
printf("Saved buildings.out.txt and places.out.txt\n");
return 0;
}
void TRegKey::GetKeyNmV(TStrV& KeyNmV) const {
KeyNmV.Clr();
if (!Ok){return;}
// get subkey count
DWORD SubKeys; // number of subkeys
DWORD MxSubKeyNmLen; // longest subkey size
DWORD RetCd=RegQueryInfoKey(
hKey, // key handle
NULL, // buffer for class name
NULL, // length of class string
NULL, // reserved
&SubKeys, // number of subkeys
&MxSubKeyNmLen, // longest subkey size
NULL, // longest class string
NULL, // number of values for this key
NULL, // longest value name
NULL, // longest value data
NULL, // security descriptor
NULL); // last write time
if (RetCd!=ERROR_SUCCESS){return;}
// retrieve subkey-names
if (SubKeys>0){
KeyNmV.Gen(SubKeys, 0);
char* SubKeyNmCStr=new char[MxSubKeyNmLen+1];
DWORD SubKeyN=0;
forever{
DWORD SubKeyNmCStrLen=MxSubKeyNmLen+1;
DWORD RetCd=RegEnumKeyEx(
hKey, // handle of key to enumerate
SubKeyN, // index of subkey to enumerate
SubKeyNmCStr, // address of buffer for subkey name
&SubKeyNmCStrLen, // address for size of subkey buffer
NULL, // reserved
NULL, // address of buffer for class string
NULL, // address for size of class buffer
NULL); // address for time key last written to
if (RetCd==ERROR_SUCCESS){
TStr KeyNm(SubKeyNmCStr);
KeyNmV.Add(KeyNm);
} else {
break;
}
SubKeyN++;
}
delete[] SubKeyNmCStr;
}
void TGnuPlot::LoadTs(const TStr& FNm, TStrV& ColNmV, TVec<TFltKdV>& ColV) {
PSs Ss = TSs::LoadTxt(ssfTabSep, FNm);
int row = 0;
ColNmV.Clr();
while (Ss->At(0, row)[0] == '#') { row++; }
for (int c = 1; c < Ss->GetXLen(row); c+=2) {
ColNmV.Add(Ss->At(c, row));
}
row++;
ColV.Gen(ColNmV.Len(), ColNmV.Len());
for (; row < Ss->GetYLen(); row++) {
for (int c = 0; c < Ss->GetXLen(row); c+=2) {
if (Ss->At(c,row).Empty()) break;
ColV[c/2].Add(TFltKd(Ss->At(c,row).GetFlt(), Ss->At(c+1,row).GetFlt()));
}
}
}
///////////////////////////////
// Tokenizer-Simple
void TTokenizerSimple::GetTokens(const PSIn& SIn, TStrV& TokenV) const {
TStr LineStr; TStrV WordStrV;
while (SIn->GetNextLn(LineStr)) {
WordStrV.Clr(false);
LineStr.SplitOnAllAnyCh(" .,!?\n\r()+=-{}[]%$#@\\/", WordStrV, true);
for (int WordStrN = 0; WordStrN < WordStrV.Len(); WordStrN++) {
const TStr& WordStr = WordStrV[WordStrN];
const TStr UcStr = WordStr.GetUc();
if (SwSet.Empty() || (!SwSet->IsIn(UcStr))) {
TStr TokenStr = ToUcP ? UcStr : WordStr;
if (!Stemmer.Empty()) {
TokenStr = Stemmer->GetStem(TokenStr); }
TokenV.Add(TokenStr);
}
}
}
}
void TOdbcDb::GetTbNmV(TStrV& TbNmV) const {
#define ODBC_STR_LEN 254+1
SQLCHAR szCatalog[ODBC_STR_LEN], szSchema[ODBC_STR_LEN];
SQLCHAR szTableName[ODBC_STR_LEN], szTableTypeName[ODBC_STR_LEN];
SQLCHAR szRemarks[ODBC_STR_LEN];
SQLHSTMT StmtHnd;
SQLAllocHandle(SQL_HANDLE_STMT, DbcHnd, &StmtHnd);
/* Declare buffers for bytes available to return */
SQLLEN cbCatalog, cbSchema, cbTableName, cbTableTypeName, cbRemarks;
SQLRETURN RetCd=SQLTables(StmtHnd,
NULL, 0, /* All catalogs */
NULL, 0, /* All schemas */
NULL, 0, /* All tables */
NULL, 0); /* All table-typescolumns */
TbNmV.Clr();
if (RetCd == SQL_SUCCESS || RetCd == SQL_SUCCESS_WITH_INFO){
/* Bind columns in result set to buffers */
SQLBindCol(StmtHnd, 1, SQL_C_CHAR, szCatalog, ODBC_STR_LEN, &cbCatalog);
SQLBindCol(StmtHnd, 2, SQL_C_CHAR, szSchema, ODBC_STR_LEN, &cbSchema);
SQLBindCol(StmtHnd, 3, SQL_C_CHAR, szTableName, ODBC_STR_LEN, &cbTableName);
SQLBindCol(StmtHnd, 4, SQL_C_CHAR, szTableTypeName, ODBC_STR_LEN, &cbTableTypeName);
SQLBindCol(StmtHnd, 5, SQL_C_CHAR, szRemarks, ODBC_STR_LEN, &cbRemarks);
//
forever{
RetCd=SQLFetch(StmtHnd);
if (RetCd==SQL_SUCCESS || RetCd==SQL_SUCCESS_WITH_INFO){
TbNmV.Add((char*)szTableName);
//printf("[%s]", szTableName);
} else {
break;
}
}
}
void TNmObjBs::ExtrCandWordStrV(
const TStr& HtmlStr, TStrV& CandWordStrV, const bool& DumpP){
// prepare named-object vector
CandWordStrV.Clr();
// prepare html parsing
PSIn HtmlSIn=TStrIn::New(HtmlStr);
PHtmlDoc HtmlDoc=THtmlDoc::New(HtmlSIn, hdtAll, false);
PHtmlTok Tok;
THtmlLxSym Sym; TStr Str; TStr NrStr;
CandWordStrV.Add(PeriodTagStr);
bool InTitle=false; bool InScript=false; int LastNmObjTokN=-1;
// traverse html tokens
if (DumpP){printf("Tokens: ");}
for (int TokN=0; TokN<HtmlDoc->GetToks(); TokN++){
PHtmlTok Tok=HtmlDoc->GetTok(TokN);
HtmlDoc->GetTok(TokN, Sym, Str);
switch (Sym){
case hsyUndef: break;
case hsyStr:
case hsyNum:
if (InTitle){break;}
if (InScript){break;}
NrStr=GetNrWordStr(Str);
if (DumpP){
if (Str==NrStr){printf("%s ", Str.CStr());}
else {printf("%s(%s) ", Str.CStr(), NrStr.CStr());}
}
if (IsFirstCapWordStr(NrStr)||IsNmObjAttr(NrStr, noaAsCapitalized)){
if ((LastNmObjTokN!=-1)&&(LastNmObjTokN<TokN-1)){
if (CandWordStrV.Last()!=PeriodTagStr){
CandWordStrV.Add(BreakTagStr);
}
}
CandWordStrV.Add(NrStr); LastNmObjTokN=TokN;
}
break;
case hsySSym:
if (InTitle){break;}
if (InScript){break;}
if (DumpP){
printf("%s ", Str.CStr());}
if (
(Str==".")||(Str=="!")||(Str=="?")||
(Str=="\"")||(Str=="-")||(Str=="/")||
(Str==":")||(Str==";")){
if (CandWordStrV.Last()!=PeriodTagStr){
CandWordStrV.Add(PeriodTagStr);
}
}
break;
case hsyBTag:
case hsyETag:
if (Str=="<TITLE>"){
InTitle=(Sym==hsyBTag);
} else
if (Str=="<SCRIPT>"){
InScript=(Sym==hsyBTag);
} else
if (Str=="<P>"){
if ((!CandWordStrV.Empty())&&(CandWordStrV.Last()!=ParagraphTagStr)){
CandWordStrV.Add(ParagraphTagStr);
CandWordStrV.Add(PeriodTagStr);
}
} else
if ((Str=="<TD>")||(Str=="<BR>")){
CandWordStrV.Add(PeriodTagStr);
}
break;
case hsyEof: break;
default: break;
}
}
CandWordStrV.Add(EofTagStr);
if (DumpP){printf("\n");}
if (DumpP){
printf("Candidates: ");
for (int CandWordStrN=0; CandWordStrN<CandWordStrV.Len(); CandWordStrN++){
printf("%s ", CandWordStrV[CandWordStrN].CStr());}
printf("\n");
}
}
void TStemmer::GetStemmerTypeNmV(TStrV& StemmerTypeNmV, TStrV& StemmerTypeDNmV) {
StemmerTypeNmV.Clr(); StemmerTypeDNmV.Clr();
StemmerTypeNmV.Add("none"); StemmerTypeDNmV.Add("None");
StemmerTypeNmV.Add("porter"); StemmerTypeDNmV.Add("English-Porter");
}
