void SearcherTest::testSearcherFullField(void)
{
UnicodeString strQuery;
const UChar *uStr;
// Populate the database
Entry e;
EntrySet entrySet;
UnicodeString word = "A full field";
// Add one word and entry
e.Set(1, 1, 1);
entrySet.insert(e);
mDbNdx->AddWordEntry(word, e);
// Search the database for a field only
// This will use Lexer::ReadFullField as the field is in <quotes>
strQuery = "<A full field>";
uStr = strQuery.getTerminatedBuffer();
mSearcher->SetQuery(cStrField, uStr);
mSearcher->Compute();
CPPUNIT_ASSERT(entrySet == *(mSearcher->GetEntrySet()));
// Add a new entry to a word
e.Set(2, 2, 2);
word = "test";
entrySet.insert(e);
mDbNdx->AddWordEntry(word, e);
// Search the database for a field or a word
strQuery = "<A full field> test";
uStr = strQuery.getTerminatedBuffer();
mSearcher->SetQuery(cStrField, uStr);
mSearcher->Compute();
CPPUNIT_ASSERT(entrySet == *(mSearcher->GetEntrySet()));
}
void SearcherTest::testSearcherTerm(void)
{
UnicodeString strQuery;
const UChar *uStr;
// Populate the database
Entry e;
EntrySet entrySet;
UnicodeString word = "test";
// Add one word and entry
e.Set(1, 1, 1);
entrySet.insert(e);
mDbNdx->AddWordEntry(word, e);
// Add a new entry to this word
e.Set(2, 2, 2);
entrySet.insert(e);
mDbNdx->AddWordEntry(word, e);
// Search the database
strQuery = "test";
uStr = strQuery.getTerminatedBuffer();
mSearcher->SetQuery(cStrField, uStr);
mSearcher->Compute();
CPPUNIT_ASSERT(entrySet == *(mSearcher->GetEntrySet()));
}
void SearcherTest::testSearcherRange(void)
{
UnicodeString strQuery;
const UChar *uStr;
// Populate the database
Entry e;
EntrySet entrySet;
UnicodeString word;
// Add one word and entry
e.Set(1, 1, 1);
word = "a";
entrySet.insert(e);
mDbNdx->AddWordEntry(word, e);
// Add a new word and entry
e.Set(2, 2, 2);
word = "b";
entrySet.insert(e);
mDbNdx->AddWordEntry(word, e);
// Add a new word and entry
e.Set(3, 3, 3);
word = "c";
entrySet.insert(e);
mDbNdx->AddWordEntry(word, e);
// Search the database
strQuery = "[a TO c]";
uStr = strQuery.getTerminatedBuffer();
mSearcher->SetQuery(cStrField, uStr);
mSearcher->Compute();
CPPUNIT_ASSERT_MESSAGE("[a TO c]", entrySet == *(mSearcher->GetEntrySet()));
// Add a new word and entry
e.Set(4, 4, 4);
word = "d";
mDbNdx->AddWordEntry(word, e);
// Search the database (without "TO")
strQuery = "[a c]";
uStr = strQuery.getTerminatedBuffer();
mSearcher->SetQuery(cStrField, uStr);
mSearcher->Compute();
CPPUNIT_ASSERT_MESSAGE("[a c]", entrySet == *(mSearcher->GetEntrySet()));
// Keep only the "b" in the result set
entrySet.clear();
e.Set(2, 2, 2);
entrySet.insert(e);
// Search the database
strQuery = "{a TO c}";
uStr = strQuery.getTerminatedBuffer();
mSearcher->SetQuery(cStrField, uStr);
mSearcher->Compute();
CPPUNIT_ASSERT_MESSAGE("{a TO c}", entrySet == *(mSearcher->GetEntrySet()));
}
void EntryTest::testEmpty(void)
{
Entry e;
CPPUNIT_ASSERT(e.IsEmpty());
e.Set(0, 0, 0);
CPPUNIT_ASSERT(e.IsEmpty());
e.Set(0, 0, 1);
CPPUNIT_ASSERT(e.IsEmpty() == false);
e.Set(0, 1, 0);
CPPUNIT_ASSERT(e.IsEmpty() == false);
e.Set(1, 0, 0);
CPPUNIT_ASSERT(e.IsEmpty() == false);
}
void SearcherTest::testSearcherPrefix(void)
{
bool isLogError;
UnicodeString strQuery;
const UChar *uStr;
// Populate the database
Entry e;
EntrySet entrySet;
UnicodeString word = "test";
// Add one word and entry
e.Set(1, 1, 1);
entrySet.insert(e);
mDbNdx->AddWordEntry(word, e);
// Add a new entry to this word
e.Set(2, 2, 2);
entrySet.insert(e);
mDbNdx->AddWordEntry(word, e);
// Search the database
strQuery = cStrField;
strQuery += ":test";
uStr = strQuery.getTerminatedBuffer();
mSearcher->SetQuery(cStrField, uStr);
mSearcher->Compute();
CPPUNIT_ASSERT(entrySet == *(mSearcher->GetEntrySet()));
// Search with a wrong prefix
isLogError = gLog.isInLogLevel(eTypLogError);
gLog.removeLogLevel(eTypLogError); // Remove errors
entrySet.clear();
strQuery = "wrong_prefix:test";
uStr = strQuery.getTerminatedBuffer();
mSearcher->SetQuery(cStrField, uStr);
mSearcher->Compute();
CPPUNIT_ASSERT(entrySet == *(mSearcher->GetEntrySet()));
strQuery = "wrong_prefix:test*";
uStr = strQuery.getTerminatedBuffer();
mSearcher->SetQuery(cStrField, uStr);
mSearcher->Compute();
CPPUNIT_ASSERT(entrySet == *(mSearcher->GetEntrySet()));
if (isLogError)
gLog.addLogLevel(eTypLogError); // Show errors
}
void EntryTest::testEntrySet(void)
{
Entry e;
EntrySet set1, set2;
e.Set(1, 0, 0);
set1.insert(e);
e.Set(111, 222, 333);
set1.insert(e);
e.Set(DEF_IntMax, DEF_LongMax, DEF_LongMax);
set1.insert(e);
// Binary storage into flat buffer
size_t dataSize;
BinaryBuffer buffer;
size_t nb1 = EntrySetToBuf(buffer, dataSize, set1);
size_t nb2 = BufToEntrySet(set2, buffer.getBuffer(), buffer.getOccupancy(), NULL);
size_t nb3 = GetBufEntrySetCount(buffer.getBuffer());
// Check the size of the sets
CPPUNIT_ASSERT(nb1 == nb2);
CPPUNIT_ASSERT(set1.size() == nb1);
CPPUNIT_ASSERT(set2.size() == nb2);
CPPUNIT_ASSERT(nb2 == nb3);
// Check the content of the sets
Entry e1, e2;
EntrySet::iterator it1, it2; // Set iterator
for (it1 = set1.begin(), it2 = set2.begin(); it1 != set1.end(); ++it1, ++it2) {
e1 = (*it1);
e2 = (*it2);
CPPUNIT_ASSERT(e1 == e2);
}
// Check the binary storage of an empty set
EntrySet set;
EntrySetToBuf(buffer, dataSize, set);
size_t nb = BufToEntrySet(set, buffer.getBuffer(), buffer.getOccupancy(), NULL);
CPPUNIT_ASSERT(nb == 0);
}
void SearcherTest::testSearcherFuzzy(void)
{
UnicodeString strQuery;
const UChar *uStr;
// Populate the database
Entry e;
EntrySet entrySet;
UnicodeString word = "test1";
// Add one word and entry
e.Set(1, 1, 0);
entrySet.insert(e);
mDbNdx->AddWordEntry(word, e);
// Search the database
strQuery = "test~";
uStr = strQuery.getTerminatedBuffer();
mSearcher->SetQuery(cStrField, uStr);
mSearcher->Compute();
CPPUNIT_ASSERT(entrySet == *(mSearcher->GetEntrySet()));
entrySet.clear();
// Add a new word and entry
e.Set(1, 1, 1);
entrySet.insert(e);
word = "bate";
mDbNdx->AddWordEntry(word, e);
// Add a new word and entry
e.Set(1, 1, 2);
entrySet.insert(e);
word = "fate";
mDbNdx->AddWordEntry(word, e);
// Add a new word and entry
e.Set(1, 1, 3);
entrySet.insert(e);
word = "tate";
mDbNdx->AddWordEntry(word, e);
// Add a new word and entry
e.Set(1, 1, 4);
entrySet.insert(e);
word = "tatu";
mDbNdx->AddWordEntry(word, e);
// Search the database
strQuery = "tate~";
uStr = strQuery.getTerminatedBuffer();
mSearcher->SetQuery(cStrField, uStr);
mSearcher->Compute();
CPPUNIT_ASSERT(entrySet == *(mSearcher->GetEntrySet()));
// Add a new word and entry
e.Set(1, 1, 5);
word = "baty";
mDbNdx->AddWordEntry(word, e);
// Search the database
strQuery = "tate~";
uStr = strQuery.getTerminatedBuffer();
mSearcher->SetQuery(cStrField, uStr);
mSearcher->Compute();
CPPUNIT_ASSERT(entrySet == *(mSearcher->GetEntrySet()));
}
void SearcherTest::testSearcherWildcard(void)
{
UnicodeString strQuery;
const UChar *uStr;
// Populate the database
Entry e;
EntrySet entrySet;
UnicodeString word;
// Add one word and entry
word = "test";
e.Set(1, 1, 1);
entrySet.insert(e);
mDbNdx->AddWordEntry(word, e);
// Add one word and entry
word = "testa";
e.Set(1, 1, 2);
entrySet.insert(e);
mDbNdx->AddWordEntry(word, e);
// Add one word and entry
word = "testb";
e.Set(1, 1, 3);
entrySet.insert(e);
mDbNdx->AddWordEntry(word, e);
// Search the database
strQuery = "test*";
uStr = strQuery.getTerminatedBuffer();
mSearcher->SetQuery(cStrField, uStr);
mSearcher->Compute();
CPPUNIT_ASSERT(entrySet == *(mSearcher->GetEntrySet()));
// Add a new word and entry
e.Set(1, 1, 2);
word = "tes";
mDbNdx->AddWordEntry(word, e);
// Add a new entry to this word
e.Set(2, 2, 2);
mDbNdx->AddWordEntry(word, e);
// Search the database
strQuery = "test*";
uStr = strQuery.getTerminatedBuffer();
mSearcher->SetQuery(cStrField, uStr);
mSearcher->Compute();
CPPUNIT_ASSERT(entrySet == *(mSearcher->GetEntrySet()));
// Add a new word and entry
e.Set(1, 1, 3);
entrySet.insert(e);
word = "test3";
mDbNdx->AddWordEntry(word, e);
// Add a new entry to this word
e.Set(2, 2, 2);
entrySet.insert(e);
mDbNdx->AddWordEntry(word, e);
// Add a new entry to this word
e.Set(3, 3, 3);
entrySet.insert(e);
mDbNdx->AddWordEntry(word, e);
// Search the database
strQuery = "test*";
uStr = strQuery.getTerminatedBuffer();
mSearcher->SetQuery(cStrField, uStr);
mSearcher->Compute();
CPPUNIT_ASSERT(entrySet == *(mSearcher->GetEntrySet()));
// Test with accents
entrySet.clear();
// Add a new word and entry
e.Set(1, 1, 1);
entrySet.insert(e);
word = "bateaux";
mDbNdx->AddWordEntry(word, e);
// Add a new word and entry
e.Set(1, 1, 2);
entrySet.insert(e);
UChar s1[] = { 'b', 0x00E2, 't', 'i', 0 }; // "bâti"
word = s1;
mDbNdx->AddWordEntry(word, e);
// Add a new word and entry
e.Set(1, 1, 3);
entrySet.insert(e);
word = "bats";
mDbNdx->AddWordEntry(word, e);
// Search the database
strQuery = "bat*";
uStr = strQuery.getTerminatedBuffer();
mSearcher->SetQuery(cStrField, uStr);
mSearcher->Compute();
CPPUNIT_ASSERT_MESSAGE("Wildcard query with accents", entrySet == *(mSearcher->GetEntrySet()));
// Test with wildcard in the middle
entrySet.clear();
// Add a new word and entry
e.Set(1, 1, 1);
entrySet.insert(e);
UChar s2[] = { 'b', 0x00E9, 'n', 'e', 'd', 'i', 'c', 't', 'i', 'o', 'n', 0 }; // "bénediction"
word = s2;
mDbNdx->AddWordEntry(word, e);
// Add a new word and entry
e.Set(1, 1, 2);
UChar s3[] = { 'b', 0x00E9, 'n', 0x00E9, 'f', 'i', 'c', 'e', 0 }; // "bénéfice"
word = s3;
mDbNdx->AddWordEntry(word, e);
// Add a new word and entry
e.Set(1, 1, 3);
entrySet.insert(e);
word = "benjamin";
mDbNdx->AddWordEntry(word, e);
// Search the database
strQuery = "ben*n";
uStr = strQuery.getTerminatedBuffer();
mSearcher->SetQuery(cStrField, uStr);
mSearcher->Compute();
CPPUNIT_ASSERT_MESSAGE("Wildcard query with accents", entrySet == *(mSearcher->GetEntrySet()));
}
void SearcherTest::testSearcherPhraseSlop(void)
{
UnicodeString strQuery;
const UChar *uStr;
// Populate the database
Entry e;
EntrySet entrySet;
UnicodeString word = "test1";
// Add one word and entry
e.Set(1, 1, 1);
entrySet.insert(e);
// Add one word and entry
mDbNdx->AddWordEntry(word, e);
// Add a new word and entry
e.Set(1, 1, 2);
word = "test2";
entrySet.insert(e);
mDbNdx->AddWordEntry(word, e);
// Add a new entry to this word
e.Set(2, 2, 2);
mDbNdx->AddWordEntry(word, e);
// Search the database
strQuery = "\"test1 test2\"~2";
uStr = strQuery.getTerminatedBuffer();
mSearcher->SetQuery(cStrField, uStr);
mSearcher->Compute();
CPPUNIT_ASSERT(entrySet == *(mSearcher->GetEntrySet()));
// Add a new word and entry
e.Set(1, 1, 3);
word = "test3";
entrySet.insert(e);
mDbNdx->AddWordEntry(word, e);
// Add a new entry to this word
e.Set(1, 1, 5);
mDbNdx->AddWordEntry(word, e);
// Add a new entry to this word
e.Set(3, 3, 3);
mDbNdx->AddWordEntry(word, e);
// Search the database
strQuery = "\"test1 test2 test3\"~3";
uStr = strQuery.getTerminatedBuffer();
mSearcher->SetQuery(cStrField, uStr);
mSearcher->Compute();
CPPUNIT_ASSERT(entrySet == *(mSearcher->GetEntrySet()));
// Search the database
e.Set(1, 1, 5);
entrySet.insert(e);
strQuery = "\"test1 test2 test3\"~10";
uStr = strQuery.getTerminatedBuffer();
mSearcher->SetQuery(cStrField, uStr);
mSearcher->Compute();
CPPUNIT_ASSERT(entrySet == *(mSearcher->GetEntrySet()));
// Add a new word and entry
e.Set(1, 1, 4);
word = "test4";
entrySet.insert(e);
mDbNdx->AddWordEntry(word, e);
// Search the database
strQuery = "\"test4 test3 test2 test1\"~4";
uStr = strQuery.getTerminatedBuffer();
mSearcher->SetQuery(cStrField, uStr);
mSearcher->Compute();
CPPUNIT_ASSERT(entrySet == *(mSearcher->GetEntrySet()));
// Add a new word and entry
e.Set(1, 1, 50);
word = "test4";
mDbNdx->AddWordEntry(word, e);
// Search the database
strQuery = "\"test1 test2 test3 test4\"~40";
uStr = strQuery.getTerminatedBuffer();
mSearcher->SetQuery(cStrField, uStr);
mSearcher->Compute();
CPPUNIT_ASSERT(entrySet == *(mSearcher->GetEntrySet()));
// Search the database
/*entrySet.clear();
strQuery = "\"test1 test2 test3 test4\"~3";
uStr = strQuery.getTerminatedBuffer();
mSearcher->SetQuery(cStrField, uStr);
mSearcher->Compute();
CPPUNIT_ASSERT(entrySet == *(mSearcher->GetEntrySet()));*/
}
CINI::CINI(const char *buffer)
{
CSyntax syntax(buffer, strlen(buffer));
Entry *section = NULL;
do
{
size_t len;
const char *token = syntax.ParseToken( &len );
if ( len == 1 )
{
if ( *token == '[' )
{
char *name;
unsigned long offset = syntax.GetOffset();
// Read section and select it
if ( !syntax.ScanCharacterEx( ']', true, true, false ) )
continue;
size_t tokLen = syntax.GetOffset() - offset - 1;
name = (char*)malloc( tokLen + 1 );
strncpy( name, token + 1, tokLen );
name[tokLen] = 0;
section = new Entry( name );
entries.push_back( section );
}
else if ( *token == ';' || *token == '#' )
{
syntax.ScanCharacter( '\n' );
syntax.Seek( -1 );
continue;
}
else
goto sectionDo;
}
else if ( section )
{
sectionDo:
char name[256];
const char *set;
len = std::min(len, (size_t)255);
strncpy(name, token, len );
name[len] = 0;
set = syntax.ParseToken( &len );
if ( len != 1 || *set != '=' )
continue;
// Read everything until new line.
size_t contentSize;
const char *lineBuf = syntax.ReadUntilNewLine( &contentSize );
std::string strbuf( lineBuf, contentSize );
section->Set( name, strbuf.c_str() );
}
} while ( syntax.GotoNewLine() );
}
