void DateTimeIndexTestCase::buildDateTimeIndex(const string& sDocs)
{
try
{
DocumentSchema schema;
schema.addField("DateTime1", "DATETIME_I", true);
Index index;
index.open(getIndexPath(), Index::WRITE, &schema);
IndexWriterPtr pIndexWriter = index.acquireWriter();
StringTokenizer st(sDocs, ";", StringTokenizer::TOKEN_TRIM |
StringTokenizer::TOKEN_IGNORE_EMPTY);
for (StringTokenizer::Iterator it = st.begin(); it != st.end(); ++it)
{
DocumentPtr pDoc = new Document(pIndexWriter->getDocSchema());
pDoc->addField(0, (*it).c_str());
pIndexWriter->addDocument(pDoc);
}
pIndexWriter->close();
}
catch (const FirteXException& )
{
CPPUNIT_ASSERT(false);
}
}
void IndexTestCase::testIndexOptimize()
{
GLOBAL_CONF().Merge.maxAllowedOpenFiles = 16;
DocumentSchema schema;
schema.addTextField("BODY");
stringstream ss;
const size_t NUM_DOCS = 1000;
for (size_t i = 0; i < NUM_DOCS; ++i)
{
ss << "body" << i << " hot;";
}
try
{
buildIndex(schema, ss.str());
buildIndex(schema, ss.str(), true);
}
catch (const FirteXException& e)
{
cout << "ERROR: " << e.what() << endl;
CPPUNIT_ASSERT(false);
}
tstring str = getTestPath();
{
Index index;
index.open(str, Index::APPEND, NULL);
IndexWriterPtr pIndexWriter = index.acquireWriter();
CPPUNIT_ASSERT(pIndexWriter != NULL);
try
{
pIndexWriter->optimizeIndex();
}
catch (const FirteXException& e)
{
cout << "ERROR: " << e.what() << endl;
CPPUNIT_ASSERT(false);
}
}
Index index;
try
{
index.open(str, Index::READ, NULL);
IndexReaderPtr pIndexReader = index.acquireReader();
checkDocFreq(pIndexReader, "BODY", "hot", 2 * NUM_DOCS);
}
catch (const FirteXException& e)
{
cout << "ERROR: " << e.what() << endl;
CPPUNIT_ASSERT(false);
}
}
void IndexTestCase::testDocumentDeletion()
{
DocumentSchema schema;
schema.addField("URL", "PRIMARY_KEY", true);
schema.addTextField("BODY");
schema.addField("MODIFIED", "INT64", true);
stringstream ss1;
const size_t NUM_DOCS = 1000;
size_t i = 0;
for (; i < NUM_DOCS; ++i)
{
ss1 << "url" << i << ", body" << i << " hot,"
<< (i * 100) % 1000 << ";";
}
buildIndex(schema, ss1.str());
stringstream ss2;
for (; i < 2 * NUM_DOCS; ++i)
{
ss2 << "url" << i << ", body" << i << " hot,"
<< (i * 100) % 1000 << ";";
}
buildIndex(schema, ss2.str(), true);
StandardAnalyzerPtr sa(new StandardAnalyzer());
sa->init();
TokenViewPtr pTokens = sa->tokenize("hot", 3);
CPPUNIT_ASSERT(pTokens);
CPPUNIT_ASSERT(pTokens->getNumTokens() == 1);
TokenView::Iterator it = pTokens->iterator();
TermPtr pTerm(new Term("BODY", it.next().getTextValue()));
tstring str = getTestPath();
std::set<docid_t> answer;
{
Index index;
index.open(str, Index::RDWR, NULL);
IndexWriterPtr pIndexWriter = index.acquireWriter();
CPPUNIT_ASSERT(pIndexWriter != NULL);
IndexReaderPtr pIndexReader = index.acquireReader();
CPPUNIT_ASSERT(pIndexReader != NULL);
for (size_t i = 0; i < 2 * NUM_DOCS; ++i)
{
stringstream ss;
ss << "url" << i;
if (i == 1000 || i == 1500 || i == 1505 || i == 1999)
{
pIndexWriter->deleteDocument(ss.str());
}
else
{
TermReaderPtr pTermReader = pIndexReader->termReader();
TermPtr pTerm(new Term("URL", ss.str()));
TermPostingIteratorPtr pIt = pTermReader->seek(pTerm.get());
docid_t docId = pIt->skipTo(0);
answer.insert(docId);
}
}
TermReaderPtr pTermReader = pIndexReader->termReader();
TermPostingIteratorPtr pDocFreqs = pTermReader->seek(pTerm.get());
CPPUNIT_ASSERT(pDocFreqs);
CPPUNIT_ASSERT_EQUAL((df_t)NUM_DOCS * 2, pDocFreqs->getTermMeta().getDocFreq());
std::set<docid_t>::const_iterator it = answer.begin();
for (docid_t i = 0; i < (docid_t)(2 * NUM_DOCS); )
{
docid_t docId = pDocFreqs->skipTo((docid_t)i);
i = docId + 1;
if (docId == INVALID_DOCID)
{
break;
}
CPPUNIT_ASSERT_EQUAL(*it, docId);
++it;
}
CPPUNIT_ASSERT(it == answer.end());
}
{
Index index;
index.open(str, Index::READ, NULL);
IndexReaderPtr pIndexReader = index.acquireReader();
CPPUNIT_ASSERT(pIndexReader != NULL);
TermReaderPtr pTermReader = pIndexReader->termReader();
TermPostingIteratorPtr pDocFreqs = pTermReader->seek(pTerm.get());
CPPUNIT_ASSERT(pDocFreqs);
CPPUNIT_ASSERT_EQUAL((df_t)(2 * NUM_DOCS), pDocFreqs->getTermMeta().getDocFreq());
std::set<docid_t>::const_iterator it = answer.begin();
for (docid_t i = 0; i < (docid_t)(2 * NUM_DOCS); )
{
docid_t docId = pDocFreqs->skipTo((docid_t)i);
i = docId + 1;
if (docId == INVALID_DOCID)
{
break;
}
CPPUNIT_ASSERT_EQUAL(*it, docId);
++it;
}
CPPUNIT_ASSERT(it == answer.end());
// for (std::set<docid_t>::const_iterator it = answer.begin();
// it != answer.end(); ++it)
// {
// docid_t docId = pDocFreqs->skipTo(*it);
// CPPUNIT_ASSERT_EQUAL(*it, docId);
// }
// docid_t docId = pDocFreqs->skipTo(NUM_DOCS + 0);
// CPPUNIT_ASSERT_EQUAL((docid_t)NUM_DOCS + 1, docId);
// docId = pDocFreqs->skipTo(NUM_DOCS + 500);
// CPPUNIT_ASSERT_EQUAL((docid_t)NUM_DOCS + 501, docId);
// docId = pDocFreqs->skipTo(NUM_DOCS + 505);
// CPPUNIT_ASSERT_EQUAL((docid_t)NUM_DOCS + 506, docId);
// docId = pDocFreqs->skipTo(2 * NUM_DOCS - 1);
// CPPUNIT_ASSERT_EQUAL((docid_t)INVALID_DOCID, docId);
}
}
