SegmentTermEnum::~SegmentTermEnum(){
//Func - Destructor
//Pre - true
//Post - The instance has been destroyed. If this instance was a clone
// then the inputstream is closed and deleted too.
//todo: revisit this... close() should clean up most of everything.
//Finalize prev
_CLDECDELETE(prev );
//Finalize term
_CLDECDELETE( _term );
//Delete the buffer if necessary
if ( buffer != NULL ) free(buffer);
//Delete termInfo if necessary
_CLDELETE(termInfo);
//Check if this instance is a clone
if ( isClone ){
//Close the inputstream
input->close();
//delete the inputstream
_CLDELETE(input);
}
}
void testSrchWildcard(CuTest *tc ){
#ifdef NO_WILDCARD_QUERY
CuNotImpl(tc,_T("Wildcard"));
#else
CuAssert(tc,_T("Searcher was not open"),s!=NULL);
//testWildcard
_TestSearchesRun(tc, &a,s, _T("term*") );
_TestSearchesRun(tc, &a,s, _T("term*^2") );
_TestSearchesRun(tc, &a,s, _T("term~") );
_TestSearchesRun(tc, &a,s, _T("term^2~") );
_TestSearchesRun(tc, &a,s, _T("term~^2") );
_TestSearchesRun(tc, &a,s, _T("term*germ") );
_TestSearchesRun(tc, &a,s, _T("term*germ^3") );
//test problem reported by Gary Mangum
BooleanQuery* bq = _CLNEW BooleanQuery();
Term* upper = _CLNEW Term(_T("contents"),_T("0105"));
Term* lower = _CLNEW Term(_T("contents"),_T("0105"));
RangeQuery* rq=_CLNEW RangeQuery(lower,upper,true);
bq->add(rq,true,true,false);
_CLDECDELETE(upper);
_CLDECDELETE(lower);
Term* prefix = _CLNEW Term(_T("contents"),_T("reuters21578"));
PrefixQuery* pq = _CLNEW PrefixQuery(prefix);
_CLDECDELETE(prefix);
bq->add(pq,true,true,false);
Hits* h = NULL;
try{
h = s->search( bq );
}_CLFINALLY(
_CLDELETE(h);
_CLDELETE(bq);
);
void SegmentTermEnum::seek(const int64_t pointer, const int32_t p, Term* t, TermInfo* ti) {
//Func - Repositions term and termInfo within the enumeration
//Pre - pointer >= 0
// p >= 0 and contains the new position within the enumeration
// t is a valid reference to a Term and is the new current term in the enumeration
// ti is a valid reference to a TermInfo and is corresponding TermInfo form the new
// current Term
//Post - term and terminfo have been repositioned within the enumeration
//Reset the IndexInput input to pointer
input->seek(pointer);
//Assign the new position
position = p;
//finalize the current term
if ( _term == NULL || _LUCENE_ATOMIC_INT_GET(_term->__cl_refcount) > 1 ){
_CLDECDELETE(_term);
//Get a pointer from t and increase the reference counter of t
_term = _CLNEW Term; //cannot use reference, because TermInfosReader uses non ref-counted array
}
_term->set(t,t->text());
//finalize prev
_CLDECDELETE(prev);
//Change the current termInfo so it matches the new current term
termInfo->set(ti);
//Have the buffer grown if needed
if ( bufferLength <= _term->textLength() )
growBuffer(_term->textLength(), true ); // copy term text into buffer
else
_tcsncpy(buffer,_term->text(),bufferLength); //just copy the buffer
}
RangeQuery::~RangeQuery() {
//Func - Destructor
//Pre - true
//Post - The instance has been destroyed
_CLDECDELETE(lowerTerm);
_CLDECDELETE(upperTerm);
}
bool SegmentTermEnum::next(){
//Func - Moves the current of the set to the next in the set
//Pre - true
//Post - If the end has been reached NULL is returned otherwise the term has
// become the next Term in the enumeration
//Increase position by and and check if the end has been reached
if (position++ >= size-1) {
//delete term
_CLDECDELETE(_term);
return false;
}
//delete the previous enumerated term
Term* tmp=NULL;
if ( prev != NULL ){
int32_t usage = prev->__cl_refcount;
if ( usage > 1 ){
_CLDECDELETE(prev); //todo: tune other places try and delete its term
}else
tmp = prev; //we are going to re-use this term
}
//prev becomes the current enumerated term
prev = _term;
//term becomes the next term read from inputStream input
_term = readTerm(tmp);
//Read docFreq, the number of documents which contain the term.
termInfo->docFreq = input->readVInt();
//Read freqPointer, a pointer into the TermFreqs file (.frq)
termInfo->freqPointer += input->readVLong();
//Read proxPointer, a pointer into the TermPosition file (.prx).
termInfo->proxPointer += input->readVLong();
if(format == -1){
// just read skipOffset in order to increment file pointer;
// value is never used since skipTo is switched off
if (!isIndex) {
if (termInfo->docFreq > formatM1SkipInterval) {
termInfo->skipOffset = input->readVInt();
}
}
}else{
if (termInfo->docFreq >= skipInterval)
termInfo->skipOffset = input->readVInt();
}
//Check if the enumeration is an index
if (isIndex)
//read index pointer
indexPointer += input->readVLong();
return true;
}
bool SegmentMergeInfo::next() {
//Func - Moves the current term of the enumeration termEnum to the next and term
// points to this new current term
//Pre - true
//Post - Returns true if the term has been moved to the next otherwise false
if (termEnum->next()) {
_CLDECDELETE(term);
term = termEnum->term();
return true;
} else {
_CLDECDELETE(term); //TODO: test HighFreqTerms errors with this
term = NULL;
return false;
}
}
_LUCENE_THREAD_FUNC(atomicIndexTest, _writer){
IndexWriter* writer= (IndexWriter*)_writer;
uint64_t stopTime = Misc::currentTimeMillis() + 1000*ATOMIC_SEARCH_RUN_TIME_SEC;
int count = 0;
try {
while(Misc::currentTimeMillis() < stopTime && !atomicSearchFailed) {
// Update all 100 docs...
TCHAR buf[30];
StringBuffer sb;
for(int i=0; i<100; i++) {
Document d;
_i64tot(rand(), buf, 10);
sb.clear();
English::IntToEnglish(i+10*count, &sb);
d.add(*_CLNEW Field(_T("contents"), sb.getBuffer() , Field::STORE_NO | Field::INDEX_TOKENIZED));
_i64tot(i,buf,10);
d.add(*_CLNEW Field(_T("id"), buf, Field::STORE_YES | Field::INDEX_UNTOKENIZED));
Term* t = _CLNEW Term(_T("id"), buf);
writer->updateDocument(t, &d);
_CLDECDELETE(t);
}
count++;
}
} catch (CLuceneError& e) {
fprintf(stderr, "err 1: #%d: %s\n", e.number(), e.what());
atomicSearchFailed = true;
}
_LUCENE_THREAD_FUNC_RETURN(0);
}
/** Creates new WildcardTermEnum */
WildcardTermEnum::WildcardTermEnum(IndexReader* reader, Term* term):
FilteredTermEnum(),
__term(_CL_POINTER(term)),
fieldMatch(false),
_endEnum(false)
{
pre = stringDuplicate(term->text());
const TCHAR* sidx = _tcschr( pre, LUCENE_WILDCARDTERMENUM_WILDCARD_STRING );
const TCHAR* cidx = _tcschr( pre, LUCENE_WILDCARDTERMENUM_WILDCARD_CHAR );
const TCHAR* tidx = sidx;
if (tidx == NULL)
tidx = cidx;
else if ( cidx && cidx > pre)
tidx = min(sidx, cidx);
CND_PRECONDITION(tidx != NULL, "tidx==NULL");
int32_t idx = (int32_t)(tidx - pre);
preLen = idx;
CND_PRECONDITION(preLen<term->textLength(), "preLen >= term->textLength()");
pre[preLen]=0; //trim end
Term* t = _CLNEW Term(__term, pre);
setEnum( reader->terms(t) );
_CLDECDELETE(t);
}
RAMDirectory::RAMDirectory(const char* dir):
Directory(),files(true,true)
{
Directory* fsdir = FSDirectory::getDirectory(dir,false);
try{
_copyFromDir(fsdir,false);
}_CLFINALLY(fsdir->close();_CLDECDELETE(fsdir););
int64_t IndexReader::getCurrentVersion(const char* directory){
Directory* dir = FSDirectory::getDirectory(directory);
int64_t version = getCurrentVersion(dir);
dir->close();
_CLDECDELETE(dir);
return version;
}
void commit() {
// update all documents
// remove previous instances
if ( indexPresent() ) {
for ( QHash<Node, lucene::document::Document*>::iterator it = documentCache.begin();
it != documentCache.end(); ++it ) {
lucene::document::Document* doc = it.value();
if ( const TCHAR* id = doc->get( idFieldName().data() ) ) { // this check is only for testing, it should NEVER fail
lucene::index::Term* idTerm = _CLNEW lucene::index::Term( idFieldName().data(), id );
getIndexReader()->deleteDocuments( idTerm );
_CLDECDELETE( idTerm );
}
}
}
// add the updated ones
for ( QHash<Node, lucene::document::Document*>::iterator it = documentCache.begin();
it != documentCache.end(); ++it ) {
lucene::document::Document* doc = it.value();
// never add empty docs
if ( !docEmpty( doc ) ) {
getIndexWriter()->addDocument( doc );
}
_CLDELETE( doc );
}
documentCache.clear();
}
/**
* FIXME: Describe <code>rewrite</code> method here.
*
* @param reader an <code>IndexReader</code> value
* @return a <code>Query</code> value
* @exception IOException if an error occurs
*/
Query* RangeQuery::rewrite(IndexReader* reader){
BooleanQuery* query = _CLNEW BooleanQuery;
TermEnum* enumerator = reader->terms(lowerTerm);
Term* lastTerm = NULL;
try {
bool checkLower = false;
if (!inclusive) // make adjustments to set to exclusive
checkLower = true;
const TCHAR* testField = getField();
do {
lastTerm = enumerator->term();
if (lastTerm != NULL && lastTerm->field() == testField ) {
if (!checkLower || _tcscmp(lastTerm->text(),lowerTerm->text()) > 0) {
checkLower = false;
if (upperTerm != NULL) {
int compare = _tcscmp(upperTerm->text(),lastTerm->text());
/* if beyond the upper term, or is exclusive and
* this is equal to the upper term, break out */
if ((compare < 0) || (!inclusive && compare == 0))
break;
}
TermQuery* tq = _CLNEW TermQuery(lastTerm); // found a match
tq->setBoost(getBoost()); // set the boost
query->add(tq, true, false, false); // add to query
}
}else {
break;
}
_CLDECDELETE(lastTerm);
}
while (enumerator->next());
}catch(...){
_CLDECDELETE(lastTerm); //always need to delete this
_CLDELETE(query); //in case of error, delete the query
enumerator->close();
_CLDELETE(enumerator);
throw; //rethrow
}
_CLDECDELETE(lastTerm); //always need to delete this
enumerator->close();
_CLDELETE(enumerator);
return query;
}
PrefixQuery::~PrefixQuery(){
//Func - Destructor
//Pre - true
//Post - The instance has been destroyed.
//Delete prefix by finalizing it
_CLDECDELETE(prefix);
}
IndexReader::~IndexReader()
{
if (writeLock != NULL) {
writeLock->release();
_CLDELETE(writeLock);
}
_CLDELETE(segmentInfos);
_CLDECDELETE(directory);
}
/// TestBooleanScorer.java, ported 5/9/2009
void testBooleanScorer(CuTest *tc) {
const TCHAR* FIELD = _T("category");
RAMDirectory directory;
TCHAR* values[] = { _T("1"), _T("2"), _T("3"), _T("4"), NULL};
try {
WhitespaceAnalyzer a;
IndexWriter* writer = _CLNEW IndexWriter(&directory, &a, true);
for (size_t i = 0; values[i]!=NULL; i++) {
Document* doc = _CLNEW Document();
doc->add(*_CLNEW Field(FIELD, values[i], Field::STORE_YES | Field::INDEX_TOKENIZED));
writer->addDocument(doc);
_CLLDELETE(doc);
}
writer->close();
_CLLDELETE(writer);
BooleanQuery* booleanQuery1 = _CLNEW BooleanQuery();
Term *t = _CLNEW Term(FIELD, _T("1"));
booleanQuery1->add(_CLNEW TermQuery(t), true, BooleanClause::SHOULD);
_CLDECDELETE(t);
t = _CLNEW Term(FIELD, _T("2"));
booleanQuery1->add(_CLNEW TermQuery(t), true, BooleanClause::SHOULD);
_CLDECDELETE(t);
BooleanQuery* query = _CLNEW BooleanQuery();
query->add(booleanQuery1, true, BooleanClause::MUST);
t = _CLNEW Term(FIELD, _T("9"));
query->add(_CLNEW TermQuery(t), true, BooleanClause::MUST_NOT);
_CLDECDELETE(t);
IndexSearcher *indexSearcher = _CLNEW IndexSearcher(&directory);
Hits *hits = indexSearcher->search(query);
CLUCENE_ASSERT(2 == hits->length()); // Number of matched documents
_CLLDELETE(hits);
_CLLDELETE(indexSearcher);
_CLLDELETE(query);
}
catch (CLuceneError& e) {
CuFail(tc, e.twhat());
}
}
Query* PrefixQuery::rewrite(IndexReader* reader){
BooleanQuery* query = _CLNEW BooleanQuery();
TermEnum* enumerator = reader->terms(prefix);
Term* lastTerm = NULL;
try {
const TCHAR* prefixText = prefix->text();
const TCHAR* prefixField = prefix->field();
const TCHAR* tmp;
size_t i;
int32_t prefixLen = prefix->textLength();
do {
lastTerm = enumerator->term();
if (lastTerm != NULL && lastTerm->field() == prefixField ){
//now see if term->text() starts with prefixText
int32_t termLen = lastTerm->textLength();
if ( prefixLen>termLen )
break; //the prefix is longer than the term, can't be matched
tmp = lastTerm->text();
//check for prefix match in reverse, since most change will be at the end
for ( i=prefixLen-1;i!=-1;--i ){
if ( tmp[i] != prefixText[i] ){
tmp=NULL;//signals inequality
break;
}
}
if ( tmp == NULL )
break;
TermQuery* tq = _CLNEW TermQuery(lastTerm); // found a match
tq->setBoost(getBoost()); // set the boost
query->add(tq,true,false, false); // add to query
} else
break;
_CLDECDELETE(lastTerm);
} while (enumerator->next());
}_CLFINALLY(
enumerator->close();
_CLDELETE(enumerator);
_CLDECDELETE(lastTerm);
);
void WildcardTermEnum::close()
{
if ( __term != NULL ){
FilteredTermEnum::close();
_CLDECDELETE(__term);
__term = NULL;
_CLDELETE_CARRAY( pre );
}
}
/*
Run above stress test against FSDirectory.
*/
void testFSThreading(CuTest *tc){
//setup some variables
char tmpfsdirectory[1024];
strcpy(tmpfsdirectory,cl_tempDir);
strcat(tmpfsdirectory,"/threading-index");
// Second in an FSDirectory:
Directory* directory = FSDirectory::getDirectory(tmpfsdirectory);
runThreadingTests(tc,*directory);
directory->close();
_CLDECDELETE(directory);
}
void FuzzyTermEnum::close(){
FilteredTermEnum::close();
//Finalize the searchTerm
_CLDECDELETE(searchTerm);
free(d);
d=NULL;
_CLDELETE_CARRAY(text);
_CLDELETE_CARRAY(prefix);
}
bool FilteredTermEnum::next() {
//Func - Increments the enumeration to the next element.
//Pre - true
//Post - Returns True if the enumeration has been moved to the next element otherwise false
//The actual enumerator is not initialized!
if (actualEnum == NULL){
return false;
}
//Finalize the currentTerm and reset it to NULL
_CLDECDELETE( currentTerm );
//Iterate through the enumeration
while (currentTerm == NULL) {
if (endEnum())
return false;
if (actualEnum->next()) {
//Order term not to return reference ownership here. */
Term* term = actualEnum->term(false);
//Compare the retrieved term
if (termCompare(term)){
//Matched so finalize the current
_CLDECDELETE(currentTerm);
//Get a reference to the matched term
currentTerm = _CL_POINTER(term);
return true;
}
}else
return false;
}
_CLDECDELETE(currentTerm);
currentTerm = NULL;
return false;
}
void FilteredTermEnum::close(){
//Func - Closes the enumeration to further activity, freeing resources.
//Pre - true
//Post - The Enumeration has been closed
//Check if actualEnum is valid
if (actualEnum){
//Close the enumeration
actualEnum->close();
//Destroy the enumeration
_CLDELETE(actualEnum);
}
//Destroy currentTerm
_CLDECDELETE(currentTerm);
}
IndexReader* IndexReader::open(const char* path, bool closeDirectoryOnCleanup, IndexDeletionPolicy* deletionPolicy){
//Func - Static method.
// Returns an IndexReader reading the index in an FSDirectory in the named path.
//Pre - path != NULL and contains the path of the index for which an IndexReader must be
// instantiated
// closeDir indicates if the directory needs to be closed
//Post - An IndexReader has been returned that reads tnhe index located at path
CND_PRECONDITION(path != NULL, "path is NULL");
Directory* dir = FSDirectory::getDirectory(path);
IndexReader* reader = open(dir,closeDirectoryOnCleanup,deletionPolicy);
//because fsdirectory will now have a refcount of 1 more than
//if the reader had been opened with a directory object,
//we need to do a refdec
_CLDECDELETE(dir);
return reader;
}
void SegmentMergeInfo::close() {
//Func - Closes the the resources
//Pre - true
//Post - The resources have been closed
//First make sure posting has been closed
if ( postings != NULL ){
postings->close();
_CLVDELETE(postings); //todo: not a clucene object... should be
}
if ( termEnum != NULL ){
termEnum->close();
_CLDELETE(termEnum);
}
_CLDECDELETE(term);
_CLDELETE_ARRAY(docMap);
}
void FilteredTermEnum::setEnum(TermEnum* actualEnum) {
//Func - Sets the actual Enumeration
//Pre - actualEnum != NULL
//Post - The instance has been created
CND_PRECONDITION(actualEnum != NULL,"actualEnum is NULL");
_CLLDELETE(this->actualEnum);
this->actualEnum = actualEnum;
// Find the first term that matches
//Ordered term not to return reference ownership here.
Term* term = actualEnum->term(false);
if (term != NULL && termCompare(term)){
_CLDECDELETE(currentTerm);
currentTerm = _CL_POINTER(term);
}else{
next();
}
}
void IndexWriter::_finalize(){
//Func - Releases all the resources of the instance
//Pre - true
//Post - All the releases have been released
if(writeLock != NULL){
//release write lock
writeLock->release();
_CLDELETE( writeLock );
}
//Delete the ramDirectory
if ( ramDirectory != NULL ){
ramDirectory->close();
_CLDECDELETE(ramDirectory);
}
//clear segmentInfos and delete it
_CLDELETE(segmentInfos);
}
/// TestBooleanQuery.java, ported 5/9/2009
void testEquality(CuTest *tc) {
BooleanQuery* bq1 = _CLNEW BooleanQuery();
Term* t = _CLNEW Term(_T("field"), _T("value1"));
bq1->add(_CLNEW TermQuery(t), true, BooleanClause::SHOULD);
_CLDECDELETE(t);
t = _CLNEW Term(_T("field"), _T("value2"));
bq1->add(_CLNEW TermQuery(t), true, BooleanClause::SHOULD);
_CLDECDELETE(t);
BooleanQuery* nested1 = _CLNEW BooleanQuery();
t = _CLNEW Term(_T("field"), _T("nestedvalue1"));
nested1->add(_CLNEW TermQuery(t), true, BooleanClause::SHOULD);
_CLDECDELETE(t);
t = _CLNEW Term(_T("field"), _T("nestedvalue2"));
nested1->add(_CLNEW TermQuery(t), true, BooleanClause::SHOULD);
_CLDECDELETE(t);
bq1->add(nested1, true, BooleanClause::SHOULD);
BooleanQuery* bq2 = _CLNEW BooleanQuery();
t = _CLNEW Term(_T("field"), _T("value1"));
bq2->add(_CLNEW TermQuery(t), true, BooleanClause::SHOULD);
_CLDECDELETE(t);
t = _CLNEW Term(_T("field"), _T("value2"));
bq2->add(_CLNEW TermQuery(t), true, BooleanClause::SHOULD);
_CLDECDELETE(t);
BooleanQuery* nested2 = _CLNEW BooleanQuery();
t = _CLNEW Term(_T("field"), _T("nestedvalue1"));
nested2->add(_CLNEW TermQuery(t), true, BooleanClause::SHOULD);
_CLDECDELETE(t);
t = _CLNEW Term(_T("field"), _T("nestedvalue2"));
nested2->add(_CLNEW TermQuery(t), true, BooleanClause::SHOULD);
_CLDECDELETE(t);
bq2->add(nested2, true, BooleanClause::SHOULD);
CLUCENE_ASSERT(bq1->equals(bq2));
_CLLDELETE(bq1);
_CLLDELETE(bq2);
}
QCLuceneFieldPrivate::~QCLuceneFieldPrivate()
{
if (deleteCLuceneField)
_CLDECDELETE(field);
}
QCLuceneTokenPrivate::~QCLuceneTokenPrivate()
{
if (deleteCLuceneToken)
_CLDECDELETE(token);
}
QCLuceneTokenStreamPrivate::~QCLuceneTokenStreamPrivate()
{
if (deleteCLuceneTokenStream)
_CLDECDELETE(tokenStream);
}
QCLuceneHitsPrivate::~QCLuceneHitsPrivate()
{
if (deleteCLuceneHits)
_CLDECDELETE(hits);
}
