BRSnowballAnalyzer::~BRSnowballAnalyzer(){
SavedStreams* t = reinterpret_cast<SavedStreams*>(this->getPreviousTokenStream());
if (t) _CLDELETE(t->filteredTokenStream);
_CLDELETE_CARRAY(language);
if ( stopSet != NULL )
_CLDELETE(stopSet);
}
SegmentTermEnum::~SegmentTermEnum(){
//Func - Destructor
//Pre - true
//Post - The instance has been destroyed. If this instance was a clone
// then the inputstream is closed and deleted too.
//todo: revisit this... close() should clean up most of everything.
//Finalize prev
_CLDECDELETE(prev );
//Finalize term
_CLDECDELETE( _term );
//Delete the buffer if necessary
if ( buffer != NULL ) free(buffer);
//Delete termInfo if necessary
_CLDELETE(termInfo);
//Check if this instance is a clone
if ( isClone ){
//Close the inputstream
input->close();
//delete the inputstream
_CLDELETE(input);
}
}
FieldCacheAuto::~FieldCacheAuto(){
if ( contentType == FieldCacheAuto::INT_ARRAY ){
_CLDELETE_ARRAY(intArray);
}else if ( contentType == FieldCacheAuto::FLOAT_ARRAY ){
_CLDELETE_ARRAY(floatArray);
}else if ( contentType == FieldCacheAuto::STRING_INDEX ){
_CLDELETE(stringIndex);
}else if ( contentType == FieldCacheAuto::STRING_ARRAY ){
if ( ownContents ){
for ( int32_t i=0;i<contentLen;i++ )
_CLDELETE_CARRAY(stringArray[i]);
}
_CLDELETE_ARRAY(stringArray);
}else if ( contentType == FieldCacheAuto::COMPARABLE_ARRAY ){
if ( ownContents ){
for ( int32_t i=0;i<contentLen;i++ )
_CLDELETE(comparableArray[i]);
}
_CLDELETE_ARRAY(comparableArray);
}else if ( contentType == FieldCacheAuto::SORT_COMPARATOR ){
_CLDELETE(sortComparator);
}else if ( contentType == FieldCacheAuto::SCOREDOC_COMPARATOR ){
_CLDELETE(scoreDocComparator);
}
}
TopDocs* MultiSearcher::_search(Query* query, Filter* filter, const int32_t nDocs) {
HitQueue* hq = _CLNEW HitQueue(nDocs);
int32_t totalHits = 0;
TopDocs* docs;
int32_t j;
ScoreDoc* scoreDocs;
for (int32_t i = 0; i < searchablesLen; i++) { // search each searcher
docs = searchables[i]->_search(query, filter, nDocs);
totalHits += docs->totalHits; // update totalHits
scoreDocs = docs->scoreDocs;
for ( j = 0; j <docs->scoreDocsLength; ++j) { // merge scoreDocs int_to hq
scoreDocs[j].doc += starts[i]; // convert doc
if ( !hq->insert(scoreDocs[j]))
break; // no more scores > minScore
}
_CLDELETE(docs);
}
int32_t scoreDocsLen = hq->size();
scoreDocs = _CL_NEWARRAY(ScoreDoc, scoreDocsLen);
{//MSVC 6 scope fix
for (int32_t i = scoreDocsLen-1; i >= 0; --i) // put docs in array
scoreDocs[i] = hq->pop();
}
//cleanup
_CLDELETE(hq);
return _CLNEW TopDocs(totalHits, scoreDocs, scoreDocsLen);
}
void RAMDirectory::_copyFromDir(Directory* dir, bool closeDir)
{
vector<string> names;
dir->list(&names);
uint8_t buf[CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE];
for (size_t i=0;i<names.size();++i ){
if ( !CL_NS(index)::IndexReader::isLuceneFile(names[i].c_str()))
continue;
// make place on ram disk
IndexOutput* os = createOutput(names[i].c_str());
// read current file
IndexInput* is = dir->openInput(names[i].c_str());
// and copy to ram disk
//todo: this could be a problem when copying from big indexes...
int64_t len = is->length();
int64_t readCount = 0;
while (readCount < len) {
int32_t toRead = (int32_t)(readCount + CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE > len ? len - readCount : CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE);
is->readBytes(buf, toRead);
os->writeBytes(buf, toRead);
readCount += toRead;
}
// graceful cleanup
is->close();
_CLDELETE(is);
os->close();
_CLDELETE(os);
}
if (closeDir)
dir->close();
}
void testEqualScores()
{
// NOTE: uses index build in *this* setUp
IndexReader * pReader = IndexReader::open( m_pSmall );
IndexSearcher * pSearch = _CLNEW IndexSearcher( pReader );
Hits * pResult;
// some hits match more terms then others, score should be the same
Query * q = csrq( _T( "data" ), _T( "1" ), _T( "6" ), true, true );
pResult = pSearch->search( q );
size_t numHits = pResult->length();
assertEqualsMsg( _T( "wrong number of results" ), 6, numHits );
float_t score = pResult->score( 0 );
for( size_t i = 1; i < numHits; i++ )
{
assertTrueMsg( _T( "score was not the same" ), score == pResult->score( i ));
}
_CLDELETE( pResult );
_CLDELETE( q );
pSearch->close();
_CLDELETE( pSearch );
pReader->close();
_CLDELETE( pReader );
}
void _TestSearchesRun(CuTest *tc, Analyzer* analyzer, Searcher* search, const TCHAR* qry){
Query* q = NULL;
Hits* h = NULL;
try{
q = QueryParser::parse(qry , _T("contents"), analyzer);
if ( q != NULL ){
h = search->search( q );
if ( h->length() > 0 ){
//check for explanation memory leaks...
CL_NS(search)::Explanation expl1;
search->explain(q, h->id(0), &expl1);
TCHAR* tmp = expl1.toString();
_CLDELETE_CARRAY(tmp);
if ( h->length() > 1 ){ //do a second one just in case
CL_NS(search)::Explanation expl2;
search->explain(q, h->id(1), &expl2);
tmp = expl2.toString();
_CLDELETE_CARRAY(tmp);
}
}
}
}catch(CLuceneError& err){
CuFail(tc,_T("Error: %s\n"), err.twhat());
}catch(...){
CuFail(tc,_T("Error: unknown\n"));
}
_CLDELETE(h);
_CLDELETE(q);
}
void Soprano::Index::CLuceneIndex::dump( QTextStream& s ) const
{
QMutexLocker lock( &d->mutex );
clearError();
try {
lucene::index::IndexReader* reader = d->getIndexReader();
for ( int i = 0; i < reader->numDocs(); ++i ) {
lucene::document::Document* doc = reader->document( i );
s << "Document " << i << endl
<< "====================" << endl;
lucene::document::DocumentFieldEnumeration* e = doc->fields();
while ( e->hasMoreElements() ) {
lucene::document::Field* field = e->nextElement();
s << TString( field->name(), true ).toQString() << ": "
<< TString( field->stringValue(), true ).toQString()
<< endl;
}
s << endl;
_CLDELETE( e );
_CLDELETE( doc );
}
s << flush;
}
catch( CLuceneError& err ) {
qDebug() << "(Soprano::Index::CLuceneIndex) failed to dump index.";
setError( exceptionToError( err ) );
}
}
void testSrchWildcard(CuTest *tc ){
#ifdef NO_WILDCARD_QUERY
CuNotImpl(tc,_T("Wildcard"));
#else
CuAssert(tc,_T("Searcher was not open"),s!=NULL);
//testWildcard
_TestSearchesRun(tc, &a,s, _T("term*") );
_TestSearchesRun(tc, &a,s, _T("term*^2") );
_TestSearchesRun(tc, &a,s, _T("term~") );
_TestSearchesRun(tc, &a,s, _T("term^2~") );
_TestSearchesRun(tc, &a,s, _T("term~^2") );
_TestSearchesRun(tc, &a,s, _T("term*germ") );
_TestSearchesRun(tc, &a,s, _T("term*germ^3") );
//test problem reported by Gary Mangum
BooleanQuery* bq = _CLNEW BooleanQuery();
Term* upper = _CLNEW Term(_T("contents"),_T("0105"));
Term* lower = _CLNEW Term(_T("contents"),_T("0105"));
RangeQuery* rq=_CLNEW RangeQuery(lower,upper,true);
bq->add(rq,true,true,false);
_CLDECDELETE(upper);
_CLDECDELETE(lower);
Term* prefix = _CLNEW Term(_T("contents"),_T("reuters21578"));
PrefixQuery* pq = _CLNEW PrefixQuery(prefix);
_CLDECDELETE(prefix);
bq->add(pq,true,true,false);
Hits* h = NULL;
try{
h = s->search( bq );
}_CLFINALLY(
_CLDELETE(h);
_CLDELETE(bq);
);
void testExtractFromWildcardQuery( CuTest * tc )
{
Directory * pIndex = setUpIndex();
IndexReader * pReader = IndexReader::open( pIndex );
TermSet termSet;
WildcardQuery * wildcard;
Term * t1;
Query * rewrite;
t1 = _CLNEW Term( _T("data"), _T("aaaa?") );
wildcard = _CLNEW WildcardQuery( t1 );
rewrite = wildcard->rewrite( pReader );
rewrite->extractTerms( &termSet );
_CLLDECDELETE( t1 );
assertEqualsMsg( _T( "wrong number of terms" ), 3, termSet.size() );
for( TermSet::iterator itTerms = termSet.begin(); itTerms != termSet.end(); itTerms++ )
{
Term * pTerm = *itTerms;
if( 0 != _tcscmp( _T( "aaaaa" ), pTerm->text())
&& 0 != _tcscmp( _T( "aaaab" ), pTerm->text())
&& 0 != _tcscmp( _T( "aaaac" ), pTerm->text()))
{
assertTrueMsg( _T( "wrong term" ), false );
}
}
clearTermSet( termSet );
if( rewrite != wildcard )
_CLDELETE( rewrite );
_CLDELETE( wildcard );
t1 = _CLNEW Term( _T("data"), _T("aaa*") );
wildcard = _CLNEW WildcardQuery( t1 );
rewrite = wildcard->rewrite( pReader );
rewrite->extractTerms( &termSet );
_CLLDECDELETE( t1 );
assertEqualsMsg( _T( "wrong number of terms" ), 5, termSet.size() );
for( TermSet::iterator itTerms = termSet.begin(); itTerms != termSet.end(); itTerms++ )
{
Term * pTerm = *itTerms;
assertTrueMsg( _T( "wrong term" ), ( 0 == _tcsncmp( _T( "aaa" ), pTerm->text(), 3 )));
}
clearTermSet( termSet );
if( rewrite != wildcard )
_CLDELETE( rewrite );
_CLDELETE( wildcard );
pReader->close();
_CLDELETE( pReader );
closeIndex( pIndex );
pIndex = NULL;
}
IndexReader::~IndexReader()
{
if (writeLock != NULL) {
writeLock->release();
_CLDELETE(writeLock);
}
_CLDELETE(segmentInfos);
_CLDECDELETE(directory);
}
void Soprano::Index::QueryHitIteratorBackend::close()
{
clearError();
if ( m_hits ) {
_CLDELETE( m_hits );
m_hits = 0;
_CLDELETE( m_query );
m_query = 0;
}
}
PhraseScorer::~PhraseScorer() {
//Func - Destructor
//Pre - true
//Post - The instance has been destroyed
//The PhraseQueue pq (which is a PriorityQueue) pq is actually empty at present, the elements
//having been transferred by pqToList() to the linked list starting with
//first. The nodes of that linked list are deleted by the destructor of
//first, rather than the destructor of pq.
_CLDELETE(first);
_CLDELETE(pq);
}
Lexer::~Lexer() {
//Func - Destructor
//Pre - true
//Post - if delSR was true the InputStream input of reader has been deleted
// The instance of Lexer has been destroyed
if (delSR) {
_CLDELETE(reader->input);
}
_CLDELETE(reader);
}
void TestSpansAdvanced::setUp()
{
directory = _CLNEW RAMDirectory();
Analyzer * analyzer = _CLNEW StandardAnalyzer();
IndexWriter * writer = _CLNEW IndexWriter( directory, analyzer, true );
addDocuments( writer );
writer->close();
_CLDELETE( writer );
_CLDELETE( analyzer );
searcher = _CLNEW IndexSearcher( directory );
}
void FieldsReader::close() {
//Func - Closes the FieldsReader
//Pre - true
//Post - The FieldsReader has been closed
if (fieldsStream){
fieldsStream->close();
_CLDELETE(fieldsStream);
}
if(indexStream){
indexStream->close();
_CLDELETE(indexStream);
}
}
TestSpansAdvanced::~TestSpansAdvanced()
{
if( searcher )
{
searcher->close();
_CLDELETE( searcher );
}
if( directory )
{
directory->close();
_CLDELETE( directory );
}
}
TestBasics::~TestBasics()
{
if( searcher )
{
searcher->close();
_CLDELETE( searcher );
}
if( directory )
{
directory->close();
_CLDELETE( directory );
}
}
void testBoost()
{
// NOTE: uses index build in *this* setUp
IndexReader * pReader = IndexReader::open( m_pSmall );
IndexSearcher * pSearch = _CLNEW IndexSearcher( pReader );
Hits * pResult;
// test for correct application of query normalization
// must use a non score normalizing method for this.
Query * q = csrq( _T( "data" ), _T( "1" ), _T( "6" ), true, true );
q->setBoost( 100 );
pResult = pSearch->search( q );
for( size_t i = 1; i < pResult->length(); i++ )
{
assertTrueMsg( _T( "score was not was not correct" ), 1.0f == pResult->score( i ));
}
_CLDELETE( pResult );
_CLDELETE( q );
//
// Ensure that boosting works to score one clause of a query higher
// than another.
//
Query * q1 = csrq( _T( "data" ), _T( "A" ), _T( "A" ), true, true ); // matches document #0
q1->setBoost( .1f );
Query * q2 = csrq( _T( "data" ), _T( "Z" ), _T( "Z" ), true, true ); // matches document #1
BooleanQuery * bq = _CLNEW BooleanQuery( true );
bq->add( q1, true, BooleanClause::SHOULD );
bq->add( q2, true, BooleanClause::SHOULD );
pResult = pSearch->search( bq );
assertEquals( 1, pResult->id( 0 ));
assertEquals( 0, pResult->id( 1 ));
assertTrue( pResult->score( 0 ) > pResult->score( 1 ));
_CLDELETE( pResult );
_CLDELETE( bq );
q1 = csrq( _T( "data" ), _T( "A" ), _T( "A" ), true, true ); // matches document #0
q1->setBoost( 10.0f );
q2 = csrq( _T( "data" ), _T( "Z" ), _T( "Z" ), true, true ); // matches document #1
bq = _CLNEW BooleanQuery( true );
bq->add( q1, true, BooleanClause::SHOULD );
bq->add( q2, true, BooleanClause::SHOULD );
pResult = pSearch->search( bq );
assertEquals( 0, pResult->id( 0 ));
assertEquals( 1, pResult->id( 1 ));
assertTrue( pResult->score( 0 ) > pResult->score( 1 ));
_CLDELETE( pResult );
_CLDELETE( bq );
pSearch->close();
_CLDELETE( pSearch );
pReader->close();
_CLDELETE( pReader );
}
FieldsReader::FieldsStreamHolder::~FieldsStreamHolder(){
delete subStream;
delete indexInputStream;
indexInput->close();
_CLDELETE(indexInput);
}
PhrasePositions::~PhrasePositions(){
//Func - Destructor
//Pre - true
//Post - The instance has been deleted
//delete next Phrase position and by doing that
//all PhrasePositions in the list
_CLDELETE(_next);
//Check if tp is valid
if ( tp != NULL ){
//Close TermPositions tp
tp->close();
_CLDELETE(tp);
}
}
_LUCENE_THREAD_FUNC(atomicSearchTest, _directory){
Directory* directory = (Directory*)_directory;
uint64_t stopTime = Misc::currentTimeMillis() + 1000*ATOMIC_SEARCH_RUN_TIME_SEC;
int count = 0;
try {
while(Misc::currentTimeMillis() < stopTime && !atomicSearchFailed) {
IndexReader* r = IndexReader::open(directory);
try {
if ( 100 != r->numDocs() ){
fprintf(stderr, "err 2: 100 != %d \n", r->numDocs());
atomicSearchFailed = true;
}
} catch (CLuceneError& e) {
fprintf(stderr, "err 3: %d:%s\n", e.number(), e.what());
atomicSearchFailed = true;
break;
}
r->close();
_CLDELETE(r);
count++;
}
} catch (CLuceneError& e) {
fprintf(stderr, "err 4: #%d: %s\n", e.number(), e.what());
atomicSearchFailed = true;
}
_LUCENE_THREAD_FUNC_RETURN(0);
}
Document::~Document(){
//Func - Destructor
//Pre - true
//Post - Instance has been destroyed
boost = 1.0f;
_CLDELETE(fieldList);
}
void Document::removeFields(const TCHAR* name) {
CND_PRECONDITION(name != NULL, "name is NULL");
DocumentFieldEnumeration::DocumentFieldList* previous = NULL;
DocumentFieldEnumeration::DocumentFieldList* current = fieldList;
while (current != NULL) {
//cannot use interning here, because name is probably not interned
if ( _tcscmp(current->field->name(),name) == 0 ){
if (previous){
previous->next = current->next;
}else
fieldList = current->next;
current->next=NULL; //ensure fieldlist destructor doesnt delete it
_CLDELETE(current);
if ( previous )
current = previous->next;
else
current = fieldList;
}else{
previous = current;
current = current->next;
}
}
}
void createIndex(CuTest* tc, Directory* dir, bool multiSegment) {
WhitespaceAnalyzer whitespaceAnalyzer;
IndexWriter w(dir, &whitespaceAnalyzer, true);
w.setMergePolicy(_CLNEW LogDocMergePolicy());
Document doc;
for (int i = 0; i < 100; i++) {
createDocument(doc, i, 4);
w.addDocument(&doc);
if (multiSegment && (i % 10) == 0) {
w.flush();
}
}
if (!multiSegment) {
w.optimize();
}
w.close();
IndexReader* r = IndexReader::open(dir);
if (multiSegment) {
CuAssert(tc,_T("check is multi"), strcmp(r->getObjectName(),"MultiSegmentReader")==0);
} else {
CuAssert(tc,_T("check is segment"), strcmp(r->getObjectName(),"SegmentReader")==0);
}
r->close();
_CLDELETE(r);
}
void commit() {
// update all documents
// remove previous instances
if ( indexPresent() ) {
for ( QHash<Node, lucene::document::Document*>::iterator it = documentCache.begin();
it != documentCache.end(); ++it ) {
lucene::document::Document* doc = it.value();
if ( const TCHAR* id = doc->get( idFieldName().data() ) ) { // this check is only for testing, it should NEVER fail
lucene::index::Term* idTerm = _CLNEW lucene::index::Term( idFieldName().data(), id );
getIndexReader()->deleteDocuments( idTerm );
_CLDECDELETE( idTerm );
}
}
}
// add the updated ones
for ( QHash<Node, lucene::document::Document*>::iterator it = documentCache.begin();
it != documentCache.end(); ++it ) {
lucene::document::Document* doc = it.value();
// never add empty docs
if ( !docEmpty( doc ) ) {
getIndexWriter()->addDocument( doc );
}
_CLDELETE( doc );
}
documentCache.clear();
}
void Hits::getMoreDocs(const size_t m){
size_t _min = m;
{
size_t nHits = hitDocs.size();
if ( nHits > _min)
_min = nHits;
}
size_t n = _min * 2; // double # retrieved
TopDocs* topDocs = NULL;
if ( sort==NULL )
topDocs = (TopDocs*)((Searchable*)searcher)->_search(query, filter, n);
else
topDocs = (TopDocs*)((Searchable*)searcher)->_search(query, filter, n, sort);
_length = topDocs->totalHits;
ScoreDoc* scoreDocs = topDocs->scoreDocs;
int32_t scoreDocsLength = topDocs->scoreDocsLength;
float_t scoreNorm = 1.0f;
//Check that scoreDocs is a valid pointer before using it
if (scoreDocs != NULL){
if (_length > 0 && scoreDocs[0].score > 1.0f){
scoreNorm = 1.0f / scoreDocs[0].score;
}
int32_t end = scoreDocsLength < _length ? scoreDocsLength : _length;
for (int32_t i = hitDocs.size(); i < end; i++) {
hitDocs.push_back(_CLNEW HitDoc(scoreDocs[i].score*scoreNorm, scoreDocs[i].doc));
}
}
_CLDELETE(topDocs);
}
/**
* Verifies that the index has the correct number of documents.
*/
void TestSpansAdvanced2::testVerifyIndex()
{
IndexReader * reader = IndexReader::open( directory );
assertEquals( 8, reader->numDocs() );
reader->close();
_CLDELETE( reader );
}
HitDoc::~HitDoc(){
//Func - Destructor
//Pre - true
//Post - The instance has been destroyed
_CLDELETE(doc);
}
double Soprano::Index::CLuceneIndex::getScore( const Soprano::Node& resource, lucene::search::Query* query )
{
QMutexLocker lock( &d->mutex );
clearError();
try {
// rewrite the query
lucene::index::Term queryTerm( idFieldName().data(), TString( d->getId( resource ) ).data() );
lucene::search::TermQuery idQuery( &queryTerm );
lucene::search::BooleanQuery combinedQuery;
combinedQuery.add( &idQuery, true, false );
combinedQuery.add( query, true, false );
// fetch the score when the URI matches the original query
lucene::search::TopDocs* docs = static_cast<lucene::search::Searchable*>( d->getIndexSearcher() )->_search( &combinedQuery, 0, 1 );
double r = -1.0;
if ( docs->totalHits > 0 ) {
#ifdef CL_VERSION_19_OR_GREATER
r = docs->scoreDocs[0].score;
#else
r = docs->scoreDocs[0]->score;
#endif
}
_CLDELETE( docs );
return r;
}
catch( CLuceneError& err ) {
qDebug() << "search failed: " << err.what();
setError( exceptionToError( err ) );
return 0.0;
}
}
