void testEndThreadException(CuTest *tc) {
const int MAX_DOCS=1500;
RAMDirectory ram;
WhitespaceAnalyzer an;
IndexWriter* writer = _CLNEW IndexWriter(&ram, &an, true);
// add some documents
Document doc;
for (int i = 0; i < MAX_DOCS; i++) {
TCHAR * tmp = English::IntToEnglish(i);
doc.add(* new Field(_T("content"), tmp, Field::STORE_YES | Field::INDEX_UNTOKENIZED));
writer->addDocument(&doc);
doc.clear();
_CLDELETE_ARRAY( tmp );
}
CuAssertEquals(tc, MAX_DOCS, writer->docCount());
writer->close();
_CLLDELETE(writer);
// this sequence is OK: delete searcher after search thread finish
{
IndexSearcher * searcher = _CLNEW IndexSearcher(&ram);
_LUCENE_THREADID_TYPE thread = _LUCENE_THREAD_CREATE(&searchDocs, searcher);
SCOPED_LOCK_MUTEX(searchMutex);
CONDITION_WAIT(searchMutex, searchCondition);
// _LUCENE_SLEEP(9999); //make sure that deleteMutex is being waited on...
CONDITION_NOTIFYALL(deleteCondition);
_LUCENE_THREAD_JOIN(thread);
searcher->close();
_CLLDELETE(searcher);
}
// this produces memory exception: delete searcher after search finish but before thread finish
{
IndexSearcher * searcher = _CLNEW IndexSearcher(&ram);
_LUCENE_THREADID_TYPE thread = _LUCENE_THREAD_CREATE(&searchDocs, searcher);
SCOPED_LOCK_MUTEX(searchMutex);
CONDITION_WAIT(searchMutex, searchCondition);
searcher->close();
_CLLDELETE(searcher);
CONDITION_NOTIFYALL(deleteCondition);
_LUCENE_THREAD_JOIN(thread);
}
ram.close();
}
void testIncludeLowerTrue(CuTest* tc)
{
WhitespaceAnalyzer a;
RAMDirectory* index = _CLNEW RAMDirectory();
IndexWriter* writer = _CLNEW IndexWriter(index,
&a, true);
Document doc;
doc.add(*_CLNEW Field(_T("Category"), _T("a 1"), Field::STORE_YES | Field::INDEX_TOKENIZED));
writer->addDocument(&doc); doc.clear();
doc.add(*_CLNEW Field(_T("Category"), _T("a 2"), Field::STORE_YES | Field::INDEX_TOKENIZED));
writer->addDocument(&doc); doc.clear();
doc.add(*_CLNEW Field(_T("Category"), _T("a 3"), Field::STORE_YES | Field::INDEX_TOKENIZED));
writer->addDocument(&doc); doc.clear();
writer->close();
_CLLDELETE(writer);
IndexSearcher* s = _CLNEW IndexSearcher(index);
Filter* f = _CLNEW RangeFilter(_T("Category"), _T("3"), _T("3"), true, true);
Term* t = _CLNEW Term(_T("Category"), _T("a"));
Query* q1 = _CLNEW TermQuery(t);
_CLLDECDELETE(t);
t = _CLNEW Term(_T("Category"), _T("3"));
Query* q2 = _CLNEW TermQuery(t);
_CLLDECDELETE(t);
Hits* h = s->search(q1);
assertTrue(h->length() == 3);
_CLLDELETE(h);
h = s->search(q2);
assertTrue(h->length() == 1);
_CLLDELETE(h);
h = s->search(q1, f);
assertTrue(h->length() == 1);
_CLLDELETE(h);
s->close();
_CLLDELETE(s);
_CLLDELETE(q1);
_CLLDELETE(q2);
_CLLDELETE(f);
index->close();
_CLLDECDELETE(index);
}
void TestSpansAdvanced::setUp()
{
directory = _CLNEW RAMDirectory();
Analyzer * analyzer = _CLNEW StandardAnalyzer();
IndexWriter * writer = _CLNEW IndexWriter( directory, analyzer, true );
addDocuments( writer );
writer->close();
_CLDELETE( writer );
_CLDELETE( analyzer );
searcher = _CLNEW IndexSearcher( directory );
}
/// TestBooleanScorer.java, ported 5/9/2009
void testBooleanScorer(CuTest *tc) {
const TCHAR* FIELD = _T("category");
RAMDirectory directory;
TCHAR* values[] = { _T("1"), _T("2"), _T("3"), _T("4"), NULL};
try {
WhitespaceAnalyzer a;
IndexWriter* writer = _CLNEW IndexWriter(&directory, &a, true);
for (size_t i = 0; values[i]!=NULL; i++) {
Document* doc = _CLNEW Document();
doc->add(*_CLNEW Field(FIELD, values[i], Field::STORE_YES | Field::INDEX_TOKENIZED));
writer->addDocument(doc);
_CLLDELETE(doc);
}
writer->close();
_CLLDELETE(writer);
BooleanQuery* booleanQuery1 = _CLNEW BooleanQuery();
Term *t = _CLNEW Term(FIELD, _T("1"));
booleanQuery1->add(_CLNEW TermQuery(t), true, BooleanClause::SHOULD);
_CLDECDELETE(t);
t = _CLNEW Term(FIELD, _T("2"));
booleanQuery1->add(_CLNEW TermQuery(t), true, BooleanClause::SHOULD);
_CLDECDELETE(t);
BooleanQuery* query = _CLNEW BooleanQuery();
query->add(booleanQuery1, true, BooleanClause::MUST);
t = _CLNEW Term(FIELD, _T("9"));
query->add(_CLNEW TermQuery(t), true, BooleanClause::MUST_NOT);
_CLDECDELETE(t);
IndexSearcher *indexSearcher = _CLNEW IndexSearcher(&directory);
Hits *hits = indexSearcher->search(query);
CLUCENE_ASSERT(2 == hits->length()); // Number of matched documents
_CLLDELETE(hits);
_CLLDELETE(indexSearcher);
_CLLDELETE(query);
}
catch (CLuceneError& e) {
CuFail(tc, e.twhat());
}
}
void TestBasics::setUp()
{
directory = _CLNEW RAMDirectory();
Analyzer * analyzer = _CLNEW SimpleAnalyzer();
IndexWriter * writer = _CLNEW IndexWriter( directory, analyzer, true );
TCHAR buffer[ 200 ];
for( int32_t i = 0; i < 1000; i++ )
{
Document doc;
English::IntToEnglish( i, buffer, 200 );
doc.add( * _CLNEW Field( _T( "field" ), buffer, Field::STORE_YES | Field::INDEX_TOKENIZED ));
writer->addDocument( &doc );
}
writer->close();
_CLDELETE( writer );
_CLDELETE( analyzer );
searcher = _CLNEW IndexSearcher( directory );
}
void testRAMDirectorySize(CuTest * tc) {
MockRAMDirectory * ramDir = _CLNEW MockRAMDirectory(indexDir);
WhitespaceAnalyzer analyzer;
IndexWriter * writer = _CLNEW IndexWriter(ramDir, &analyzer, false);
writer->optimize();
CuAssertTrue(tc, ramDir->sizeInBytes == ramDir->getRecomputedSizeInBytes(), _T("RAMDir size"));
_LUCENE_THREADID_TYPE* threads = _CL_NEWARRAY(_LUCENE_THREADID_TYPE, numThreads);
ThreadData * tdata = _CL_NEWARRAY(ThreadData, numThreads);
for (int i=0; i<numThreads; i++) {
tdata[i].num = i;
tdata[i].dir = ramDir;
tdata[i].tc = tc;
tdata[i].writer = writer;
threads[i] = _LUCENE_THREAD_CREATE(&indexDocs, &tdata[i]);
}
for (int i=0; i<numThreads; i++) {
_LUCENE_THREAD_JOIN(threads[i]);
}
_CLDELETE_ARRAY(threads);
_CLDELETE_ARRAY(tdata);
writer->optimize();
CuAssertTrue(tc, ramDir->sizeInBytes == ramDir->getRecomputedSizeInBytes(), _T("RAMDir size"));
CuAssertEquals(tc, docsToAdd + (numThreads * (docsPerThread-1)), writer->docCount(), _T("document count"));
writer->close();
_CLLDELETE(writer);
ramDir->close();
_CLLDELETE(ramDir);
}
void setUp()
{
TCHAR tbuffer[16];
const TCHAR* data[] =
{
_T( "A 1 2 3 4 5 6" ),
_T( "Z 4 5 6" ),
NULL,
_T( "B 2 4 5 6" ),
_T( "Y 3 5 6" ),
NULL,
_T( "C 3 6" ),
_T( "X 4 5 6" )
};
m_pSmall = _CLNEW RAMDirectory();
Analyzer * pAnalyzer = _CLNEW WhitespaceAnalyzer();
IndexWriter * pWriter = _CLNEW IndexWriter( m_pSmall, pAnalyzer, true );
for( size_t i = 0; i < sizeof( data ) / sizeof( data[0] ); i++ )
{
_itot( i, tbuffer, 10 );
Document doc;
doc.add( * _CLNEW Field( _T( "id" ), tbuffer, Field::STORE_YES | Field::INDEX_UNTOKENIZED ));
doc.add( * _CLNEW Field( _T( "all" ), _T( "all" ), Field::STORE_YES | Field::INDEX_UNTOKENIZED ));
if( data[ i ] )
doc.add( * _CLNEW Field( _T( "data" ), data[ i ], Field::STORE_YES | Field::INDEX_TOKENIZED ));
pWriter->addDocument( &doc );
}
pWriter->optimize();
pWriter->close();
_CLDELETE( pWriter );
_CLDELETE( pAnalyzer );
}
/// TestBooleanPrefixQuery.java, ported 5/9/2009
void testBooleanPrefixQuery(CuTest* tc) {
RAMDirectory directory;
WhitespaceAnalyzer a;
TCHAR* categories[] = {_T("food"), _T("foodanddrink"),
_T("foodanddrinkandgoodtimes"), _T("food and drink"), NULL};
Query* rw1 = NULL;
Query* rw2 = NULL;
try {
IndexWriter* writer = _CLNEW IndexWriter(&directory, &a, true);
for (size_t i = 0; categories[i]!=NULL; i++) {
Document* doc = new Document();
doc->add(*_CLNEW Field(_T("category"), categories[i], Field::STORE_YES | Field::INDEX_UNTOKENIZED));
writer->addDocument(doc);
_CLLDELETE(doc);
}
writer->close();
_CLLDELETE(writer);
IndexReader* reader = IndexReader::open(&directory);
Term* t = _CLNEW Term(_T("category"), _T("foo"));
PrefixQuery* query = _CLNEW PrefixQuery(t);
_CLDECDELETE(t);
rw1 = query->rewrite(reader);
BooleanQuery* bq = _CLNEW BooleanQuery();
bq->add(query, true, BooleanClause::MUST);
rw2 = bq->rewrite(reader);
reader->close(); // TODO: check necessity (_CLLDELETE(reader) alone will not do the same cleanup)
_CLLDELETE(reader);
_CLLDELETE(bq);
} catch (CLuceneError& e) {
CuFail(tc, e.twhat());
}
BooleanQuery* bq1 = NULL;
if (rw1->instanceOf(BooleanQuery::getClassName())) {
bq1 = (BooleanQuery*) rw1;
}
BooleanQuery* bq2 = NULL;
if (rw2->instanceOf(BooleanQuery::getClassName())) {
bq2 = (BooleanQuery*) rw2;
} else {
CuFail(tc, _T("Rewrite"));
}
bool bClausesMatch = bq1->getClauseCount() == bq2->getClauseCount();
_CLLDELETE(rw1);
_CLLDELETE(rw2);
if (!bClausesMatch) {
CuFail(tc, _T("Number of Clauses Mismatch"));
}
}
bool CreatePakFile(const TCHAR* Filename, TArray<FPakInputPair>& FilesToAdd, const FPakCommandLineParameters& CmdLineParameters)
{
const double StartTime = FPlatformTime::Seconds();
// Create Pak
TAutoPtr<FArchive> PakFileHandle(CreatePakWriter(Filename));
if (!PakFileHandle.IsValid())
{
UE_LOG(LogPakFile, Error, TEXT("Unable to create pak file \"%s\"."), Filename);
return false;
}
FPakInfo Info;
TArray<FPakEntryPair> Index;
FString MountPoint = GetCommonRootPath(FilesToAdd);
uint8* ReadBuffer = NULL;
int64 BufferSize = 0;
ECompressionFlags CompressionMethod = COMPRESS_None;
FCompressedFileBuffer CompressedFileBuffer;
for (int32 FileIndex = 0; FileIndex < FilesToAdd.Num(); FileIndex++)
{
// Remember the offset but don't serialize it with the entry header.
int64 NewEntryOffset = PakFileHandle->Tell();
FPakEntryPair NewEntry;
//check if this file requested to be compression
int64 OriginalFileSize = IFileManager::Get().FileSize(*FilesToAdd[FileIndex].Source);
int64 RealFileSize = OriginalFileSize + NewEntry.Info.GetSerializedSize(FPakInfo::PakFile_Version_Latest);
CompressionMethod = (FilesToAdd[FileIndex].bNeedsCompression && OriginalFileSize > 0) ? COMPRESS_Default : COMPRESS_None;
if (CompressionMethod != COMPRESS_None)
{
if (CompressedFileBuffer.CompressFileToWorkingBuffer(FilesToAdd[FileIndex], ReadBuffer, BufferSize, CompressionMethod, CmdLineParameters.CompressionBlockSize))
{
// Check the compression ratio, if it's too low just store uncompressed. Also take into account read size
// if we still save 64KB it's probably worthwhile compressing, as that saves a file read operation in the runtime.
// TODO: drive this threashold from the command line
float PercentLess = ((float)CompressedFileBuffer.TotalCompressedSize/(OriginalFileSize/100.f));
if (PercentLess > 90.f && (OriginalFileSize-CompressedFileBuffer.TotalCompressedSize) < 65536)
{
CompressionMethod = COMPRESS_None;
}
else
{
NewEntry.Info.CompressionMethod = CompressionMethod;
NewEntry.Info.CompressionBlocks.AddUninitialized(CompressedFileBuffer.CompressedBlocks.Num());
RealFileSize = CompressedFileBuffer.TotalCompressedSize + NewEntry.Info.GetSerializedSize(FPakInfo::PakFile_Version_Latest);
NewEntry.Info.CompressionBlocks.Reset();
}
}
}
// Account for file system block size, which is a boundary we want to avoid crossing.
if (CmdLineParameters.FileSystemBlockSize > 0 && OriginalFileSize != INDEX_NONE && RealFileSize <= CmdLineParameters.FileSystemBlockSize)
{
if ((NewEntryOffset / CmdLineParameters.FileSystemBlockSize) != ((NewEntryOffset+RealFileSize) / CmdLineParameters.FileSystemBlockSize))
{
//File crosses a block boundary, so align it to the beginning of the next boundary
NewEntryOffset = AlignArbitrary(NewEntryOffset, CmdLineParameters.FileSystemBlockSize);
PakFileHandle->Seek(NewEntryOffset);
}
}
bool bCopiedToPak;
if (FilesToAdd[FileIndex].bNeedsCompression && CompressionMethod != COMPRESS_None)
{
bCopiedToPak = CopyCompressedFileToPak(*PakFileHandle, MountPoint, FilesToAdd[FileIndex], CompressedFileBuffer, NewEntry);
}
else
{
bCopiedToPak = CopyFileToPak(*PakFileHandle, MountPoint, FilesToAdd[FileIndex], ReadBuffer, BufferSize, NewEntry);
}
if (bCopiedToPak)
{
// Update offset now and store it in the index (and only in index)
NewEntry.Info.Offset = NewEntryOffset;
Index.Add(NewEntry);
if (FilesToAdd[FileIndex].bNeedsCompression && CompressionMethod != COMPRESS_None)
{
float PercentLess = ((float)NewEntry.Info.Size/(NewEntry.Info.UncompressedSize/100.f));
UE_LOG(LogPakFile, Display, TEXT("Added compressed file \"%s\", %.2f%% of original size. Compressed Size %lld bytes, Original Size %lld bytes. "), *NewEntry.Filename, PercentLess, NewEntry.Info.Size, NewEntry.Info.UncompressedSize);
}
else
{
UE_LOG(LogPakFile, Display, TEXT("Added file \"%s\", %lld bytes."), *NewEntry.Filename, NewEntry.Info.Size);
}
}
else
{
UE_LOG(LogPakFile, Warning, TEXT("Missing file \"%s\" will not be added to PAK file."), *FilesToAdd[FileIndex].Source);
}
}
FMemory::Free(ReadBuffer);
ReadBuffer = NULL;
// Remember IndexOffset
Info.IndexOffset = PakFileHandle->Tell();
// Serialize Pak Index at the end of Pak File
TArray<uint8> IndexData;
FMemoryWriter IndexWriter(IndexData);
IndexWriter.SetByteSwapping(PakFileHandle->ForceByteSwapping());
int32 NumEntries = Index.Num();
IndexWriter << MountPoint;
IndexWriter << NumEntries;
for (int32 EntryIndex = 0; EntryIndex < Index.Num(); EntryIndex++)
{
FPakEntryPair& Entry = Index[EntryIndex];
IndexWriter << Entry.Filename;
Entry.Info.Serialize(IndexWriter, Info.Version);
}
PakFileHandle->Serialize(IndexData.GetData(), IndexData.Num());
FSHA1::HashBuffer(IndexData.GetData(), IndexData.Num(), Info.IndexHash);
Info.IndexSize = IndexData.Num();
// Save trailer (offset, size, hash value)
Info.Serialize(*PakFileHandle);
UE_LOG(LogPakFile, Display, TEXT("Added %d files, %lld bytes total, time %.2lfs."), Index.Num(), PakFileHandle->TotalSize(), FPlatformTime::Seconds() - StartTime);
PakFileHandle->Close();
PakFileHandle.Reset();
return true;
}
