Lexer::Lexer(QueryParserBase* queryparser, const TCHAR* query) {
//Func - Constructor
//Pre - query != NULL and contains the query string
//Post - An instance of Lexer has been created
this->queryparser = queryparser;
CND_PRECONDITION(query != NULL, "query is NULL");
//The InputStream of Reader must be destroyed in the destructor
delSR = true;
StringReader *r = _CLNEW StringReader(query);
//Check to see if r has been created properly
CND_CONDITION(r != NULL, "Could not allocate memory for StringReader r");
//Instantie a FastCharStream instance using r and assign it to reader
reader = _CLNEW FastCharStream(r);
//Check to see if reader has been created properly
CND_CONDITION(reader != NULL, "Could not allocate memory for FastCharStream reader");
//The InputStream of Reader must be destroyed in the destructor
delSR = true;
}
int32_t SegmentMerger::mergeFields()
{
//Func - Merge the fields of all segments
//Pre - true
//Post - The field infos and field values of all segments have been merged.
//Create a new FieldInfos
fieldInfos = _CLNEW FieldInfos(); // merge field names
//Condition check to see if fieldInfos points to a valid instance
CND_CONDITION(fieldInfos != NULL, "Memory allocation for fieldInfos failed");
IndexReader* reader = NULL;
int32_t docCount = 0;
//Iterate through all readers
for (uint32_t i = 0; i < readers.size(); i++) {
//get the i-th reader
reader = readers[i];
//Condition check to see if reader points to a valid instance
CND_CONDITION(reader != NULL,"No IndexReader found");
StringArrayWithDeletor tmp;
tmp.clear();
reader->getFieldNames(IndexReader::TERMVECTOR_WITH_POSITION_OFFSET, tmp);
addIndexed(reader, fieldInfos, tmp, true, true, true);
tmp.clear();
reader->getFieldNames(IndexReader::TERMVECTOR_WITH_POSITION, tmp);
addIndexed(reader, fieldInfos, tmp, true, true, false);
tmp.clear();
reader->getFieldNames(IndexReader::TERMVECTOR_WITH_OFFSET, tmp);
addIndexed(reader, fieldInfos, tmp, true, false, true);
tmp.clear();
reader->getFieldNames(IndexReader::TERMVECTOR, tmp);
addIndexed(reader, fieldInfos, tmp, true, false, false);
tmp.clear();
reader->getFieldNames(IndexReader::INDEXED, tmp);
addIndexed(reader, fieldInfos, tmp, false, false, false);
tmp.clear();
reader->getFieldNames(IndexReader::UNINDEXED, tmp);
if (tmp.size() > 0) {
TCHAR** arr = _CL_NEWARRAY(TCHAR*,tmp.size()+1);
tmp.toArray(arr);
fieldInfos->add((const TCHAR**)arr, false);
_CLDELETE_ARRAY(arr);
//no need to delete the contents, since tmp is responsible for it
}
}
bool TransactionalRAMDirectory::archiveOrigFileIfNecessary(const QString& name)
{
// If a file named $name was present when the transaction started and the
// original RAMFile object has not been archived for restoration upon
// transaction abort, then do so, and return true.
// In any other case, return false.
if (fileExists(name) && filesToRemoveOnAbort.find(name) == filesToRemoveOnAbort.end()) {
// The file exists, but isn't recorded as having been created after the
// start of the transaction, so it must've been present at the start of
// the transaction.
// Transfer memory ownership of both the key and the value from files to
// filesToRestoreOnAbort.
QString origName = files.getKey(name);
RAMFile* origFile = files.get(name);
files.remove(name, true, true);
filesToRestoreOnAbort.put(origName, origFile);
CND_CONDITION(!fileExists(name),
"File should not exist immediately after archival.");
return true;
}
return false;
}
void TransactionalRAMDirectory::transAbort()
{
if (!transOpen)
_CLTHROWA(CL_ERR_RAMTransaction, "There is no open transaction.");
// Delete each file in filesToRemoveOnAbort.
FilenameSet::const_iterator itrDel = filesToRemoveOnAbort.begin();
for ( ; itrDel != filesToRemoveOnAbort.end(); ++itrDel) {
size_t nameLength = itrDel->first.length();
// Special exception: Refrain from deleting a lock's flag file, as that
// would interfere with the operation of the lock.
if (!(nameLength >= 5
&& itrDel->first.rightRef(5) == QLatin1String(".lock"))) {
RAMDirectory::deleteFile(itrDel->first);
}
}
// Ownership of the memory of both the key and the value never left files,
// so there's no need for a special directive to filesToRemoveOnAbort.
filesToRemoveOnAbort.clear();
// Now that any new-since-trans-start files with the same names as
// already-present-at-trans-start files are out of the way, restore each
// file in filesToRestoreOnAbort.
TransFileMap::const_iterator itr = filesToRestoreOnAbort.begin();
for ( ; itr != filesToRestoreOnAbort.end(); ++itr) {
files.put(itr->first, itr->second);
filesToRestoreOnAbort.remove(itr->first);
}
CND_CONDITION(filesToRestoreOnAbort.size() == 0,
"filesToRestoreOnAbort should be empty.");
transResolved();
}
void TransactionalRAMDirectory::transStart() {
if (transOpen) {
_CLTHROWA(CL_ERR_RAMTransaction,"Must resolve previous transaction before starting another.");
}
CND_CONDITION(filesToRemoveOnAbort.size() == 0,
"filesToRemoveOnAbort should have been cleared by either its"
" constructor or transResolved."
);
CND_CONDITION(filesToRestoreOnAbort.size() == 0,
"filesToRestoreOnAbort should have been cleared by either its"
" constructor or transResolved."
);
transOpen = true;
}
TermInfosReader::TermInfosReader(Directory* dir, const char* seg, FieldInfos* fis, const int32_t readBufferSize):
directory (dir),fieldInfos (fis), indexTerms(NULL), indexInfos(NULL), indexPointers(NULL), indexDivisor(1)
{
//Func - Constructor.
// Reads the TermInfos file (.tis) and eventually the Term Info Index file (.tii)
//Pre - dir is a reference to a valid Directory
// Fis contains a valid reference to an FieldInfos instance
// seg != NULL and contains the name of the segment
//Post - An instance has been created and the index named seg has been read. (Remember
// a segment is nothing more then an independently readable index)
CND_PRECONDITION(seg != NULL, "seg is NULL");
//Initialize the name of the segment
segment = seg;
//Create a filname fo a Term Info File
string tisFile = Misc::segmentname(segment,".tis");
string tiiFile = Misc::segmentname(segment,".tii");
bool success = false;
origEnum = indexEnum = NULL;
_size = indexTermsLength = totalIndexInterval = 0;
try {
//Create an SegmentTermEnum for storing all the terms read of the segment
origEnum = _CLNEW SegmentTermEnum( directory->openInput( tisFile.c_str(), readBufferSize ), fieldInfos, false);
_size = origEnum->size;
totalIndexInterval = origEnum->indexInterval;
indexEnum = _CLNEW SegmentTermEnum( directory->openInput( tiiFile.c_str(), readBufferSize ), fieldInfos, true);
//Check if enumerator points to a valid instance
CND_CONDITION(origEnum != NULL, "No memory could be allocated for orig enumerator");
CND_CONDITION(indexEnum != NULL, "No memory could be allocated for index enumerator");
success = true;
} _CLFINALLY({
// With lock-less commits, it's entirely possible (and
// fine) to hit a FileNotFound exception above. In
// this case, we want to explicitly close any subset
// of things that were opened so that we don't have to
// wait for a GC to do so.
if (!success) {
close();
}
});
PhraseScorer::PhraseScorer(Weight* weight, TermPositions** tps,
int32_t* positions, Similarity* similarity, uint8_t* norms):
Scorer(similarity)
{
//Func - Constructor
//Pre - tps != NULL and is an array of TermPositions
// tpsLength >= 0
// n != NULL
//Post - The instance has been created
CND_PRECONDITION(tps != NULL,"tps is NULL");
//norms are only used if phraseFreq returns more than 0.0
//phraseFreq should only return more than 0.0 if norms != NULL
//CND_PRECONDITION(n != NULL,"n is NULL");
firstTime = true;
more = true;
this->norms = norms;
this->weight = weight;
this->value = weight->getValue();
//reset internal pointers
first = NULL;
last = NULL;
//use pq to build a sorted list of PhrasePositions
int32_t i = 0;
while(tps[i] != NULL){
PhrasePositions *pp = _CLNEW PhrasePositions(tps[i], positions[i]);
CND_CONDITION(pp != NULL,"Could not allocate memory for pp");
//Store PhrasePos into the PhrasePos pq
if (last != NULL) { // add next to end of list
last->_next = pp;
} else
first = pp;
last = pp;
i++;
}
pq = _CLNEW PhraseQueue(i); //i==tps.length
CND_CONDITION(pq != NULL,"Could not allocate memory for pq");
}
@@ -249,7 +249,7 @@ int32_t SegmentMerger::mergeFields() {
CND_CONDITION(fieldsWriter != NULL,"Memory allocation for fieldsWriter failed");
try {
- IndexReader* reader = NULL;
+ reader = NULL;
int32_t maxDoc = 0;
//Iterate through all readers
for (uint32_t i = 0; i < readers.size(); i++) {
void TransactionalRAMDirectory::transAbort() {
if (!transOpen) {
_CLTHROWA(CL_ERR_RAMTransaction,"There is no open transaction.");
}
// Delete each file in filesToRemoveOnAbort.
for (FilenameSet::const_iterator itrDel = filesToRemoveOnAbort.begin();
itrDel != filesToRemoveOnAbort.end();
++itrDel
)
{
const char* name = itrDel->first;
size_t nameLength = strlen(name);
// Special exception:
// Refrain from deleting a lock's flag file, as that would interfere with
// the operation of the lock.
if (!(nameLength >= 5 && strcmp(name + nameLength - 5, ".lock"))) {
RAMDirectory::deleteFile(name);
}
}
// Ownership of the memory of both the key and the value never left files,
// so there's no need for a special directive to filesToRemoveOnAbort.
filesToRemoveOnAbort.clear();
// Now that any new-since-trans-start files with the same names as
// already-present-at-trans-start files are out of the way, restore each
// file in filesToRestoreOnAbort.
AStringArrayConst removeTheseWithoutDeletingMem;
for (TransFileMap::const_iterator itr = filesToRestoreOnAbort.begin();
itr != filesToRestoreOnAbort.end();
++itr
)
{
const char* name = itr->first;
files.put(name, itr->second);
// We've just transferred ownership of the memory of both the key and the
// value to files; we must now direct filesToRestoreOnAbort not to delete
// that memory as it removes the entry. This is performed in a separate
// loop to avoid modifying filesToRestoreOnAbort while iterating over it.
removeTheseWithoutDeletingMem.push_back(name);
}
for (AStringArrayConst::iterator itrRem = removeTheseWithoutDeletingMem.begin();
itrRem != removeTheseWithoutDeletingMem.end();
++itrRem
)
{
filesToRestoreOnAbort.remove(*itrRem); //remove iterator
}
CND_CONDITION(filesToRestoreOnAbort.size() == 0, "filesToRestoreOnAbort should be empty.");
transResolved();
}
IndexReader* SegmentMerger::segmentReader(const int32_t i) {
//Func - Returns a reference to the i-th IndexReader
//Pre - 0 <= i < readers.size()
//Post - A reference to the i-th IndexReader has been returned
CND_PRECONDITION(i >= 0, "i is a negative number");
CND_PRECONDITION((size_t)i < readers.size(), "i is bigger than the number of IndexReader instances");
//Retrieve the i-th IndexReader
IndexReader* ret = readers[i];
CND_CONDITION(ret != NULL,"No IndexReader found");
return ret;
}
SegmentTermPositions::SegmentTermPositions(const SegmentReader* _parent):
SegmentTermDocs(_parent){
//Func - Constructor
//Pre - Parent != NULL
//Post - The instance has been created
CND_PRECONDITION(_parent != NULL, "Parent is NULL");
proxStream = _parent->proxStream->clone();
CND_CONDITION(proxStream != NULL,"proxStream is NULL");
position = 0;
proxCount = 0;
}
SegmentInfo* SegmentInfos::info(int32_t i) const
{
//Func - Returns a reference to the i-th SegmentInfo in the list.
//Pre - i >= 0
//Post - A reference to the i-th SegmentInfo instance has been returned
CND_PRECONDITION(i >= 0, "i contains negative number");
//Get the i-th SegmentInfo instance
SegmentInfo *ret = infos.value(i, 0);
//Condition check to see if the i-th SegmentInfo has been retrieved
CND_CONDITION(ret != NULL, "No SegmentInfo instance found");
return ret;
}
Lexer::Lexer(QueryParserBase* queryparser, Reader* source) {
//Func - Constructor
// Initializes a new instance of the Lexer class with the specified
// TextReader to lex.
//Pre - Source contains a valid reference to a Reader
//Post - An instance of Lexer has been created using source as the reader
this->queryparser = queryparser;
//Instantie a FastCharStream instance using r and assign it to reader
reader = _CLNEW FastCharStream(source);
//Check to see if reader has been created properly
CND_CONDITION(reader != NULL, "Could not allocate memory for FastCharStream reader");
//The InputStream of Reader must not be destroyed in the destructor
delSR = false;
}
Query* QueryParser::parse(const TCHAR* query){
//Func - Returns a parsed Query instance
//Pre - query != NULL and contains the query value to be parsed
//Post - Returns a parsed Query Instance
CND_PRECONDITION(query != NULL, "query is NULL");
//Instantie a Stringer that can read the query string
Reader* r = _CLNEW StringReader(query);
//Check to see if r has been created properly
CND_CONDITION(r != NULL, "Could not allocate memory for StringReader r");
//Pointer for the return value
Query* ret = NULL;
try{
//Parse the query managed by the StringReader R and return a parsed Query instance
//into ret
ret = parse(r);
}_CLFINALLY (
_CLDELETE(r);
);
void PhraseScorer::pqToList(){
//Func - Transfers the PhrasePositions from the PhraseQueue pq to
// the PhrasePositions list with first as its first element
//Pre - pq != NULL
// first = NULL
// last = NULL
//Post - All PhrasePositions have been transfered to the list
// of PhrasePositions of which the first element is pointed to by first
// and the last element is pointed to by last
CND_PRECONDITION(pq != NULL,"pq is NULL");
last = first = NULL;
PhrasePositions* PhrasePos = NULL;
//As long pq is not empty
while (pq->top() != NULL){
//Pop a PhrasePositions instance
PhrasePos = pq->pop();
// add next to end of list
if (last != NULL) {
last->_next = PhrasePos;
} else {
first = PhrasePos;
}
//Let last point to the new last PhrasePositions instance just added
last = PhrasePos;
//Reset the next of last to NULL
last->_next = NULL;
}
//Check to see that pq is empty now
CND_CONDITION(pq->size()==0, "pq is not empty while it should be");
}
void SegmentInfos::read(Directory* directory)
{
//Func - Reads segments file that resides in directory.
//Pre - directory contains a valid reference
//Post - The segments file has been read and for each segment found
// a SegmentsInfo intance has been created and stored.
//Open an IndexInput to the segments file and check if valid
IndexInput* input = directory->openInput(QLatin1String("segments"));
if (input) {
try {
int32_t format = input->readInt();
// file contains explicit format info
if (format < 0) {
// check that it is a format we can understand
if (format < FORMAT) {
TCHAR err[30];
_sntprintf(err, 30, _T("Unknown format version: %d"), format);
_CLTHROWT(CL_ERR_Runtime, err);
}
// read version
version = input->readLong();
// read counter
counter = input->readInt();
} else {
// file is in old format without explicit format info
counter = format;
}
//Temporary variable for storing the name of the segment
char aname[CL_MAX_PATH] = { 0 };
TCHAR tname[CL_MAX_PATH] = { 0 };
//read segmentInfos
for (int32_t i = input->readInt(); i > 0; --i) {
// read the name of the segment
input->readString(tname, CL_MAX_PATH);
STRCPY_TtoA(aname, tname, CL_MAX_PATH);
//Instantiate a new SegmentInfo Instance
SegmentInfo* si = _CLNEW SegmentInfo(QLatin1String(aname),
input->readInt(), directory);
//Condition check to see if si points to an instance
CND_CONDITION(si != NULL, "Memory allocation for si failed") ;
//store SegmentInfo si
infos.push_back(si);
}
if (format >= 0) {
// in old format the version number may be at the end of the file
if (input->getFilePointer() >= input->length()) {
// old file format without version number
version = Misc::currentTimeMillis();
} else {
// read version
version = input->readLong();
}
}
} _CLFINALLY (
//destroy the inputStream input. The destructor of IndexInput will
//also close the Inputstream input
_CLDELETE(input);
);
}
void IndexWriter::_IndexWriter(const bool create){
//Func - Initialises the instances
//Pre - create indicates if the indexWriter must create a new index located at path or just open it
//Post -
similarity = CL_NS(search)::Similarity::getDefault();
useCompoundFile = true;
if ( directory->getDirectoryType() == RAMDirectory::DirectoryType() )
useCompoundFile = false;
//Create a ramDirectory
ramDirectory = _CLNEW TransactionalRAMDirectory;
CND_CONDITION(ramDirectory != NULL,"ramDirectory is NULL");
//Initialize the writeLock to
writeLock = NULL;
//initialise the settings...
maxFieldLength = DEFAULT_MAX_FIELD_LENGTH;
mergeFactor = DEFAULT_MERGE_FACTOR;
maxMergeDocs = DEFAULT_MAX_MERGE_DOCS;
writeLockTimeout = WRITE_LOCK_TIMEOUT;
commitLockTimeout = COMMIT_LOCK_TIMEOUT;
minMergeDocs = DEFAULT_MAX_BUFFERED_DOCS;
termIndexInterval = DEFAULT_TERM_INDEX_INTERVAL;
//Create a new lock using the name "write.lock"
LuceneLock* newLock = directory->makeLock(IndexWriter::WRITE_LOCK_NAME);
//Condition check to see if newLock has been allocated properly
CND_CONDITION(newLock != NULL, "No memory could be allocated for LuceneLock newLock");
//Try to obtain a write lock
if (!newLock->obtain(writeLockTimeout)){
//Write lock could not be obtained so delete it
_CLDELETE(newLock);
//Reset the instance
_finalize();
//throw an exception because no writelock could be created or obtained
_CLTHROWA(CL_ERR_IO, "Index locked for write or no write access." );
}
//The Write Lock has been obtained so save it for later use
this->writeLock = newLock;
//Create a new lock using the name "commit.lock"
LuceneLock* lock = directory->makeLock(IndexWriter::COMMIT_LOCK_NAME);
//Condition check to see if lock has been allocated properly
CND_CONDITION(lock != NULL, "No memory could be allocated for LuceneLock lock");
LockWith2 with ( lock,commitLockTimeout,this, NULL, create );
{
// CPIXASYNC SCOPED_LOCK_MUTEX(directory->THIS_LOCK) // in- & inter-process sync
SCOPED_LOCK_CRUCIAL_MUTEX(directory->Directory_THIS_LOCK) // in- & inter-process sync
with.run();
}
//Release the commit lock
_CLDELETE(lock);
isOpen = true;
}
int32_t SegmentMerger::mergeFields() {
//Func - Merge the fields of all segments
//Pre - true
//Post - The field infos and field values of all segments have been merged.
if (!mergeDocStores) {
// When we are not merging by doc stores, that means
// all segments were written as part of a single
// autoCommit=false IndexWriter session, so their field
// name -> number mapping are the same. So, we start
// with the fieldInfos of the last segment in this
// case, to keep that numbering.
assert(readers[readers.size()-1]->instanceOf(SegmentReader::getClassName()));
assert(false);//check last...and remove if correct...
SegmentReader* sr = (SegmentReader*)readers[readers.size()-1];
fieldInfos = sr->fieldInfos()->clone();
} else {
//Create a new FieldInfos
fieldInfos = _CLNEW FieldInfos(); // merge field names
}
//Condition check to see if fieldInfos points to a valid instance
CND_CONDITION(fieldInfos != NULL,"Memory allocation for fieldInfos failed");
IndexReader* reader = NULL;
//Iterate through all readers
for (uint32_t i = 0; i < readers.size(); i++){
//get the i-th reader
reader = readers[i];
//Condition check to see if reader points to a valid instance
CND_CONDITION(reader != NULL,"No IndexReader found");
if (reader->instanceOf(SegmentReader::getClassName())) {
SegmentReader* segmentReader = (SegmentReader*) reader;
for (size_t j = 0; j < segmentReader->getFieldInfos()->size(); j++) {
FieldInfo* fi = segmentReader->getFieldInfos()->fieldInfo(j);
fieldInfos->add(fi->name, fi->isIndexed, fi->storeTermVector,
fi->storePositionWithTermVector, fi->storeOffsetWithTermVector,
!reader->hasNorms(fi->name), fi->storePayloads);
}
} else {
StringArrayWithDeletor tmp;
tmp.clear(); reader->getFieldNames(IndexReader::TERMVECTOR_WITH_POSITION_OFFSET, tmp);
addIndexed(reader, fieldInfos, tmp, true, true, true, false);
tmp.clear(); reader->getFieldNames(IndexReader::TERMVECTOR_WITH_POSITION, tmp);
addIndexed(reader, fieldInfos, tmp, true, true, false, false);
tmp.clear(); reader->getFieldNames(IndexReader::TERMVECTOR_WITH_OFFSET, tmp);
addIndexed(reader, fieldInfos, tmp, true, false, true, false);
tmp.clear(); reader->getFieldNames(IndexReader::TERMVECTOR, tmp);
addIndexed(reader, fieldInfos, tmp, true, false, false, false);
tmp.clear(); reader->getFieldNames(IndexReader::STORES_PAYLOADS, tmp);
addIndexed(reader, fieldInfos, tmp, false, false, false, true);
tmp.clear(); reader->getFieldNames(IndexReader::INDEXED, tmp);
addIndexed(reader, fieldInfos, tmp, false, false, false, false);
tmp.clear(); reader->getFieldNames(IndexReader::UNINDEXED, tmp);
if ( tmp.size() > 0 ){
TCHAR** arr = _CL_NEWARRAY(TCHAR*,tmp.size()+1);
tmp.toArray_nullTerminated(arr);
fieldInfos->add((const TCHAR**)arr, false);
_CLDELETE_ARRAY(arr); //no need to delete the contents, since tmp is responsible for it
}
}
}
