void FastCharStream::refill() { int32_t newPosition = bufferLength - tokenStart; if (tokenStart == 0) { // token won't fit in buffer if (buffer == NULL) { // first time: alloc buffer buffer = _CL_NEWARRAY(TCHAR, 2048); _bufferSize = 2048; } else if (bufferLength == _bufferSize) { // grow buffer _bufferSize *= 2; TCHAR* newBuffer = _CL_NEWARRAY(TCHAR, _bufferSize); _tcsncpy(newBuffer, buffer, bufferLength); _CLDELETE_LCARRAY(buffer); buffer = newBuffer; } } else { // shift token to front _tcsncpy(buffer, buffer+tokenStart,newPosition); } bufferLength = newPosition; // update state bufferPosition = newPosition; bufferStart += tokenStart; tokenStart = 0; const TCHAR* charBuf = NULL; int32_t charsRead = // fill space in buffer input->read(charBuf, newPosition, _bufferSize-newPosition); if (charsRead == -1){ _CLTHROWA(CL_ERR_IO, "read past eof"); } else { memcpy(buffer, charBuf, charsRead * sizeof(TCHAR)); // TODO: Can we use the reader buffer instead of copying to our own? bufferLength += charsRead; } }
DefaultSkipListWriter::DefaultSkipListWriter(int32_t skipInterval, int32_t numberOfSkipLevels, int32_t docCount, IndexOutput* freqOutput, IndexOutput* proxOutput): MultiLevelSkipListWriter(skipInterval, numberOfSkipLevels, docCount) { this->freqOutput = freqOutput; this->proxOutput = proxOutput; this->curDoc = this->curPayloadLength = 0; this->curFreqPointer =this->curProxPointer = 0; lastSkipDoc = _CL_NEWARRAY(int32_t,numberOfSkipLevels); lastSkipPayloadLength = _CL_NEWARRAY(int32_t,numberOfSkipLevels); lastSkipFreqPointer = _CL_NEWARRAY(int64_t,numberOfSkipLevels); lastSkipProxPointer = _CL_NEWARRAY(int64_t,numberOfSkipLevels); }
TCHAR* FastCharStream::GetImage() { size_t len = bufferPosition - tokenStart; TCHAR* ret = _CL_NEWARRAY(TCHAR, len + 1); _tcsncpy(ret, buffer+tokenStart, len); ret[len] = 0; // NULL terminated string return ret; }
void MultiReader::initialize(IndexReader** subReaders){ this->subReadersLength = 0; this->subReaders = subReaders; //count the subReaders size if ( subReaders != NULL ){ while ( subReaders[subReadersLength] != NULL ){ subReadersLength++; } } _maxDoc = 0; _numDocs = -1; ones = NULL; starts = _CL_NEWARRAY(int32_t,subReadersLength + 1); // build starts array for (int32_t i = 0; i < subReadersLength; i++) { starts[i] = _maxDoc; // compute maxDocs _maxDoc += subReaders[i]->maxDoc(); if (subReaders[i]->hasDeletions()) _hasDeletions = true; } starts[subReadersLength] = _maxDoc; }
TopDocs* MultiSearcher::_search(Query* query, Filter* filter, const int32_t nDocs) { HitQueue* hq = _CLNEW HitQueue(nDocs); int32_t totalHits = 0; TopDocs* docs; int32_t j; ScoreDoc* scoreDocs; for (int32_t i = 0; i < searchablesLen; i++) { // search each searcher docs = searchables[i]->_search(query, filter, nDocs); totalHits += docs->totalHits; // update totalHits scoreDocs = docs->scoreDocs; for ( j = 0; j <docs->scoreDocsLength; ++j) { // merge scoreDocs int_to hq scoreDocs[j].doc += starts[i]; // convert doc if ( !hq->insert(scoreDocs[j])) break; // no more scores > minScore } _CLDELETE(docs); } int32_t scoreDocsLen = hq->size(); scoreDocs = _CL_NEWARRAY(ScoreDoc, scoreDocsLen); {//MSVC 6 scope fix for (int32_t i = scoreDocsLen-1; i >= 0; --i) // put docs in array scoreDocs[i] = hq->pop(); } //cleanup _CLDELETE(hq); return _CLNEW TopDocs(totalHits, scoreDocs, scoreDocsLen); }
/* Returns the marked-up text for this text fragment */ TCHAR* TextFragment::toString(StringBuffer* buffer) { TCHAR* ret = _CL_NEWARRAY(TCHAR,_textEndPos-_textStartPos+1); _tcsncpy(ret,buffer->getBuffer()+_textStartPos,_textEndPos-_textStartPos); ret[_textEndPos-_textStartPos]=_T('\0'); return ret; }
void RAMIndexOutput::flushBuffer(const uint8_t* src, const int32_t len) { uint8_t* b = NULL; int32_t bufferPos = 0; while (bufferPos != len) { uint32_t bufferNumber = pointer/CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE; int32_t bufferOffset = pointer%CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE; int32_t bytesInBuffer = CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE - bufferOffset; int32_t remainInSrcBuffer = len - bufferPos; int32_t bytesToCopy = bytesInBuffer >= remainInSrcBuffer ? remainInSrcBuffer : bytesInBuffer; if (bufferNumber == file->buffers.size()){ b = _CL_NEWARRAY(uint8_t, CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE); file->buffers.push_back( b ); }else{ b = file->buffers[bufferNumber]; } memcpy(b+bufferOffset, src+bufferPos, bytesToCopy * sizeof(uint8_t)); bufferPos += bytesToCopy; pointer += bytesToCopy; } if (pointer > file->length) file->length = pointer; file->lastModified = Misc::currentTimeMillis(); }
/* DSR:CL_BUG: (See comment for join method in Misc.h): */ TCHAR* Misc::join (const TCHAR* a, const TCHAR* b, const TCHAR* c, const TCHAR* d, const TCHAR* e, const TCHAR* f) { #define LEN(x) (x == NULL ? 0 : _tcslen(x)) const size_t totalLen = LEN(a) + LEN(b) + LEN(c) + LEN(d) + LEN(e) + LEN(f) + sizeof(TCHAR); /* Space for terminator. */ TCHAR* buf = _CL_NEWARRAY(TCHAR, totalLen); buf[0] = 0; if (a != NULL) _tcscat(buf, a); if (b != NULL) _tcscat(buf, b); if (c != NULL) _tcscat(buf, c); if (d != NULL) _tcscat(buf, d); if (e != NULL) _tcscat(buf, e); if (f != NULL) _tcscat(buf, f); return buf; }
int32_t IndexInput::readString(char* buffer, const int32_t maxLength){ TCHAR* buf = _CL_NEWARRAY(TCHAR,maxLength); int32_t ret = -1; try{ ret = readString(buf,maxLength); STRCPY_TtoA(buffer,buf,ret+1); }_CLFINALLY ( _CLDELETE_CARRAY(buf); ) return ret;
TCHAR* QueryFilter::toString() { TCHAR* qt = query->toString(); size_t len = _tcslen(qt) + 14; TCHAR* ret = _CL_NEWARRAY( TCHAR, len ); ret[0] = 0; _sntprintf( ret, len, _T("QueryFilter(%s)"), qt ); _CLDELETE_CARRAY(qt); return ret; }
FuzzyTermEnum::FuzzyTermEnum(IndexReader* reader, Term* term, float_t minSimilarity, size_t _prefixLength): FilteredTermEnum(),d(NULL),dLen(0),_similarity(0),_endEnum(false),searchTerm(_CL_POINTER(term)), text(NULL),textLen(0),prefix(NULL)/* ISH: was STRDUP_TtoT(LUCENE_BLANK_STRING)*/,prefixLength(0), minimumSimilarity(minSimilarity) { CND_PRECONDITION(term != NULL,"term is NULL"); if (minSimilarity >= 1.0f) _CLTHROWA(CL_ERR_IllegalArgument,"minimumSimilarity cannot be greater than or equal to 1"); else if (minSimilarity < 0.0f) _CLTHROWA(CL_ERR_IllegalArgument,"minimumSimilarity cannot be less than 0"); scale_factor = 1.0f / (1.0f - minimumSimilarity); // only now we are safe from a division by zero //TODO: this.field = searchTerm.field(); //The prefix could be longer than the word. //It's kind of silly though. It means we must match the entire word. const size_t fullSearchTermLength = searchTerm->textLength(); const size_t realPrefixLength = _prefixLength > fullSearchTermLength ? fullSearchTermLength : _prefixLength; text = STRDUP_TtoT(searchTerm->text() + realPrefixLength); textLen = fullSearchTermLength - realPrefixLength; prefix = _CL_NEWARRAY(TCHAR,realPrefixLength+1); _tcsncpy(prefix, searchTerm->text(), realPrefixLength); prefix[realPrefixLength]='\0'; prefixLength = realPrefixLength; initializeMaxDistances(); Term* trm = _CLNEW Term(searchTerm->field(), prefix); // _CLNEW Term(term, prefix); -- not intern'd? setEnum(reader->terms(trm)); _CLLDECDELETE(trm); /* LEGACY: //Initialize e to NULL e = NULL; eWidth = 0; eHeight = 0; if(prefixLength > 0 && prefixLength < textLen){ this->prefixLength = prefixLength; prefix = _CL_NEWARRAY(TCHAR,prefixLength+1); _tcsncpy(prefix,text,prefixLength); prefix[prefixLength]='\0'; textLen = prefixLength; text[textLen]='\0'; } */ }
StringBuffer::StringBuffer(){ //Func - Constructor. Allocates a buffer with the default length. //Pre - true //Post - buffer of length bufferLength has been allocated //Initialize bufferLength = LUCENE_DEFAULT_TOKEN_BUFFER_SIZE; len = 0; //Allocate a buffer of length bufferLength buffer = _CL_NEWARRAY(TCHAR,bufferLength); bufferOwner = true; }
StringBuffer::StringBuffer(const int32_t initSize){ //Func - Constructor. Allocates a buffer of length initSize + 1 //Pre - initSize > 0 //Post - A buffer has been allocated of length initSize + 1 //Initialize the bufferLength to initSize + 1 The +1 is for the terminator '\0' bufferLength = initSize + 1; len = 0; //Allocate a buffer of length bufferLength buffer = _CL_NEWARRAY(TCHAR,bufferLength); bufferOwner = true; }
void testRAMDirectorySize(CuTest * tc) { MockRAMDirectory * ramDir = _CLNEW MockRAMDirectory(indexDir); WhitespaceAnalyzer analyzer; IndexWriter * writer = _CLNEW IndexWriter(ramDir, &analyzer, false); writer->optimize(); CuAssertTrue(tc, ramDir->sizeInBytes == ramDir->getRecomputedSizeInBytes(), _T("RAMDir size")); _LUCENE_THREADID_TYPE* threads = _CL_NEWARRAY(_LUCENE_THREADID_TYPE, numThreads); ThreadData * tdata = _CL_NEWARRAY(ThreadData, numThreads); for (int i=0; i<numThreads; i++) { tdata[i].num = i; tdata[i].dir = ramDir; tdata[i].tc = tc; tdata[i].writer = writer; threads[i] = _LUCENE_THREAD_CREATE(&indexDocs, &tdata[i]); } for (int i=0; i<numThreads; i++) { _LUCENE_THREAD_JOIN(threads[i]); } _CLDELETE_ARRAY(threads); _CLDELETE_ARRAY(tdata); writer->optimize(); CuAssertTrue(tc, ramDir->sizeInBytes == ramDir->getRecomputedSizeInBytes(), _T("RAMDir size")); CuAssertEquals(tc, docsToAdd + (numThreads * (docsPerThread-1)), writer->docCount(), _T("document count")); writer->close(); _CLLDELETE(writer); ramDir->close(); _CLLDELETE(ramDir); }
void StringBuffer::clear(){ //Func - Clears the Stringbuffer and resets it to it default empty state //Pre - true //Post - pre(buffer) has been destroyed and a new one has been allocated //Destroy the current buffer if present _CLDELETE_CARRAY(buffer); //Initialize len = 0; bufferLength = LUCENE_DEFAULT_TOKEN_BUFFER_SIZE; //Allocate a buffer of length bufferLength buffer = _CL_NEWARRAY(TCHAR,bufferLength); }
BufferedIndexInput::BufferedIndexInput(const BufferedIndexInput& other): IndexInput(other), buffer(NULL), bufferSize(other.bufferSize), bufferStart(other.bufferStart), bufferLength(other.bufferLength), bufferPosition(other.bufferPosition) { /* DSR: Does the fact that sometime clone.buffer is not NULL even when ** clone.bufferLength is zero indicate memory corruption/leakage? ** if ( clone.buffer != NULL) { */ if (other.bufferLength != 0 && other.buffer != NULL) { buffer = _CL_NEWARRAY(uint8_t,bufferLength); memcpy(buffer,other.buffer,bufferLength * sizeof(uint8_t)); } }
TCHAR* IndexInput::readString(const bool _unique){ int32_t len = readVInt(); if ( len == 0){ if ( _unique ) //todo: does non unique ever occur? return stringDuplicate(LUCENE_BLANK_STRING); else return LUCENE_BLANK_STRING; } TCHAR* ret = _CL_NEWARRAY(TCHAR,len+1); readChars(ret, 0, len); ret[len] = 0; return ret; }
void StringBuffer::reserve(const int32_t size){ if ( bufferLength >= size ) return; bufferLength = size; //Allocate a new buffer of length bufferLength TCHAR* tmp = _CL_NEWARRAY(TCHAR,bufferLength); _tcsncpy(tmp, buffer, len); tmp[len] = '\0'; //destroy the old buffer if (buffer){ _CLDELETE_CARRAY(buffer); } //Assign the new buffer tmp to buffer buffer = tmp; }
TCHAR* StringBuffer::toString(){ //Func - Returns a copy of the current string in the StringBuffer sized equal to the length of the string // in the StringBuffer. //Pre - true //Post - The copied string has been returned //Instantiate a buffer equal to the length len + 1 TCHAR* ret = _CL_NEWARRAY(TCHAR,len + 1); if (ret){ //Copy the string in buffer _tcsncpy(ret, buffer, len); //terminate the string ret[len] = '\0'; } //return the the copy return ret; }
void BufferedIndexInput::refill() { int64_t start = bufferStart + bufferPosition; int64_t end = start + bufferSize; if (end > length()) // don't read past EOF end = length(); bufferLength = (int32_t)(end - start); if (bufferLength == 0) _CLTHROWA(CL_ERR_IO, "IndexInput read past EOF"); if (buffer == NULL){ buffer = _CL_NEWARRAY(uint8_t,bufferSize); // allocate buffer lazily } readInternal(buffer, bufferLength); bufferStart = start; bufferPosition = 0; }
void StringBuffer::growBuffer(const int32_t minLength, const int32_t skippingNInitialChars) { //Func - Has the buffer grown to a minimum length of minLength or bigger and shifts the // current string in buffer by skippingNInitialChars forward //Pre - After growth, must have at least enough room for contents + terminator so // minLength >= skippingNInitialChars + len + 1 // skippingNInitialChars >= 0 //Post - The buffer has been grown to a minimum length of minLength or bigger and // if skippingNInitialChars > 0, the contents of the buffer has beeen shifted // forward by skippingNInitialChars positions as the buffer is reallocated, // leaving the first skippingNInitialChars uninitialized (presumably to be // filled immediately thereafter by the caller). CND_PRECONDITION (skippingNInitialChars >= 0, "skippingNInitialChars is less than zero"); CND_PRECONDITION (minLength >= skippingNInitialChars + len + 1,"skippingNInitialChars is not large enough"); //More aggressive growth strategy to offset smaller default buffer size: if ( !bufferOwner ){ if ( bufferLength<minLength ) _CLTHROWA(CL_ERR_IllegalArgument,"[StringBuffer::grow] un-owned buffer could not be grown"); return; } bufferLength *= 2; //Check that bufferLength is bigger than minLength if (bufferLength < minLength){ //Have bufferLength become minLength because it still was too small bufferLength = minLength; } //Allocate a new buffer of length bufferLength TCHAR* tmp = _CL_NEWARRAY(TCHAR,bufferLength); //The old buffer might not have been null-terminated, so we _tcsncpy //only len bytes, not len+1 bytes (the latter might read one char off the //end of the old buffer), then apply the terminator to the new buffer. _tcsncpy(tmp + skippingNInitialChars, buffer, len); tmp[skippingNInitialChars + len] = '\0'; //destroy the old buffer if (buffer){ _CLDELETE_CARRAY(buffer); } //Assign the new buffer tmp to buffer buffer = tmp; }
int32_t* SegmentMergeInfo::getDocMap(){ if ( docMap == NULL ){ // build array which maps document numbers around deletions if (reader->hasDeletions()) { //Get the total number of documents managed by the reader including the deleted ones int32_t maxDoc = reader->maxDoc(); //Create a map for all documents docMap = _CL_NEWARRAY(int32_t,maxDoc); int32_t j = 0; //Iterate through all the document numbers for (int32_t i = 0; i < maxDoc; i++) { //Check if document i is marked deleted if (reader->isDeleted(i)){ //Document i has not been marked deleted so assign -1 docMap[i] = -1; }else{ docMap[i] = j++; } } } } return docMap; }
//static const TCHAR* Misc::replace_all( const TCHAR* val, const TCHAR* srch, const TCHAR* repl ) { int32_t cnt = 0; size_t repLen = _tcslen(repl); size_t srchLen = _tcslen(srch); size_t srcLen = _tcslen(val); const TCHAR* pos = val; while( (pos = _tcsstr(pos+1, srch)) != NULL ) { ++cnt; } size_t lenNew = (srcLen - (srchLen * cnt)) + (repLen * cnt); TCHAR* ret = _CL_NEWARRAY(TCHAR,lenNew+1); ret[lenNew] = 0; if ( cnt == 0 ){ _tcscpy(ret,val); return ret; } TCHAR* cur = ret; //position of return buffer const TCHAR* lst = val; //position of value buffer pos = val; //searched position of value buffer while( (pos = _tcsstr(pos+1,srch)) != NULL ) { _tcsncpy(cur,lst,pos-lst); //copy till current cur += (pos-lst); lst = pos; //move val position _tcscpy( cur,repl); //copy replace cur += repLen; //move return buffer position lst += srchLen; //move last value buffer position } _tcscpy(cur, lst ); //copy rest of buffer return ret; }
StringBuffer::StringBuffer(const TCHAR* value){ //Func - Constructor. // Creates an instance of Stringbuffer containing a copy of the string value //Pre - value != NULL //Post - An instance of StringBuffer has been created containing the copy of the string value //Initialize the length of the string to be stored in buffer len = (int32_t) _tcslen(value); //Calculate the space occupied in buffer by a copy of value const int32_t occupiedLength = len + 1; // Minimum allocated buffer length is LUCENE_DEFAULT_TOKEN_BUFFER_SIZE. bufferLength = (occupiedLength >= LUCENE_DEFAULT_TOKEN_BUFFER_SIZE ? occupiedLength : LUCENE_DEFAULT_TOKEN_BUFFER_SIZE); //Allocate a buffer of length bufferLength buffer = _CL_NEWARRAY(TCHAR,bufferLength); bufferOwner = true; //Copy the string value into buffer _tcsncpy(buffer, value, occupiedLength); //Assert that the buffer has been terminated at the end of the string CND_PRECONDITION (buffer[len] == '\0', "Buffer was not correctly terminated"); }
/* Run one indexer and 2 searchers against single index as stress test. */ void runThreadingTests(CuTest* tc, Directory& directory){ SimpleAnalyzer ANALYZER; IndexWriter writer(&directory, &ANALYZER, true); // Establish a base index of 100 docs: StringBuffer sb; TCHAR buf[10]; for(int i=0;i<100;i++) { Document d; _i64tot(i,buf,10); d.add(*_CLNEW Field(_T("id"), buf, Field::STORE_YES | Field::INDEX_UNTOKENIZED)); sb.clear(); English::IntToEnglish(i, &sb); d.add(*_CLNEW Field(_T("contents"), sb.getBuffer(), Field::STORE_NO | Field::INDEX_TOKENIZED)); writer.addDocument(&d); } writer.flush(); //read using multiple threads... atomicSearchThreads = _CL_NEWARRAY(_LUCENE_THREADID_TYPE, 4); atomicSearchThreads[0] = _LUCENE_THREAD_CREATE(&atomicIndexTest, &writer); atomicSearchThreads[1] = _LUCENE_THREAD_CREATE(&atomicIndexTest, &writer); atomicSearchThreads[2] = _LUCENE_THREAD_CREATE(&atomicSearchTest, &directory); atomicSearchThreads[3] = _LUCENE_THREAD_CREATE(&atomicSearchTest, &directory); for ( int i=0;i<4;i++ ){ _LUCENE_THREAD_JOIN(atomicSearchThreads[i]); } _CLDELETE_ARRAY(atomicSearchThreads); writer.close(); CuAssert(tc, _T("hit unexpected exception in one of the threads\n"), !atomicSearchFailed); }
TCHAR* FastCharStream::GetSuffix(const int32_t len) { TCHAR* value = _CL_NEWARRAY(TCHAR, len + 1); _tcsncpy(value, buffer+(bufferPosition - len), len); value[len] = 0; // NULL terminated string return value; }
TCHAR* DateField::timeToString(const int64_t time) { TCHAR* buf = _CL_NEWARRAY(TCHAR,DATEFIELD_DATE_LEN + 1); timeToString(time,buf); return buf; }