void * FixedAllocator::Allocate( void )
{
// prove either emptyChunk_ points nowhere, or points to a truly empty Chunk.
assert( ( NULL == emptyChunk_ ) || ( emptyChunk_->HasAvailable( numBlocks_ ) ) );
assert( CountEmptyChunks() < 2 );
if ( ( NULL == allocChunk_ ) || allocChunk_->IsFilled() )
{
if ( NULL != emptyChunk_ )
{
allocChunk_ = emptyChunk_;
emptyChunk_ = NULL;
}
else
{
for ( ChunkIter i( chunks_.begin() ); ; ++i )
{
if ( chunks_.end() == i )
{
if ( !MakeNewChunk() )
return NULL;
break;
}
if ( !i->IsFilled() )
{
allocChunk_ = &*i;
break;
}
}
}
}
else if ( allocChunk_ == emptyChunk_)
// detach emptyChunk_ from allocChunk_, because after
// calling allocChunk_->Allocate(blockSize_); the chunk
// is no longer empty.
emptyChunk_ = NULL;
assert( allocChunk_ != NULL );
assert( !allocChunk_->IsFilled() );
void * place = allocChunk_->Allocate( blockSize_ );
// prove either emptyChunk_ points nowhere, or points to a truly empty Chunk.
assert( ( NULL == emptyChunk_ ) || ( emptyChunk_->HasAvailable( numBlocks_ ) ) );
assert( CountEmptyChunks() < 2 );
#ifdef LOKI_CHECK_FOR_CORRUPTION
if ( allocChunk_->IsCorrupt( numBlocks_, blockSize_, true ) )
{
assert( false );
return NULL;
}
#endif
return place;
}
GLSLChunker::Chunk
GLSLChunker::chunkLine(const std::string& line) const
{
Chunks chunks;
read(line, chunks);
return chunks.size() > 0 ? chunks[0] : Chunk();
}
bool FixedAllocator::Deallocate( void * p, Chunk * hint )
{
assert(!chunks_.empty());
assert(&chunks_.front() <= deallocChunk_);
assert(&chunks_.back() >= deallocChunk_);
assert( &chunks_.front() <= allocChunk_ );
assert( &chunks_.back() >= allocChunk_ );
assert( CountEmptyChunks() < 2 );
Chunk * foundChunk = ( NULL == hint ) ? VicinityFind( p ) : hint;
if ( NULL == foundChunk )
return false;
assert( foundChunk->HasBlock( p, numBlocks_ * blockSize_ ) );
#ifdef LOKI_CHECK_FOR_CORRUPTION
if ( foundChunk->IsCorrupt( numBlocks_, blockSize_, true ) )
{
assert( false );
return false;
}
if ( foundChunk->IsBlockAvailable( p, numBlocks_, blockSize_ ) )
{
assert( false );
return false;
}
#endif
deallocChunk_ = foundChunk;
DoDeallocate(p);
assert( CountEmptyChunks() < 2 );
return true;
}
const Chunk * FixedAllocator::HasBlock( void * p ) const
{
const std::size_t chunkLength = numBlocks_ * blockSize_;
for ( ChunkCIter it( chunks_.begin() ); it != chunks_.end(); ++it )
{
const Chunk & chunk = *it;
if ( chunk.HasBlock( p, chunkLength ) )
return &chunk;
}
return NULL;
}
void
allocateChunk ()
{
chunks.push_back (new Chunk); // requires c++17 new with alignment
for (Storage& storage : chunks.back ())
{
FreeListNode *node = reinterpret_cast<FreeListNode*> (&storage);
new (node) FreeListNode;
freeList.push_front (*node);
}
}
void FsDirectoryImpl::deleteFile(const std::string& filename, Chunks& deletedChunks)
{
Path path(filename);
DirNode* pDirNode;
INode* pINode;
//preconditions check
if (path.isPureDir() ){
LOG4CPLUS_WARN(m_logger, "deleteFile, but filename is a pure dir " << filename);
throw InvalidFileOrDirName("deleteFile : is pure dir");
}
//#TODO, lock
pDirNode = m_pDirTree->findDirNode(path.getDirName());
if(!pDirNode) {
LOG4CPLUS_WARN(m_logger, "directory not exist :" << path.getDirName());
throw NoSuchFileOrDir("deleteFile : directory");
}
if (!(pINode = pDirNode->findFile(path.getFileName()))){
LOG4CPLUS_WARN(m_logger, "file not exist : " << path.getFileName());
throw NoSuchFileOrDir("deleteFile : filename");
}
Chunks chunks = pINode->getChunks();
deletedChunks.assign(chunks.begin(), chunks.end());
//#BUGFIX, [#13] INode:findChunk segment fatal error
//ChunkTable not synchronized with deleteFile
//
for( int i = 0; i < chunks.size() ; i++){
m_pChunkTable->deleteItem(chunks[i].id);
}
//#BUGFIX, [#17] INode:findChunk segment fatal error
//NewChunkTable not synchronized with deleteFile
//
vector<Long> deleteNewChunkIds;
if(m_pNewChunkTable->deleteItem(pINode, deleteNewChunkIds)){
string chunkIdList;
for (int i = 0 ; i < deleteNewChunkIds.size(); i++){
chunkIdList += deleteNewChunkIds[i];
chunkIdList += " ";
}
LOG4CPLUS_WARN(m_logger, "delete newchunks not committed: " << chunkIdList);
}
pDirNode->deleteFile(path.getFileName());
}
bool FixedAllocator::TrimChunkList( void )
{
if ( chunks_.empty() )
{
assert( NULL == allocChunk_ );
assert( NULL == deallocChunk_ );
}
if ( chunks_.size() == chunks_.capacity() )
return false;
// Use the "make-a-temp-and-swap" trick to remove excess capacity.
Chunks( chunks_ ).swap( chunks_ );
return true;
}
void clear()
{
for(typename Objects::reverse_iterator i = objects.rbegin(); i != objects.rend(); ++i)
(*i)->~T();
objects.clear();
// FIXME: We don't have to delete the chunks, instead we should
// just reset the pointer to start and reuse them
for(typename Chunks::reverse_iterator i = chunks.rbegin(); i != chunks.rend(); ++i)
{
delete[] *i;
}
chunks.clear();
next_free = 0;
}
char* allocate(size_t size)
{
assert(size <= chunk_size);
if (chunks.empty() ||
(next_free + size) > chunk_size)
{
char* chunk = new char[chunk_size];
chunks.push_back(chunk);
next_free = 0;
}
char* ptr = chunks.back() + next_free;
next_free += size;
return ptr;
}
void
GLSLChunker::dump(const std::string& msg, const Chunks& chunks) const
{
OE_INFO << msg << "\n";
for (Chunks::const_iterator i = chunks.begin(); i != chunks.end(); ++i)
{
std::string type =
i->type == Chunk::TYPE_DIRECTIVE ? "DIRECTIVE" :
i->type == Chunk::TYPE_COMMENT ? "COMMENT" :
i->type == Chunk::TYPE_STATEMENT ? "STATEMENT" :
i->type == Chunk::TYPE_FUNCTION ? "FUNCTION" :
"????????";
OE_INFO << " " << type << ": " << i->text << std::endl;
}
}
void CharCommand::undo() {
switch (_cmd) {
case insert:
_chunks->removeAt(_charPos);
break;
case overwrite:
_chunks->overwrite(_charPos, _oldChar);
_chunks->setDataChanged(_charPos, _wasChanged);
break;
case removeAt:
_chunks->insert(_charPos, _oldChar);
_chunks->setDataChanged(_charPos, _wasChanged);
break;
}
}
void CharCommand::redo() {
switch (_cmd) {
case insert:
_chunks->insert(_charPos, _newChar);
break;
case overwrite:
_oldChar = (*_chunks)[_charPos];
_wasChanged = _chunks->dataChanged(_charPos);
_chunks->overwrite(_charPos, _newChar);
break;
case removeAt:
_oldChar = (*_chunks)[_charPos];
_wasChanged = _chunks->dataChanged(_charPos);
_chunks->removeAt(_charPos);
break;
}
}
void * FixedAllocator::Allocate()
{
if (!last_alloc_ || last_alloc_->blocks_available_ == 0)
{
// no memory available in the cached chunk, try to find one.
Chunks::iterator e = chunks_.end();
Chunks::iterator i = chunks_.begin();
for (; i!=e; ++i)
{
if (i->blocks_available_ > 0)
{
// found a chunk!
last_alloc_ = &*i;
break;
}
}
if (i == e)
{
// no chunks have blocks available, add a new one.
chunks_.reserve(chunks_.size()+1);
MemChunk chunk;
chunk.Init(block_size_, blocks_);
chunks_.push_back(chunk);
last_alloc_ = &chunks_.back();
last_dealloc_ = &chunks_.back();
}
}
// chunk pointer vaild check
assert(last_alloc_ != NULL);
assert(last_alloc_->blocks_available_ > 0);
return last_alloc_->Allocate(block_size_);
}
std::size_t FixedAllocator::CountEmptyChunks( void ) const
{
#ifdef DO_EXTRA_LOKI_TESTS
// This code is only used for specialized tests of the allocator.
// It is #ifdef-ed so that its O(C) complexity does not overwhelm the
// functions which call it.
std::size_t count = 0;
for ( ChunkCIter it( chunks_.begin() ); it != chunks_.end(); ++it )
{
const Chunk & chunk = *it;
if ( chunk.HasAvailable( numBlocks_ ) )
++count;
}
return count;
#else
return ( NULL == emptyChunk_ ) ? 0 : 1;
#endif
}
void FixedAllocator::DoDeallocate(void* p)
{
// Show that deallocChunk_ really owns the block at address p.
assert( deallocChunk_->HasBlock( p, numBlocks_ * blockSize_ ) );
// Either of the next two assertions may fail if somebody tries to
// delete the same block twice.
assert( emptyChunk_ != deallocChunk_ );
assert( !deallocChunk_->HasAvailable( numBlocks_ ) );
// prove either emptyChunk_ points nowhere, or points to a truly empty Chunk.
assert( ( NULL == emptyChunk_ ) || ( emptyChunk_->HasAvailable( numBlocks_ ) ) );
// call into the chunk, will adjust the inner list but won't release memory
deallocChunk_->Deallocate(p, blockSize_);
if ( deallocChunk_->HasAvailable( numBlocks_ ) )
{
assert( emptyChunk_ != deallocChunk_ );
// deallocChunk_ is empty, but a Chunk is only released if there are 2
// empty chunks. Since emptyChunk_ may only point to a previously
// cleared Chunk, if it points to something else besides deallocChunk_,
// then FixedAllocator currently has 2 empty Chunks.
if ( NULL != emptyChunk_ )
{
// If last Chunk is empty, just change what deallocChunk_
// points to, and release the last. Otherwise, swap an empty
// Chunk with the last, and then release it.
Chunk * lastChunk = &chunks_.back();
if ( lastChunk == deallocChunk_ )
deallocChunk_ = emptyChunk_;
else if ( lastChunk != emptyChunk_ )
std::swap( *emptyChunk_, *lastChunk );
assert( lastChunk->HasAvailable( numBlocks_ ) );
lastChunk->Release();
chunks_.pop_back();
if ( ( allocChunk_ == lastChunk ) || allocChunk_->IsFilled() )
allocChunk_ = deallocChunk_;
}
emptyChunk_ = deallocChunk_;
}
// prove either emptyChunk_ points nowhere, or points to a truly empty Chunk.
assert( ( NULL == emptyChunk_ ) || ( emptyChunk_->HasAvailable( numBlocks_ ) ) );
}
void FixedAllocator::Swap(FixedAllocator& other)
{
using namespace std;
chunks_.swap(other.chunks_);
swap(block_size_, other.block_size_);
swap(blocks_, other.blocks_);
swap(last_alloc_, other.last_alloc_);
swap(last_dealloc_, other.last_dealloc_);
}
void
GLSLChunker::write(const Chunks& input, std::string& output) const
{
std::stringstream buf;
for(int i=0; i<input.size(); ++i)
{
buf << input[i].text << "\n";
}
output = buf.str();
}
FixedAllocator::Chunk * FixedAllocator::VicinityFind( void * p )
{
if ( chunks_.empty() ) return NULL;
assert(deallocChunk_);
unsigned char * pc = static_cast< unsigned char * >( p );
const std::size_t chunkLength = numBlocks_ * blockSize_;
Chunk* lo = deallocChunk_;
Chunk* hi = deallocChunk_ + 1;
Chunk* loBound = &chunks_.front();
Chunk* hiBound = &chunks_.back() + 1;
// Special case: deallocChunk_ is the last in the array
if (hi == hiBound) hi = NULL;
for (;;)
{
if (lo)
{
if ( lo->HasBlock( pc, chunkLength ) ) return lo;
if ( lo == loBound )
{
lo = NULL;
if ( NULL == hi ) break;
}
else --lo;
}
if (hi)
{
if ( hi->HasBlock( pc, chunkLength ) ) return hi;
if ( ++hi == hiBound )
{
hi = NULL;
if ( NULL == lo ) break;
}
}
}
return NULL;
}
void
GLSLChunker::replace(Chunks& input, const std::string& pattern, const std::string& replacement) const
{
for(int i=0; i<input.size(); ++i)
{
Chunk& chunk = input[i];
osgEarth::replaceIn(chunk.text, pattern, replacement);
for (unsigned t = 0; t<chunk.tokens.size(); ++t)
osgEarth::replaceIn(chunk.tokens[t], pattern, replacement);
}
}
bool FixedAllocator::MakeNewChunk( void )
{
bool allocated = false;
try
{
std::size_t size = chunks_.size();
// Calling chunks_.reserve *before* creating and initializing the new
// Chunk means that nothing is leaked by this function in case an
// exception is thrown from reserve.
if ( chunks_.capacity() == size )
{
if ( 0 == size ) size = 4;
chunks_.reserve( size * 2 );
}
Chunk newChunk;
allocated = newChunk.Init( blockSize_, numBlocks_ );
if ( allocated )
chunks_.push_back( newChunk );
}
catch ( ... )
{
allocated = false;
}
if ( !allocated ) return false;
allocChunk_ = &chunks_.back();
deallocChunk_ = &chunks_.front();
return true;
}
void * FixedAllocator::Allocate( void )
{
// prove either emptyChunk_ points nowhere, or points to a truly empty Chunk.
assert( ( NULL == emptyChunk_ ) || ( emptyChunk_->HasAvailable( numBlocks_ ) ) );
if ( ( NULL == allocChunk_ ) || allocChunk_->IsFilled() )
{
if ( NULL != emptyChunk_ )
{
allocChunk_ = emptyChunk_;
emptyChunk_ = NULL;
}
else
{
for ( ChunkIter i( chunks_.begin() ); ; ++i )
{
if ( chunks_.end() == i )
{
if ( !MakeNewChunk() )
return NULL;
break;
}
if ( !i->IsFilled() )
{
allocChunk_ = &*i;
break;
}
}
}
}
assert( allocChunk_ != NULL );
assert( !allocChunk_->IsFilled() );
// prove either emptyChunk_ points nowhere, or points to a truly empty Chunk.
assert( ( NULL == emptyChunk_ ) || ( emptyChunk_->HasAvailable( numBlocks_ ) ) );
return allocChunk_->Allocate(blockSize_);
}
void FixedAllocator::DoDeallocate(void* p)
{
assert( deallocChunk_->HasBlock( static_cast< unsigned char * >( p ),
numBlocks_ * blockSize_ ) );
// prove either emptyChunk_ points nowhere, or points to a truly empty Chunk.
assert( ( NULL == emptyChunk_ ) || ( emptyChunk_->HasAvailable( numBlocks_ ) ) );
// call into the chunk, will adjust the inner list but won't release memory
deallocChunk_->Deallocate(p, blockSize_);
if ( deallocChunk_->HasAvailable( numBlocks_ ) )
{
assert( emptyChunk_ != deallocChunk_ );
// deallocChunk_ is empty, but a Chunk is only released if there are 2
// empty chunks. Since emptyChunk_ may only point to a previously
// cleared Chunk, if it points to something else besides deallocChunk_,
// then FixedAllocator currently has 2 empty Chunks.
if ( NULL != emptyChunk_ )
{
// If last Chunk is empty, just change what deallocChunk_
// points to, and release the last. Otherwise, swap an empty
// Chunk with the last, and then release it.
Chunk * lastChunk = &chunks_.back();
if ( lastChunk == deallocChunk_ )
deallocChunk_ = emptyChunk_;
else if ( lastChunk != emptyChunk_ )
std::swap( *emptyChunk_, *lastChunk );
assert( lastChunk->HasAvailable( numBlocks_ ) );
lastChunk->Release();
chunks_.pop_back();
allocChunk_ = deallocChunk_;
}
emptyChunk_ = deallocChunk_;
}
// prove either emptyChunk_ points nowhere, or points to a truly empty Chunk.
assert( ( NULL == emptyChunk_ ) || ( emptyChunk_->HasAvailable( numBlocks_ ) ) );
}
bool FixedAllocator::Deallocate( void * p, bool doChecks )
{
if ( doChecks )
{
assert(!chunks_.empty());
assert(&chunks_.front() <= deallocChunk_);
assert(&chunks_.back() >= deallocChunk_);
assert( &chunks_.front() <= allocChunk_ );
assert( &chunks_.back() >= allocChunk_ );
}
Chunk * foundChunk = VicinityFind( p );
if ( doChecks )
{
assert( NULL != foundChunk );
}
else if ( NULL == foundChunk )
return false;
deallocChunk_ = foundChunk;
DoDeallocate(p);
return true;
}
void Tests::chunksPerformance()
{
qDebug() << "===== chunksPerformance() =====";
Chunks chunks;
for (int i = 0; i < 100000; i++)
{
QSharedPointer<Chunk> chunk(new Chunk(nullptr, 0, 0));
chunk->setHash(Common::Hash::rand());
chunks.add(chunk);
}
Common::Hashes hashes;
const int nbHashes = 100;
for (int i = 0; i < nbHashes; i++)
hashes << Common::Hash::rand();
for (int i = 0; i < 100000000; i++)
{
if(chunks.contains(hashes[i % nbHashes]))
QFAIL("chunks cannot contains a random chunk");
}
}
MemChunk * FixedAllocator::FindChunk(void *p)
{
assert(last_dealloc_);
assert(!chunks_.empty());
MemChunk *lo_b = &chunks_.front();
MemChunk *hi_b = &chunks_.back()+1;
MemChunk *lo = last_dealloc_;
MemChunk *hi = last_dealloc_+1;
if (hi == hi_b) hi = NULL;
for (;;)
{
if (lo)
{
if (InChunk(lo, p))
return lo;
if (lo == lo_b)
lo = NULL;
else
--lo;
}
if (hi)
{
if (InChunk(hi, p))
return hi;
if (++hi == hi_b)
hi = NULL;
}
}
// Never gets here.
assert(false);
return NULL;
}
bool FixedAllocator::TrimEmptyChunk( void )
{
// prove either emptyChunk_ points nowhere, or points to a truly empty Chunk.
assert( ( NULL == emptyChunk_ ) || ( emptyChunk_->HasAvailable( numBlocks_ ) ) );
if ( NULL == emptyChunk_ ) return false;
// If emptyChunk_ points to valid Chunk, then chunk list is not empty.
assert( !chunks_.empty() );
// And there should be exactly 1 empty Chunk.
assert( 1 == CountEmptyChunks() );
Chunk * lastChunk = &chunks_.back();
if ( lastChunk != emptyChunk_ )
std::swap( *emptyChunk_, *lastChunk );
assert( lastChunk->HasAvailable( numBlocks_ ) );
lastChunk->Release();
chunks_.pop_back();
if ( chunks_.empty() )
{
allocChunk_ = NULL;
deallocChunk_ = NULL;
}
else
{
if ( deallocChunk_ == emptyChunk_ )
{
deallocChunk_ = &chunks_.front();
assert( deallocChunk_->blocksAvailable_ < numBlocks_ );
}
if ( allocChunk_ == emptyChunk_ )
{
allocChunk_ = &chunks_.back();
assert( allocChunk_->blocksAvailable_ < numBlocks_ );
}
}
emptyChunk_ = NULL;
assert( 0 == CountEmptyChunks() );
return true;
}
void FixedAllocator::Deallocate(void *p)
{
assert(last_dealloc_);
assert(!chunks_.empty());
// find the chunk that holds the pointer.
last_dealloc_ = FindChunk(p);
assert(last_dealloc_);
// found the chunk, so forwards deallocation request to it.
last_dealloc_->Deallocate(p, block_size_);
// the chunk is empty, should we free it?
if (last_dealloc_->blocks_available_ == blocks_)
{
MemChunk *last_chunk = &chunks_.back();
if (last_chunk = last_dealloc_)
{
if (chunks_.size() > 1 && last_dealloc_[-1].blocks_available_ == blocks_)
{
// there are two free chunk, so free one.
last_chunk->Free();
chunks_.pop_back();
last_alloc_ = last_dealloc_ = &chunks_.front();
}
}
else if (last_chunk->blocks_available_ == blocks_)
{
// there are two free chunk, so free one.
last_chunk->Free();
chunks_.pop_back();
last_alloc_ = last_dealloc_;
}
else
{
// move the empty chunk to the bottom.
std::swap(*last_dealloc_, *last_chunk);
last_alloc_ = last_dealloc_;
}
}
}
bool FixedAllocator::IsCorrupt( void ) const
{
const bool isEmpty = chunks_.empty();
ChunkCIter start( chunks_.begin() );
ChunkCIter last( chunks_.end() );
const size_t emptyChunkCount = CountEmptyChunks();
if ( isEmpty )
{
if ( start != last )
{
assert( false );
return true;
}
if ( 0 < emptyChunkCount )
{
assert( false );
return true;
}
if ( NULL != deallocChunk_ )
{
assert( false );
return true;
}
if ( NULL != allocChunk_ )
{
assert( false );
return true;
}
if ( NULL != emptyChunk_ )
{
assert( false );
return true;
}
}
else
{
const Chunk * front = &chunks_.front();
const Chunk * back = &chunks_.back();
if ( start >= last )
{
assert( false );
return true;
}
if ( back < deallocChunk_ )
{
assert( false );
return true;
}
if ( back < allocChunk_ )
{
assert( false );
return true;
}
if ( front > deallocChunk_ )
{
assert( false );
return true;
}
if ( front > allocChunk_ )
{
assert( false );
return true;
}
switch ( emptyChunkCount )
{
case 0:
if ( emptyChunk_ != NULL )
{
assert( false );
return true;
}
break;
case 1:
if ( emptyChunk_ == NULL )
{
assert( false );
return true;
}
if ( back < emptyChunk_ )
{
assert( false );
return true;
}
if ( front > emptyChunk_ )
{
assert( false );
return true;
}
if ( !emptyChunk_->HasAvailable( numBlocks_ ) )
{
// This may imply somebody tried to delete a block twice.
assert( false );
return true;
}
break;
default:
assert( false );
return true;
}
for ( ChunkCIter it( start ); it != last; ++it )
{
const Chunk & chunk = *it;
if ( chunk.IsCorrupt( numBlocks_, blockSize_, true ) )
return true;
}
}
return false;
}