void MaterializedArray::materialize(const shared_ptr<Query>& query,
MemChunk& materializedChunk,
ConstChunk const& chunk,
MaterializeFormat format)
{
nMaterializedChunks += 1;
materializedChunk.initialize(chunk);
materializedChunk.setBitmapChunk((Chunk*)chunk.getBitmapChunk());
boost::shared_ptr<ConstChunkIterator> src
= chunk.getConstIterator(ChunkIterator::IGNORE_DEFAULT_VALUES|ChunkIterator::IGNORE_EMPTY_CELLS|
(chunk.isSolid() ? ChunkIterator::INTENDED_TILE_MODE : 0));
boost::shared_ptr<ChunkIterator> dst
= materializedChunk.getIterator(query,
(src->getMode() & ChunkIterator::TILE_MODE)|ChunkIterator::ChunkIterator::NO_EMPTY_CHECK|ChunkIterator::SEQUENTIAL_WRITE);
size_t count = 0;
while (!src->end()) {
if (!dst->setPosition(src->getPosition()))
throw SYSTEM_EXCEPTION(SCIDB_SE_MERGE, SCIDB_LE_OPERATION_FAILED) << "setPosition";
dst->writeItem(src->getItem());
count += 1;
++(*src);
}
if (!(src->getMode() & ChunkIterator::TILE_MODE) &&
!chunk.getArrayDesc().hasOverlap()) {
materializedChunk.setCount(count);
}
dst->flush();
}
ConstChunk const& getChunk(AttributeID attr, size_t rowIndex)
{
_chunkAddress.coords[1] = _rowIndex -1;
shared_ptr<Query> query = Query::getValidQueryPtr(_query);
_chunk.initialize(this, &super::getArrayDesc(), _chunkAddress, 0);
shared_ptr<ChunkIterator> chunkIt = _chunk.getIterator(query, ChunkIterator::SEQUENTIAL_WRITE | ChunkIterator::NO_EMPTY_CHECK);
Value v;
if(_buffer[_bufferSize-1] == _delimiter) //add the null-termination character; replace the last delimiter character if one is present
{
v.setSize(_bufferSize);
char *d = (char*) v.data();
memcpy(d, _buffer, _bufferSize);
d[_bufferSize-1] = 0;
}
else
{
v.setSize(_bufferSize+1);
char *d = (char*) v.data();
memcpy(d, _buffer, _bufferSize);
d[_bufferSize] = 0;
}
chunkIt->writeItem(v);
chunkIt->flush();
return _chunk;
}
bool RemoteArray::proceedChunkMsg(AttributeID attId, MemChunk& chunk)
{
boost::shared_ptr<MessageDesc> chunkDesc = _messages[attId];
_messages[attId].reset();
StatisticsScope sScope(_statistics);
boost::shared_ptr<scidb_msg::Chunk> chunkMsg = chunkDesc->getRecord<scidb_msg::Chunk>();
currentStatistics->receivedSize += chunkDesc->getMessageSize();
currentStatistics->receivedMessages++;
if (!chunkMsg->eof())
{
LOG4CXX_TRACE(logger, "RemoteArray received next chunk message");
assert(chunkDesc->getBinary());
const int compMethod = chunkMsg->compression_method();
const size_t decompressedSize = chunkMsg->decompressed_size();
Address firstElem;
firstElem.attId = attId;
firstElem.arrId = getArrayDesc().getId();
for (int i = 0; i < chunkMsg->coordinates_size(); i++) {
firstElem.coords.push_back(chunkMsg->coordinates(i));
}
chunk.initialize(this, &desc, firstElem, compMethod);
chunk.setSparse(chunkMsg->sparse());
chunk.setCount(chunkMsg->count());
boost::shared_ptr<CompressedBuffer> compressedBuffer = dynamic_pointer_cast<CompressedBuffer>(chunkDesc->getBinary());
compressedBuffer->setCompressionMethod(compMethod);
compressedBuffer->setDecompressedSize(decompressedSize);
chunk.decompress(*compressedBuffer);
LOG4CXX_TRACE(logger, "RemoteArray initializes next chunk");
requestNextChunk(attId);
return true;
}
else
{
return false;
}
}
void MessageHandleJob::_handleChunkOrAggregateChunk(bool isAggregateChunk)
{
boost::shared_ptr<scidb_msg::Chunk> chunkRecord = _messageDesc->getRecord<scidb_msg::Chunk>();
assert(!chunkRecord->eof());
assert(_query);
RWLock::ErrorChecker noopEc;
ScopedRWLockRead shared(_query->queryLock, noopEc);
try
{
if (! isAggregateChunk) {
// TODO: Apply it to statistics of current SG
currentStatistics->receivedSize += _messageDesc->getMessageSize();
currentStatistics->receivedMessages++;
}
LOG4CXX_TRACE(logger, "Next chunk message was received")
const int compMethod = chunkRecord->compression_method();
const size_t decompressedSize = chunkRecord->decompressed_size();
const AttributeID attributeID = chunkRecord->attribute_id();
const size_t count = chunkRecord->count();
boost::shared_ptr<SGContext> sgCtx = dynamic_pointer_cast<SGContext>(_query->getOperatorContext());
if (sgCtx == NULL) {
throw (SYSTEM_EXCEPTION(SCIDB_SE_INTERNAL, SCIDB_LE_UNKNOWN_CTX)
<< typeid(*_query->getOperatorContext()).name());
}
boost::shared_ptr<Array> outputArray = sgCtx->_resultSG;
ScopedMutexLock cs(_query->resultCS);
boost::shared_ptr<ArrayIterator> outputIter = outputArray->getIterator(attributeID);
const bool shouldCacheEmptyBitmap = sgCtx->_shouldCacheEmptyBitmap;
const ArrayDesc& desc = outputArray->getArrayDesc();
const size_t sourceId = _query->mapPhysicalToLogical(_messageDesc->getSourceInstanceID());
const bool isEmptyable = (desc.getEmptyBitmapAttribute() != NULL);
const bool isEmptyIndicator = isEmptyable && (attributeID+1==desc.getAttributes().size());
const bool rle = chunkRecord->rle();
if (isAggregateChunk) {
assert(! isEmptyIndicator);
}
Coordinates coordinates;
for (int i = 0; i < chunkRecord->coordinates_size(); i++) {
coordinates.push_back(chunkRecord->coordinates(i));
}
if (!isAggregateChunk && sgCtx->_targetVersioned)
{
sgCtx->_newChunks.insert(coordinates);
}
boost::shared_ptr<CompressedBuffer> compressedBuffer = dynamic_pointer_cast<CompressedBuffer>(_messageDesc->getBinary());
if (compressedBuffer) {
assert(compressedBuffer->getData());
PinBuffer pin(*compressedBuffer); // this line protects compressedBuffer->data from being freed, allowing MemChunk::decompressed(*compressedBuffer) to be called multiple times.
compressedBuffer->setCompressionMethod(compMethod);
compressedBuffer->setDecompressedSize(decompressedSize);
Chunk* outChunk;
// temporary MemArray objects
shared_ptr<MemChunk> pTmpChunk = make_shared<MemChunk>(); // make it a shared pointer, because the bitmap chunk needs to be preserved across threads.
MemChunk closure;
// the PinBuffer objects protect tmpChunk and closure, respectively.
shared_ptr<PinBuffer> pinTmpChunk, pinClosure;
if (outputIter->setPosition(coordinates)) { // existing chunk
outChunk = &outputIter->updateChunk();
if (! isAggregateChunk) {
if (outChunk->getDiskChunk() != NULL) {
throw SYSTEM_EXCEPTION(SCIDB_SE_MERGE, SCIDB_LE_CANT_UPDATE_CHUNK);
}
outChunk->setCount(0); // unknown
}
// if (a) either dest is NULL or merge by bitwise-or is possible; and (b) src is not compressed
char* dst = static_cast<char*>(outChunk->getData());
if ( (dst == NULL || (outChunk->isPossibleToMergeByBitwiseOr() && !chunkRecord->sparse() && !chunkRecord->rle()))
&&
compMethod == 0 )
{
char const* src = (char const*)compressedBuffer->getData();
// Special care is needed if shouldCacheEmptyBitmap.
// - If this is the empty bitmap, store it in the SGContext.
// - Otherwise, add the empty bitmap from the SGContext to the chunk's data.
if (shouldCacheEmptyBitmap) {
initMemChunkFromNetwork(pTmpChunk, pinTmpChunk, outputArray, coordinates, attributeID,
compMethod, chunkRecord->sparse() || outChunk->isSparse(), rle, compressedBuffer);
if (isEmptyIndicator) {
sgCtx->setCachedEmptyBitmapChunk(sourceId, pTmpChunk, coordinates);
} else {
shared_ptr<ConstRLEEmptyBitmap> cachedBitmap = sgCtx->getCachedEmptyBitmap(sourceId, coordinates);
assert(cachedBitmap);
pinClosure = make_shared<PinBuffer>(closure);
closure.initialize(*pTmpChunk);
pTmpChunk->makeClosure(closure, cachedBitmap);
src = static_cast<char const*>(closure.getData());
}
}
if (dst == NULL) {
outChunk->allocateAndCopy(src, decompressedSize, chunkRecord->sparse(), chunkRecord->rle(), count, _query);
} else {
outChunk->mergeByBitwiseOr(src, decompressedSize, _query);
}
} else {
initMemChunkFromNetwork(pTmpChunk, pinTmpChunk, outputArray, coordinates, attributeID,
compMethod, chunkRecord->sparse() || outChunk->isSparse(), rle, compressedBuffer);
ConstChunk const* srcChunk = &(*pTmpChunk);
// Special care is needed if shouldCacheEmptyBitmap.
// - If this is the empty bitmap, store it in the SGContext.
// - Otherwise, add the empty bitmap from the SGContext to the chunk's data.
if (shouldCacheEmptyBitmap) {
if (isEmptyIndicator) {
sgCtx->setCachedEmptyBitmapChunk(sourceId, pTmpChunk, coordinates);
} else {
shared_ptr<ConstRLEEmptyBitmap> cachedBitmap = sgCtx->getCachedEmptyBitmap(sourceId, coordinates);
assert(cachedBitmap);
pinClosure = make_shared<PinBuffer>(closure);
closure.initialize(*pTmpChunk);
pTmpChunk->makeClosure(closure, cachedBitmap);
srcChunk = &closure;
}
}
if (isAggregateChunk) {
AggregatePtr aggregate = sgCtx->_aggregateList[attributeID];
if (!isEmptyable && rle) {
assert(!shouldCacheEmptyBitmap);
assert(srcChunk==&(*pTmpChunk));
outChunk->nonEmptyableAggregateMerge(*srcChunk, aggregate, _query);
} else {
outChunk->aggregateMerge(*srcChunk, aggregate, _query);
}
} else {
outChunk->merge(*srcChunk, _query);
}
}
} else { // new chunk
outChunk = &outputIter->newChunk(coordinates, compMethod);
outChunk->setSparse(chunkRecord->sparse());
outChunk->setRLE(rle);
shared_ptr<CompressedBuffer> myCompressedBuffer = compressedBuffer;
// Special care is needed if shouldCacheEmptyBitmap.
// - If this is the empty bitmap, store it in the SGContext.
// - Otherwise, add the empty bitmap from the SGContext to the chunk's data.
if (shouldCacheEmptyBitmap) {
initMemChunkFromNetwork(pTmpChunk, pinTmpChunk, outputArray, coordinates, attributeID,
compMethod, chunkRecord->sparse() || outChunk->isSparse(), rle, compressedBuffer);
if (isEmptyIndicator) {
sgCtx->setCachedEmptyBitmapChunk(sourceId, pTmpChunk, coordinates);
} else {
shared_ptr<ConstRLEEmptyBitmap> cachedBitmap = sgCtx->getCachedEmptyBitmap(sourceId, coordinates);
assert(cachedBitmap);
myCompressedBuffer = make_shared<CompressedBuffer>();
pTmpChunk->compress(*myCompressedBuffer, cachedBitmap);
}
}
outChunk->decompress(*myCompressedBuffer);
outChunk->setCount(count);
outChunk->write(_query);
} // end if (outputIter->setPosition(coordinates))
assert(checkChunkMagic(*outChunk));
} // end if (compressedBuffer)
sgSync();
LOG4CXX_TRACE(logger, "Chunk was stored")
}
catch(const Exception& e)
{
sgSync();
throw;
}
}
