void MaterializedArray::materialize(const shared_ptr<Query>& query, MemChunk& materializedChunk, ConstChunk const& chunk, MaterializeFormat format) { nMaterializedChunks += 1; materializedChunk.initialize(chunk); materializedChunk.setBitmapChunk((Chunk*)chunk.getBitmapChunk()); boost::shared_ptr<ConstChunkIterator> src = chunk.getConstIterator(ChunkIterator::IGNORE_DEFAULT_VALUES|ChunkIterator::IGNORE_EMPTY_CELLS| (chunk.isSolid() ? ChunkIterator::INTENDED_TILE_MODE : 0)); boost::shared_ptr<ChunkIterator> dst = materializedChunk.getIterator(query, (src->getMode() & ChunkIterator::TILE_MODE)|ChunkIterator::ChunkIterator::NO_EMPTY_CHECK|ChunkIterator::SEQUENTIAL_WRITE); size_t count = 0; while (!src->end()) { if (!dst->setPosition(src->getPosition())) throw SYSTEM_EXCEPTION(SCIDB_SE_MERGE, SCIDB_LE_OPERATION_FAILED) << "setPosition"; dst->writeItem(src->getItem()); count += 1; ++(*src); } if (!(src->getMode() & ChunkIterator::TILE_MODE) && !chunk.getArrayDesc().hasOverlap()) { materializedChunk.setCount(count); } dst->flush(); }
ConstChunk const& getChunk(AttributeID attr, size_t rowIndex) { _chunkAddress.coords[1] = _rowIndex -1; shared_ptr<Query> query = Query::getValidQueryPtr(_query); _chunk.initialize(this, &super::getArrayDesc(), _chunkAddress, 0); shared_ptr<ChunkIterator> chunkIt = _chunk.getIterator(query, ChunkIterator::SEQUENTIAL_WRITE | ChunkIterator::NO_EMPTY_CHECK); Value v; if(_buffer[_bufferSize-1] == _delimiter) //add the null-termination character; replace the last delimiter character if one is present { v.setSize(_bufferSize); char *d = (char*) v.data(); memcpy(d, _buffer, _bufferSize); d[_bufferSize-1] = 0; } else { v.setSize(_bufferSize+1); char *d = (char*) v.data(); memcpy(d, _buffer, _bufferSize); d[_bufferSize] = 0; } chunkIt->writeItem(v); chunkIt->flush(); return _chunk; }
bool RemoteArray::proceedChunkMsg(AttributeID attId, MemChunk& chunk) { boost::shared_ptr<MessageDesc> chunkDesc = _messages[attId]; _messages[attId].reset(); StatisticsScope sScope(_statistics); boost::shared_ptr<scidb_msg::Chunk> chunkMsg = chunkDesc->getRecord<scidb_msg::Chunk>(); currentStatistics->receivedSize += chunkDesc->getMessageSize(); currentStatistics->receivedMessages++; if (!chunkMsg->eof()) { LOG4CXX_TRACE(logger, "RemoteArray received next chunk message"); assert(chunkDesc->getBinary()); const int compMethod = chunkMsg->compression_method(); const size_t decompressedSize = chunkMsg->decompressed_size(); Address firstElem; firstElem.attId = attId; firstElem.arrId = getArrayDesc().getId(); for (int i = 0; i < chunkMsg->coordinates_size(); i++) { firstElem.coords.push_back(chunkMsg->coordinates(i)); } chunk.initialize(this, &desc, firstElem, compMethod); chunk.setSparse(chunkMsg->sparse()); chunk.setCount(chunkMsg->count()); boost::shared_ptr<CompressedBuffer> compressedBuffer = dynamic_pointer_cast<CompressedBuffer>(chunkDesc->getBinary()); compressedBuffer->setCompressionMethod(compMethod); compressedBuffer->setDecompressedSize(decompressedSize); chunk.decompress(*compressedBuffer); LOG4CXX_TRACE(logger, "RemoteArray initializes next chunk"); requestNextChunk(attId); return true; } else { return false; } }
void MessageHandleJob::_handleChunkOrAggregateChunk(bool isAggregateChunk) { boost::shared_ptr<scidb_msg::Chunk> chunkRecord = _messageDesc->getRecord<scidb_msg::Chunk>(); assert(!chunkRecord->eof()); assert(_query); RWLock::ErrorChecker noopEc; ScopedRWLockRead shared(_query->queryLock, noopEc); try { if (! isAggregateChunk) { // TODO: Apply it to statistics of current SG currentStatistics->receivedSize += _messageDesc->getMessageSize(); currentStatistics->receivedMessages++; } LOG4CXX_TRACE(logger, "Next chunk message was received") const int compMethod = chunkRecord->compression_method(); const size_t decompressedSize = chunkRecord->decompressed_size(); const AttributeID attributeID = chunkRecord->attribute_id(); const size_t count = chunkRecord->count(); boost::shared_ptr<SGContext> sgCtx = dynamic_pointer_cast<SGContext>(_query->getOperatorContext()); if (sgCtx == NULL) { throw (SYSTEM_EXCEPTION(SCIDB_SE_INTERNAL, SCIDB_LE_UNKNOWN_CTX) << typeid(*_query->getOperatorContext()).name()); } boost::shared_ptr<Array> outputArray = sgCtx->_resultSG; ScopedMutexLock cs(_query->resultCS); boost::shared_ptr<ArrayIterator> outputIter = outputArray->getIterator(attributeID); const bool shouldCacheEmptyBitmap = sgCtx->_shouldCacheEmptyBitmap; const ArrayDesc& desc = outputArray->getArrayDesc(); const size_t sourceId = _query->mapPhysicalToLogical(_messageDesc->getSourceInstanceID()); const bool isEmptyable = (desc.getEmptyBitmapAttribute() != NULL); const bool isEmptyIndicator = isEmptyable && (attributeID+1==desc.getAttributes().size()); const bool rle = chunkRecord->rle(); if (isAggregateChunk) { assert(! isEmptyIndicator); } Coordinates coordinates; for (int i = 0; i < chunkRecord->coordinates_size(); i++) { coordinates.push_back(chunkRecord->coordinates(i)); } if (!isAggregateChunk && sgCtx->_targetVersioned) { sgCtx->_newChunks.insert(coordinates); } boost::shared_ptr<CompressedBuffer> compressedBuffer = dynamic_pointer_cast<CompressedBuffer>(_messageDesc->getBinary()); if (compressedBuffer) { assert(compressedBuffer->getData()); PinBuffer pin(*compressedBuffer); // this line protects compressedBuffer->data from being freed, allowing MemChunk::decompressed(*compressedBuffer) to be called multiple times. compressedBuffer->setCompressionMethod(compMethod); compressedBuffer->setDecompressedSize(decompressedSize); Chunk* outChunk; // temporary MemArray objects shared_ptr<MemChunk> pTmpChunk = make_shared<MemChunk>(); // make it a shared pointer, because the bitmap chunk needs to be preserved across threads. MemChunk closure; // the PinBuffer objects protect tmpChunk and closure, respectively. shared_ptr<PinBuffer> pinTmpChunk, pinClosure; if (outputIter->setPosition(coordinates)) { // existing chunk outChunk = &outputIter->updateChunk(); if (! isAggregateChunk) { if (outChunk->getDiskChunk() != NULL) { throw SYSTEM_EXCEPTION(SCIDB_SE_MERGE, SCIDB_LE_CANT_UPDATE_CHUNK); } outChunk->setCount(0); // unknown } // if (a) either dest is NULL or merge by bitwise-or is possible; and (b) src is not compressed char* dst = static_cast<char*>(outChunk->getData()); if ( (dst == NULL || (outChunk->isPossibleToMergeByBitwiseOr() && !chunkRecord->sparse() && !chunkRecord->rle())) && compMethod == 0 ) { char const* src = (char const*)compressedBuffer->getData(); // Special care is needed if shouldCacheEmptyBitmap. // - If this is the empty bitmap, store it in the SGContext. // - Otherwise, add the empty bitmap from the SGContext to the chunk's data. if (shouldCacheEmptyBitmap) { initMemChunkFromNetwork(pTmpChunk, pinTmpChunk, outputArray, coordinates, attributeID, compMethod, chunkRecord->sparse() || outChunk->isSparse(), rle, compressedBuffer); if (isEmptyIndicator) { sgCtx->setCachedEmptyBitmapChunk(sourceId, pTmpChunk, coordinates); } else { shared_ptr<ConstRLEEmptyBitmap> cachedBitmap = sgCtx->getCachedEmptyBitmap(sourceId, coordinates); assert(cachedBitmap); pinClosure = make_shared<PinBuffer>(closure); closure.initialize(*pTmpChunk); pTmpChunk->makeClosure(closure, cachedBitmap); src = static_cast<char const*>(closure.getData()); } } if (dst == NULL) { outChunk->allocateAndCopy(src, decompressedSize, chunkRecord->sparse(), chunkRecord->rle(), count, _query); } else { outChunk->mergeByBitwiseOr(src, decompressedSize, _query); } } else { initMemChunkFromNetwork(pTmpChunk, pinTmpChunk, outputArray, coordinates, attributeID, compMethod, chunkRecord->sparse() || outChunk->isSparse(), rle, compressedBuffer); ConstChunk const* srcChunk = &(*pTmpChunk); // Special care is needed if shouldCacheEmptyBitmap. // - If this is the empty bitmap, store it in the SGContext. // - Otherwise, add the empty bitmap from the SGContext to the chunk's data. if (shouldCacheEmptyBitmap) { if (isEmptyIndicator) { sgCtx->setCachedEmptyBitmapChunk(sourceId, pTmpChunk, coordinates); } else { shared_ptr<ConstRLEEmptyBitmap> cachedBitmap = sgCtx->getCachedEmptyBitmap(sourceId, coordinates); assert(cachedBitmap); pinClosure = make_shared<PinBuffer>(closure); closure.initialize(*pTmpChunk); pTmpChunk->makeClosure(closure, cachedBitmap); srcChunk = &closure; } } if (isAggregateChunk) { AggregatePtr aggregate = sgCtx->_aggregateList[attributeID]; if (!isEmptyable && rle) { assert(!shouldCacheEmptyBitmap); assert(srcChunk==&(*pTmpChunk)); outChunk->nonEmptyableAggregateMerge(*srcChunk, aggregate, _query); } else { outChunk->aggregateMerge(*srcChunk, aggregate, _query); } } else { outChunk->merge(*srcChunk, _query); } } } else { // new chunk outChunk = &outputIter->newChunk(coordinates, compMethod); outChunk->setSparse(chunkRecord->sparse()); outChunk->setRLE(rle); shared_ptr<CompressedBuffer> myCompressedBuffer = compressedBuffer; // Special care is needed if shouldCacheEmptyBitmap. // - If this is the empty bitmap, store it in the SGContext. // - Otherwise, add the empty bitmap from the SGContext to the chunk's data. if (shouldCacheEmptyBitmap) { initMemChunkFromNetwork(pTmpChunk, pinTmpChunk, outputArray, coordinates, attributeID, compMethod, chunkRecord->sparse() || outChunk->isSparse(), rle, compressedBuffer); if (isEmptyIndicator) { sgCtx->setCachedEmptyBitmapChunk(sourceId, pTmpChunk, coordinates); } else { shared_ptr<ConstRLEEmptyBitmap> cachedBitmap = sgCtx->getCachedEmptyBitmap(sourceId, coordinates); assert(cachedBitmap); myCompressedBuffer = make_shared<CompressedBuffer>(); pTmpChunk->compress(*myCompressedBuffer, cachedBitmap); } } outChunk->decompress(*myCompressedBuffer); outChunk->setCount(count); outChunk->write(_query); } // end if (outputIter->setPosition(coordinates)) assert(checkChunkMagic(*outChunk)); } // end if (compressedBuffer) sgSync(); LOG4CXX_TRACE(logger, "Chunk was stored") } catch(const Exception& e) { sgSync(); throw; } }