CPLErr GDALHashSetBandBlockCache::FlushBlock( int nXBlockOff, int nYBlockOff,
int bWriteDirtyBlock )
{
GDALRasterBlock oBlockForLookup(nXBlockOff, nYBlockOff);
GDALRasterBlock* poBlock = nullptr;
{
CPLLockHolderOptionalLockD( hLock );
auto oIter = m_oSet.find(&oBlockForLookup);
if( oIter == m_oSet.end() )
return CE_None;
poBlock = *oIter;
m_oSet.erase(oIter);
}
if( !poBlock->DropLockForRemovalFromStorage() )
return CE_None;
CPLErr eErr = CE_None;
if( bWriteDirtyBlock && poBlock->GetDirty() )
eErr = poBlock->Write();
delete poBlock;
return eErr;
}
int GDALRasterBlock::FlushCacheBlock(int bDirtyBlocksOnly)
{
GDALRasterBlock *poTarget;
{
INITIALIZE_LOCK;
poTarget = poOldest;
while( poTarget != NULL && (poTarget->GetLockCount() > 0 ||
(bDirtyBlocksOnly && !poTarget->GetDirty())) )
poTarget = poTarget->poPrevious;
if( poTarget == NULL )
return FALSE;
poTarget->Detach_unlocked();
poTarget->GetBand()->UnreferenceBlock(poTarget->GetXOff(),poTarget->GetYOff());
}
if( poTarget->GetDirty() )
{
CPLErr eErr = poTarget->Write();
if( eErr != CE_None )
{
/* Save the error for later reporting */
poTarget->GetBand()->SetFlushBlockErr(eErr);
}
}
delete poTarget;
return TRUE;
}
CPLErr GDALMRFRasterBand::FillBlock(int xblk, int yblk, void *buffer) {
vector<GDALRasterBlock *> blocks;
for (int i = 0; i < poDS->nBands; i++) {
GDALRasterBand *b = poDS->GetRasterBand(i + 1);
if (b->GetOverviewCount() && 0 != m_l)
b = b->GetOverview(m_l - 1);
// Get the other band blocks, keep them around until later
if (b == this) {
FillBlock(buffer);
}
else {
GDALRasterBlock *poBlock = b->GetLockedBlockRef(xblk, yblk, 1);
if (poBlock == nullptr) // Didn't get this block
break;
FillBlock(poBlock->GetDataRef());
blocks.push_back(poBlock);
}
}
// Drop the locks for blocks we acquired
for (int i = 0; i < int(blocks.size()); i++)
blocks[i]->DropLock();
return CE_None;
}
bool GDALWMSRasterBand::IsBlockInCache(int x, int y) {
bool ret = false;
GDALRasterBlock *b = TryGetLockedBlockRef(x, y);
if (b != NULL) {
ret = true;
b->DropLock();
}
return ret;
}
/***********************************************************************
* \brief Returns true if the (only) block is stored in the cache
**********************************************************************/
GBool PostGISRasterTileRasterBand::IsCached()
{
GDALRasterBlock * poBlock = TryGetLockedBlockRef(0, 0);
if (poBlock != NULL) {
poBlock->DropLock();
return true;
}
return false;
}
GDALRasterBlock *GDALArrayBandBlockCache::TryGetLockedBlockRef( int nXBlockOff,
int nYBlockOff )
{
GDALRasterBlock *poBlock;
int nBlockIndex = 0;
/* -------------------------------------------------------------------- */
/* Simple case for single level caches. */
/* -------------------------------------------------------------------- */
if( !bSubBlockingActive )
{
nBlockIndex = nXBlockOff + nYBlockOff * poBand->nBlocksPerRow;
while( TRUE )
{
poBlock = u.papoBlocks[nBlockIndex];
if( poBlock == NULL )
return NULL;
if( poBlock->TakeLock() )
break;
}
}
else
{
/* -------------------------------------------------------------------- */
/* Identify our subblock. */
/* -------------------------------------------------------------------- */
int nSubBlock = TO_SUBBLOCK(nXBlockOff)
+ TO_SUBBLOCK(nYBlockOff) * nSubBlocksPerRow;
/* -------------------------------------------------------------------- */
/* Check within subblock. */
/* -------------------------------------------------------------------- */
GDALRasterBlock **papoSubBlockGrid = u.papapoBlocks[nSubBlock];
if( papoSubBlockGrid == NULL )
return NULL;
int nBlockInSubBlock = WITHIN_SUBBLOCK(nXBlockOff)
+ WITHIN_SUBBLOCK(nYBlockOff) * SUBBLOCK_SIZE;
while( TRUE )
{
poBlock = papoSubBlockGrid[nBlockInSubBlock];
if( poBlock == NULL )
return NULL;
if( poBlock->TakeLock() )
break;
}
}
return poBlock;
}
void GDALRasterBlock::DumpAll()
{
int iBlock = 0;
for( GDALRasterBlock *poBlock = poNewest;
poBlock != NULL;
poBlock = poBlock->poNext )
{
printf("Block %d\n", iBlock);
poBlock->DumpBlock();
printf("\n");
iBlock++;
}
}
int RawRasterBand::IsLineLoaded( int nLineOff, int nLines )
{
int iLine;
for ( iLine = nLineOff; iLine < nLineOff + nLines; iLine++ )
{
GDALRasterBlock *poBlock = TryGetLockedBlockRef( 0, iLine );
if( poBlock != NULL )
{
poBlock->DropLock();
return TRUE;
}
}
return FALSE;
}
GDALRasterBlock *GDALHashSetBandBlockCache::TryGetLockedBlockRef(
int nXBlockOff, int nYBlockOff )
{
GDALRasterBlock oBlockForLookup(nXBlockOff, nYBlockOff);
GDALRasterBlock* poBlock;
{
CPLLockHolderOptionalLockD( hLock );
auto oIter = m_oSet.find(&oBlockForLookup);
if( oIter == m_oSet.end() )
return nullptr;
poBlock = *oIter;
}
if( !poBlock->TakeLock() )
return nullptr;
return poBlock;
}
CPLErr GDALMRFRasterBand::RB(int xblk, int yblk, buf_mgr /*src*/, void *buffer) {
vector<GDALRasterBlock *> blocks;
for (int i = 0; i < poDS->nBands; i++) {
GDALRasterBand *b = poDS->GetRasterBand(i+1);
if (b->GetOverviewCount() && m_l)
b = b->GetOverview(m_l-1);
void *ob = buffer;
// Get the other band blocks, keep them around until later
if (b != this)
{
GDALRasterBlock *poBlock = b->GetLockedBlockRef(xblk, yblk, 1);
if( poBlock == NULL )
break;
ob = poBlock->GetDataRef();
blocks.push_back(poBlock);
}
// Just the right mix of templates and macros make deinterleaving tidy
#define CpySI(T) cpy_stride_in<T> (ob, (T *)poDS->GetPBuffer() + i,\
blockSizeBytes()/sizeof(T), img.pagesize.c)
// Page is already in poDS->pbuffer, not empty
// There are only four cases, since only the real data type matters
switch (GDALGetDataTypeSize(eDataType)/8)
{
case 1: CpySI(GByte); break;
case 2: CpySI(GInt16); break;
case 4: CpySI(GInt32); break;
case 8: CpySI(GIntBig); break;
}
}
#undef CpySI
// Drop the locks we acquired
for (int i=0; i < int(blocks.size()); i++)
blocks[i]->DropLock();
return CE_None;
}
int GDALRasterBlock::FlushCacheBlock( int bDirtyBlocksOnly )
{
GDALRasterBlock *poTarget;
{
INITIALIZE_LOCK;
poTarget = poOldest;
while( poTarget != NULL )
{
if( !bDirtyBlocksOnly || poTarget->GetDirty() )
{
if( CPLAtomicCompareAndExchange(
&(poTarget->nLockCount), 0, -1) )
break;
}
poTarget = poTarget->poPrevious;
}
if( poTarget == NULL )
return FALSE;
if( bSleepsForBockCacheDebug )
CPLSleep(CPLAtof(
CPLGetConfigOption(
"GDAL_RB_FLUSHBLOCK_SLEEP_AFTER_DROP_LOCK", "0")));
poTarget->Detach_unlocked();
poTarget->GetBand()->UnreferenceBlock(poTarget);
}
if( bSleepsForBockCacheDebug )
CPLSleep(CPLAtof(
CPLGetConfigOption("GDAL_RB_FLUSHBLOCK_SLEEP_AFTER_RB_LOCK", "0")));
if( poTarget->GetDirty() )
{
const CPLErr eErr = poTarget->Write();
if( eErr != CE_None )
{
// Save the error for later reporting.
poTarget->GetBand()->SetFlushBlockErr(eErr);
}
}
VSIFree(poTarget->pData);
poTarget->pData = NULL;
poTarget->GetBand()->AddBlockToFreeList(poTarget);
return TRUE;
}
int RawRasterBand::IsSignificantNumberOfLinesLoaded( int nLineOff, int nLines )
{
int iLine;
int nCountLoaded = 0;
for ( iLine = nLineOff; iLine < nLineOff + nLines; iLine++ )
{
GDALRasterBlock *poBlock = TryGetLockedBlockRef( 0, iLine );
if( poBlock != NULL )
{
poBlock->DropLock();
nCountLoaded ++;
if( nCountLoaded > nLines / 20 )
{
return TRUE;
}
}
}
return FALSE;
}
int GDALRasterBlock::FlushCacheBlock()
{
int nXOff, nYOff;
GDALRasterBand *poBand;
{
CPLMutexHolderD( &hRBMutex );
GDALRasterBlock *poTarget = (GDALRasterBlock *) poOldest;
while( poTarget != NULL && poTarget->GetLockCount() > 0 )
poTarget = poTarget->poPrevious;
if( poTarget == NULL )
return FALSE;
poTarget->Detach();
nXOff = poTarget->GetXOff();
nYOff = poTarget->GetYOff();
poBand = poTarget->GetBand();
}
CPLErr eErr = poBand->FlushBlock( nXOff, nYOff );
if (eErr != CE_None)
{
/* Save the error for later reporting */
poBand->SetFlushBlockErr(eErr);
}
return TRUE;
}
GDALRasterBlock* GDALAbstractBandBlockCache::CreateBlock(int nXBlockOff,
int nYBlockOff)
{
GDALRasterBlock* poBlock;
{
CPLLockHolderOptionalLockD(hSpinLock);
poBlock = psListBlocksToFree;
if( poBlock )
{
#ifdef DEBUG_VERBOSE_ABBC
CPLAtomicDec(&nAllBandsKeptAlivedBlocks);
fprintf(stderr, "CreateBlock(): nAllBandsKeptAlivedBlocks=%d\n", nAllBandsKeptAlivedBlocks);
#endif
psListBlocksToFree = poBlock->poNext;
}
}
if( poBlock )
poBlock->RecycleFor(nXBlockOff, nYBlockOff);
else
poBlock = new (std::nothrow) GDALRasterBlock(
poBand, nXBlockOff, nYBlockOff );
return poBlock;
}
CPLErr GDALWMSRasterBand::ZeroBlock(int x, int y, int to_buffer_band, void *buffer) {
CPLErr ret = CE_None;
for (int ib = 1; ib <= m_parent_dataset->nBands; ++ib) {
if (ret == CE_None) {
void *p = NULL;
GDALRasterBlock *b = NULL;
if ((buffer != NULL) && (ib == to_buffer_band)) {
p = buffer;
} else {
GDALWMSRasterBand *band = static_cast<GDALWMSRasterBand *>(m_parent_dataset->GetRasterBand(ib));
if (m_overview >= 0) band = static_cast<GDALWMSRasterBand *>(band->GetOverview(m_overview));
if (!band->IsBlockInCache(x, y)) {
b = band->GetLockedBlockRef(x, y, true);
if (b != NULL) {
p = b->GetDataRef();
if (p == NULL) {
CPLError(CE_Failure, CPLE_AppDefined, "GDALWMS: GetDataRef returned NULL.");
ret = CE_Failure;
}
}
}
}
if (p != NULL) {
unsigned char *b = reinterpret_cast<unsigned char *>(p);
int block_size = nBlockXSize * nBlockYSize * (GDALGetDataTypeSize(eDataType) / 8);
for (int i = 0; i < block_size; ++i) b[i] = 0;
}
if (b != NULL) {
b->DropLock();
}
}
}
return ret;
}
void GDALRasterBlock::CheckNonOrphanedBlocks( GDALRasterBand* poBand )
{
TAKE_LOCK;
for( GDALRasterBlock *poBlock = poNewest;
poBlock != NULL;
poBlock = poBlock->poNext )
{
if ( poBlock->GetBand() == poBand )
{
printf("Cache has still blocks of band %p\n", poBand);
printf("Band : %d\n", poBand->GetBand());
printf("nRasterXSize = %d\n", poBand->GetXSize());
printf("nRasterYSize = %d\n", poBand->GetYSize());
int nBlockXSize, nBlockYSize;
poBand->GetBlockSize(&nBlockXSize, &nBlockYSize);
printf("nBlockXSize = %d\n", nBlockXSize);
printf("nBlockYSize = %d\n", nBlockYSize);
printf("Dataset : %p\n", poBand->GetDataset());
if( poBand->GetDataset() )
printf("Dataset : %s\n",
poBand->GetDataset()->GetDescription());
}
}
}
CPLErr VRTWarpedRasterBand::IReadBlock( int nBlockXOff, int nBlockYOff,
void * pImage )
{
CPLErr eErr;
VRTWarpedDataset *poWDS = (VRTWarpedDataset *) poDS;
GDALRasterBlock *poBlock;
poBlock = GetLockedBlockRef( nBlockXOff, nBlockYOff, TRUE );
eErr = poWDS->ProcessBlock( nBlockXOff, nBlockYOff );
if( eErr == CE_None && pImage != poBlock->GetDataRef() )
{
int nDataBytes;
nDataBytes = (GDALGetDataTypeSize(poBlock->GetDataType()) / 8)
* poBlock->GetXSize() * poBlock->GetYSize();
memcpy( pImage, poBlock->GetDataRef(), nDataBytes );
}
poBlock->DropLock();
return eErr;
}
CPLErr EpsilonRasterBand::IReadBlock( int nBlockXOff,
int nBlockYOff, void * pImage)
{
EpsilonDataset* poGDS = (EpsilonDataset*) poDS;
//CPLDebug("EPSILON", "IReadBlock(nBand=%d,nBlockXOff=%d,nBlockYOff=%d)",
// nBand, nBlockXOff, nBlockYOff);
int l_nBlocksPerRow = (poGDS->nRasterXSize + nBlockXSize - 1) / nBlockXSize;
int nBlock = nBlockXOff + nBlockYOff * l_nBlocksPerRow;
BlockDesc* psDesc = &poGDS->pasBlocks[nBlock];
#ifdef DEBUG
int l_nBlocksPerColumn = (poGDS->nRasterYSize + nBlockYSize - 1) / nBlockYSize;
CPLAssert(psDesc->x == nBlockXOff * nBlockXSize);
CPLAssert(psDesc->y == nBlockYOff * nBlockYSize);
CPLAssert(psDesc->w == (nBlockXOff < l_nBlocksPerRow - 1) ?
nBlockXSize : poGDS->nRasterXSize - psDesc->x);
CPLAssert(psDesc->h == (nBlockYOff < l_nBlocksPerColumn - 1) ?
nBlockYSize : poGDS->nRasterYSize - psDesc->y);
#endif
poGDS->Seek(psDesc->offset);
if (!poGDS->GetNextBlockData())
{
memset(pImage, 0, nBlockXSize * nBlockYSize);
return CE_Failure;
}
eps_block_header hdr;
if (eps_read_block_header (poGDS->pabyBlockData,
poGDS->nBlockDataSize, &hdr) != EPS_OK)
{
CPLError(CE_Warning, CPLE_AppDefined, "cannot read block header");
memset(pImage, 0, nBlockXSize * nBlockYSize);
return CE_Failure;
}
if (hdr.chk_flag == EPS_BAD_CRC ||
hdr.crc_flag == EPS_BAD_CRC)
{
CPLError(CE_Warning, CPLE_AppDefined, "bad CRC");
memset(pImage, 0, nBlockXSize * nBlockYSize);
return CE_Failure;
}
int w = GET_FIELD(hdr, w);
int h = GET_FIELD(hdr, h);
int i;
if (poGDS->nBands == 1)
{
unsigned char ** pTempData =
(unsigned char **) CPLMalloc(h * sizeof(unsigned char*));
for(i=0;i<h;i++)
pTempData[i] = ((GByte*)pImage) + i * nBlockXSize;
if (w != nBlockXSize || h != nBlockYSize)
memset(pImage, 0, nBlockXSize * nBlockYSize);
if (eps_decode_grayscale_block (pTempData,
poGDS->pabyBlockData, &hdr) != EPS_OK)
{
CPLFree(pTempData);
memset(pImage, 0, nBlockXSize * nBlockYSize);
return CE_Failure;
}
CPLFree(pTempData);
}
else
{
if (poGDS->pabyRGBData == NULL)
{
poGDS->pabyRGBData =
(GByte*) VSIMalloc3(nBlockXSize, nBlockYSize, 3);
if (poGDS->pabyRGBData == NULL)
{
memset(pImage, 0, nBlockXSize * nBlockYSize);
return CE_Failure;
}
}
if (poGDS->nBufferedBlock == nBlock)
{
memcpy(pImage,
poGDS->pabyRGBData + (nBand - 1) * nBlockXSize * nBlockYSize,
nBlockXSize * nBlockYSize);
return CE_None;
}
unsigned char ** pTempData[3];
int iBand;
for(iBand=0;iBand<3;iBand++)
{
pTempData[iBand] =
(unsigned char **) CPLMalloc(h * sizeof(unsigned char*));
for(i=0;i<h;i++)
pTempData[iBand][i] = poGDS->pabyRGBData +
iBand * nBlockXSize * nBlockYSize + i * nBlockXSize;
}
if (w != nBlockXSize || h != nBlockYSize)
memset(poGDS->pabyRGBData, 0, 3 * nBlockXSize * nBlockYSize);
if (eps_decode_truecolor_block (pTempData[0], pTempData[1], pTempData[2],
poGDS->pabyBlockData, &hdr) != EPS_OK)
{
for(iBand=0;iBand<3;iBand++)
CPLFree(pTempData[iBand]);
memset(pImage, 0, nBlockXSize * nBlockYSize);
return CE_Failure;
}
for(iBand=0;iBand<3;iBand++)
CPLFree(pTempData[iBand]);
poGDS->nBufferedBlock = nBlock;
memcpy(pImage,
poGDS->pabyRGBData + (nBand - 1) * nBlockXSize * nBlockYSize,
nBlockXSize * nBlockYSize);
if (nBand == 1)
{
int iOtherBand;
for(iOtherBand=2;iOtherBand<=3;iOtherBand++)
{
GDALRasterBlock *poBlock = poGDS->GetRasterBand(iOtherBand)->
GetLockedBlockRef(nBlockXOff,nBlockYOff, TRUE);
if (poBlock == NULL)
break;
GByte* pabySrcBlock = (GByte *) poBlock->GetDataRef();
if( pabySrcBlock == NULL )
{
poBlock->DropLock();
break;
}
memcpy(pabySrcBlock,
poGDS->pabyRGBData + (iOtherBand - 1) * nBlockXSize * nBlockYSize,
nBlockXSize * nBlockYSize);
poBlock->DropLock();
}
}
}
return CE_None;
}
CPLErr GDALArrayBandBlockCache::FlushBlock( int nXBlockOff, int nYBlockOff,
int bWriteDirtyBlock )
{
int nBlockIndex;
GDALRasterBlock *poBlock = NULL;
/* -------------------------------------------------------------------- */
/* Simple case for single level caches. */
/* -------------------------------------------------------------------- */
if( !bSubBlockingActive )
{
nBlockIndex = nXBlockOff + nYBlockOff * poBand->nBlocksPerRow;
poBlock = u.papoBlocks[nBlockIndex];
u.papoBlocks[nBlockIndex] = NULL;
}
/* -------------------------------------------------------------------- */
/* Identify our subblock. */
/* -------------------------------------------------------------------- */
else
{
int nSubBlock = TO_SUBBLOCK(nXBlockOff)
+ TO_SUBBLOCK(nYBlockOff) * nSubBlocksPerRow;
/* -------------------------------------------------------------------- */
/* Check within subblock. */
/* -------------------------------------------------------------------- */
GDALRasterBlock **papoSubBlockGrid = u.papapoBlocks[nSubBlock];
if( papoSubBlockGrid == NULL )
return CE_None;
int nBlockInSubBlock = WITHIN_SUBBLOCK(nXBlockOff)
+ WITHIN_SUBBLOCK(nYBlockOff) * SUBBLOCK_SIZE;
poBlock = papoSubBlockGrid[nBlockInSubBlock];
papoSubBlockGrid[nBlockInSubBlock] = NULL;
}
if( poBlock == NULL )
return CE_None;
if( !poBlock->DropLockForRemovalFromStorage() )
return CE_None;
/* -------------------------------------------------------------------- */
/* Is the target block dirty? If so we need to write it. */
/* -------------------------------------------------------------------- */
CPLErr eErr = CE_None;
poBlock->Detach();
if( bWriteDirtyBlock && poBlock->GetDirty() )
eErr = poBlock->Write();
/* -------------------------------------------------------------------- */
/* Deallocate the block; */
/* -------------------------------------------------------------------- */
delete poBlock;
return eErr;
}
CPLErr VRTWarpedDataset::ProcessBlock( int iBlockX, int iBlockY )
{
if( poWarper == NULL )
return CE_Failure;
const GDALWarpOptions *psWO = poWarper->GetOptions();
/* -------------------------------------------------------------------- */
/* Allocate block of memory large enough to hold all the bands */
/* for this block. */
/* -------------------------------------------------------------------- */
int iBand;
GByte *pabyDstBuffer;
int nDstBufferSize;
int nWordSize = (GDALGetDataTypeSize(psWO->eWorkingDataType) / 8);
// FIXME? : risk of overflow in multiplication if nBlockXSize or nBlockYSize are very large
nDstBufferSize = nBlockXSize * nBlockYSize * psWO->nBandCount * nWordSize;
pabyDstBuffer = (GByte *) VSIMalloc(nDstBufferSize);
if( pabyDstBuffer == NULL )
{
CPLError( CE_Failure, CPLE_OutOfMemory,
"Out of memory allocating %d byte buffer in VRTWarpedDataset::ProcessBlock()",
nDstBufferSize );
return CE_Failure;
}
memset( pabyDstBuffer, 0, nDstBufferSize );
/* -------------------------------------------------------------------- */
/* Process INIT_DEST option to initialize the buffer prior to */
/* warping into it. */
/* NOTE:The following code is 99% similar in gdalwarpoperation.cpp and */
/* vrtwarped.cpp. Be careful to keep it in sync ! */
/* -------------------------------------------------------------------- */
const char *pszInitDest = CSLFetchNameValue( psWO->papszWarpOptions,
"INIT_DEST" );
if( pszInitDest != NULL && !EQUAL(pszInitDest, "") )
{
char **papszInitValues =
CSLTokenizeStringComplex( pszInitDest, ",", FALSE, FALSE );
int nInitCount = CSLCount(papszInitValues);
for( iBand = 0; iBand < psWO->nBandCount; iBand++ )
{
double adfInitRealImag[2];
GByte *pBandData;
int nBandSize = nBlockXSize * nBlockYSize * nWordSize;
const char *pszBandInit = papszInitValues[MIN(iBand,nInitCount-1)];
if( EQUAL(pszBandInit,"NO_DATA")
&& psWO->padfDstNoDataReal != NULL )
{
adfInitRealImag[0] = psWO->padfDstNoDataReal[iBand];
adfInitRealImag[1] = psWO->padfDstNoDataImag[iBand];
}
else
{
CPLStringToComplex( pszBandInit,
adfInitRealImag + 0, adfInitRealImag + 1);
}
pBandData = ((GByte *) pabyDstBuffer) + iBand * nBandSize;
if( psWO->eWorkingDataType == GDT_Byte )
memset( pBandData,
MAX(0,MIN(255,(int)adfInitRealImag[0])),
nBandSize);
else if( adfInitRealImag[0] == 0.0 && adfInitRealImag[1] == 0 )
{
memset( pBandData, 0, nBandSize );
}
else if( adfInitRealImag[1] == 0.0 )
{
GDALCopyWords( &adfInitRealImag, GDT_Float64, 0,
pBandData,psWO->eWorkingDataType,nWordSize,
nBlockXSize * nBlockYSize );
}
else
{
GDALCopyWords( &adfInitRealImag, GDT_CFloat64, 0,
pBandData,psWO->eWorkingDataType,nWordSize,
nBlockXSize * nBlockYSize );
}
}
CSLDestroy( papszInitValues );
}
/* -------------------------------------------------------------------- */
/* Warp into this buffer. */
/* -------------------------------------------------------------------- */
CPLErr eErr;
eErr =
poWarper->WarpRegionToBuffer(
iBlockX * nBlockXSize, iBlockY * nBlockYSize,
nBlockXSize, nBlockYSize,
pabyDstBuffer, psWO->eWorkingDataType );
if( eErr != CE_None )
{
VSIFree( pabyDstBuffer );
return eErr;
}
/* -------------------------------------------------------------------- */
/* Copy out into cache blocks for each band. */
/* -------------------------------------------------------------------- */
for( iBand = 0; iBand < psWO->nBandCount; iBand++ )
{
GDALRasterBand *poBand;
GDALRasterBlock *poBlock;
poBand = GetRasterBand(iBand+1);
poBlock = poBand->GetLockedBlockRef( iBlockX, iBlockY, TRUE );
CPLAssert( poBlock != NULL && poBlock->GetDataRef() != NULL );
GDALCopyWords( pabyDstBuffer + iBand*nBlockXSize*nBlockYSize*nWordSize,
psWO->eWorkingDataType, nWordSize,
poBlock->GetDataRef(),
poBlock->GetDataType(),
GDALGetDataTypeSize(poBlock->GetDataType())/8,
nBlockXSize * nBlockYSize );
poBlock->DropLock();
}
VSIFree( pabyDstBuffer );
return CE_None;
}
CPLErr GDALMRFRasterBand::IWriteBlock(int xblk, int yblk, void *buffer)
{
GInt32 cstride = img.pagesize.c;
ILSize req(xblk, yblk, 0, (nBand-1)/cstride, m_l);
GUIntBig infooffset = IdxOffset(req, img);
CPLDebug("MRF_IB", "IWriteBlock %d,%d,0,%d, level %d, stride %d\n", xblk, yblk,
nBand, m_l, cstride);
// Finish the Create call
if (!poDS->bCrystalized)
poDS->Crystalize();
if (1 == cstride) { // Separate bands, we can write it as is
// Empty page skip
int success;
double val = GetNoDataValue(&success);
if (!success) val = 0.0;
if (isAllVal(eDataType, buffer, img.pageSizeBytes, val))
return poDS->WriteTile(nullptr, infooffset, 0);
// Use the pbuffer to hold the compressed page before writing it
poDS->tile = ILSize(); // Mark it corrupt
buf_mgr src;
src.buffer = (char *)buffer;
src.size = static_cast<size_t>(img.pageSizeBytes);
buf_mgr dst = {(char *)poDS->GetPBuffer(), poDS->GetPBufferSize()};
// Swab the source before encoding if we need to
if (is_Endianess_Dependent(img.dt, img.comp) && (img.nbo != NET_ORDER))
swab_buff(src, img);
// Compress functions need to return the compressed size in
// the bytes in buffer field
Compress(dst, src);
void *usebuff = dst.buffer;
if (deflatep) {
usebuff = DeflateBlock(dst, poDS->pbsize - dst.size, deflate_flags);
if (!usebuff) {
CPLError(CE_Failure,CPLE_AppDefined, "MRF: Deflate error");
return CE_Failure;
}
}
return poDS->WriteTile(usebuff, infooffset , dst.size);
}
// Multiple bands per page, use a temporary to assemble the page
// Temporary is large because we use it to hold both the uncompressed and the compressed
poDS->tile=req; poDS->bdirty=0;
// Keep track of what bands are empty
GUIntBig empties=0;
void *tbuffer = VSIMalloc(img.pageSizeBytes + poDS->pbsize);
if (!tbuffer) {
CPLError(CE_Failure,CPLE_AppDefined, "MRF: Can't allocate write buffer");
return CE_Failure;
}
// Get the other bands from the block cache
for (int iBand=0; iBand < poDS->nBands; iBand++ )
{
char *pabyThisImage = nullptr;
GDALRasterBlock *poBlock = nullptr;
if (iBand == nBand-1)
{
pabyThisImage = reinterpret_cast<char *>(buffer);
poDS->bdirty |= bandbit();
} else {
GDALRasterBand *band = poDS->GetRasterBand(iBand +1);
// Pick the right overview
if (m_l) band = band->GetOverview(m_l -1);
poBlock = (reinterpret_cast<GDALMRFRasterBand *>(band))->TryGetLockedBlockRef(xblk, yblk);
if (nullptr==poBlock) continue;
// This is where the image data is for this band
pabyThisImage = reinterpret_cast<char*>(poBlock->GetDataRef());
poDS->bdirty |= bandbit(iBand);
}
// Keep track of empty bands, but encode them anyhow, in case some are not empty
int success;
double val = GetNoDataValue(&success);
if (!success) val = 0.0;
if (isAllVal(eDataType, pabyThisImage, blockSizeBytes(), val))
empties |= bandbit(iBand);
// Copy the data into the dataset buffer here
// Just the right mix of templates and macros make this real tidy
#define CpySO(T) cpy_stride_out<T> ((reinterpret_cast<T *>(tbuffer))+iBand, pabyThisImage,\
blockSizeBytes()/sizeof(T), cstride)
// Build the page in tbuffer
switch (GDALGetDataTypeSize(eDataType)/8)
{
case 1: CpySO(GByte); break;
case 2: CpySO(GInt16); break;
case 4: CpySO(GInt32); break;
case 8: CpySO(GIntBig); break;
default:
{
CPLError(CE_Failure,CPLE_AppDefined, "MRF: Write datatype of %d bytes "
"not implemented", GDALGetDataTypeSize(eDataType)/8);
if (poBlock != nullptr)
{
poBlock->MarkClean();
poBlock->DropLock();
}
CPLFree(tbuffer);
return CE_Failure;
}
}
if (poBlock != nullptr)
{
poBlock->MarkClean();
poBlock->DropLock();
}
}
// Should keep track of the individual band buffers and only mix them if this is not
// an empty page ( move the Copy with Stride Out from above below this test
// This way works fine, but it does work extra for empty pages
if (GIntBig(empties) == AllBandMask()) {
CPLFree(tbuffer);
return poDS->WriteTile(nullptr, infooffset, 0);
}
if (poDS->bdirty != AllBandMask())
CPLError(CE_Warning, CPLE_AppDefined,
"MRF: IWrite, band dirty mask is " CPL_FRMT_GIB " instead of " CPL_FRMT_GIB,
poDS->bdirty, AllBandMask());
buf_mgr src;
src.buffer = (char *)tbuffer;
src.size = static_cast<size_t>(img.pageSizeBytes);
// Use the space after pagesizebytes for compressed output, it is of pbsize
char *outbuff = (char *)tbuffer + img.pageSizeBytes;
buf_mgr dst = {outbuff, poDS->pbsize};
CPLErr ret;
ret = Compress(dst, src);
if (ret != CE_None) {
// Compress failed, write it as an empty tile
CPLFree(tbuffer);
poDS->WriteTile(nullptr, infooffset, 0);
return CE_None; // Should report the error, but it triggers partial band attempts
}
// Where the output is, in case we deflate
void *usebuff = outbuff;
if (deflatep) {
// Move the packed part at the start of tbuffer, to make more space available
memcpy(tbuffer, outbuff, dst.size);
dst.buffer = (char *)tbuffer;
usebuff = DeflateBlock(dst, img.pageSizeBytes + poDS->pbsize - dst.size, deflate_flags);
if (!usebuff) {
CPLError(CE_Failure,CPLE_AppDefined, "MRF: Deflate error");
CPLFree(tbuffer);
poDS->WriteTile(nullptr, infooffset, 0);
poDS->bdirty = 0;
return CE_Failure;
}
}
ret = poDS->WriteTile(usebuff, infooffset, dst.size);
CPLFree(tbuffer);
poDS->bdirty = 0;
return ret;
}
CPLErr GDALRasterBlock::Internalize()
{
void *pNewData = NULL;
int nSizeInBytes;
CPLAssert( pData == NULL );
// This call will initialize the hRBLock mutex. Other call places can
// only be called if we have go through there.
GIntBig nCurCacheMax = GDALGetCacheMax64();
/* No risk of overflow as it is checked in GDALRasterBand::InitBlockInfo() */
nSizeInBytes = GetBlockSize();
/* -------------------------------------------------------------------- */
/* Flush old blocks if we are nearing our memory limit. */
/* -------------------------------------------------------------------- */
int bFirstIter = TRUE;
int bLoopAgain;
do
{
bLoopAgain = FALSE;
GDALRasterBlock* apoBlocksToFree[64];
int nBlocksToFree = 0;
{
TAKE_LOCK;
if( bFirstIter )
nCacheUsed += nSizeInBytes;
GDALRasterBlock *poTarget = poOldest;
while( nCacheUsed > nCurCacheMax )
{
while( poTarget != NULL && poTarget->GetLockCount() > 0 )
poTarget = poTarget->poPrevious;
if( poTarget != NULL )
{
GDALRasterBlock* _poPrevious = poTarget->poPrevious;
poTarget->Detach_unlocked();
poTarget->GetBand()->UnreferenceBlock(poTarget->GetXOff(),poTarget->GetYOff());
apoBlocksToFree[nBlocksToFree++] = poTarget;
if( poTarget->GetDirty() )
{
// Only free one dirty block at a time so that
// other dirty blocks of other bands with the same coordinates
// can be found with TryGetLockedBlock()
bLoopAgain = ( nCacheUsed > nCurCacheMax );
break;
}
if( nBlocksToFree == 64 )
{
CPLDebug("GDAL", "More than 64 blocks are flagged to be flushed. Not trying more");
break;
}
poTarget = _poPrevious;
}
else
break;
}
/* -------------------------------------------------------------------- */
/* Add this block to the list. */
/* -------------------------------------------------------------------- */
if( !bLoopAgain )
Touch_unlocked();
}
bFirstIter = FALSE;
/* Now free blocks we have detached and removed from their band */
for(int i=0; i<nBlocksToFree; i++)
{
GDALRasterBlock *poBlock = apoBlocksToFree[i];
if( poBlock->GetDirty() )
{
CPLErr eErr = poBlock->Write();
if( eErr != CE_None )
{
/* Save the error for later reporting */
poBlock->GetBand()->SetFlushBlockErr(eErr);
}
}
/* Try to recycle the data of an existing block */
void* pDataBlock = poBlock->pData;
if( pNewData == NULL && pDataBlock != NULL &&
poBlock->GetBlockSize() >= nSizeInBytes )
{
pNewData = pDataBlock;
poBlock->pData = NULL;
}
delete poBlock;
}
}
while(bLoopAgain);
if( pNewData == NULL )
{
pNewData = VSIMalloc( nSizeInBytes );
if( pNewData == NULL )
{
CPLError( CE_Failure, CPLE_OutOfMemory,
"GDALRasterBlock::Internalize : Out of memory allocating %d bytes.",
nSizeInBytes);
return( CE_Failure );
}
}
pData = pNewData;
return( CE_None );
}
CPLErr GDALRasterBlock::Internalize()
{
CPLAssert( pData == NULL );
void *pNewData = NULL;
// This call will initialize the hRBLock mutex. Other call places can
// only be called if we have go through there.
const GIntBig nCurCacheMax = GDALGetCacheMax64();
// No risk of overflow as it is checked in GDALRasterBand::InitBlockInfo().
const int nSizeInBytes = GetBlockSize();
/* -------------------------------------------------------------------- */
/* Flush old blocks if we are nearing our memory limit. */
/* -------------------------------------------------------------------- */
bool bFirstIter = true;
bool bLoopAgain = false;
do
{
bLoopAgain = false;
GDALRasterBlock* apoBlocksToFree[64] = { NULL };
int nBlocksToFree = 0;
{
TAKE_LOCK;
if( bFirstIter )
nCacheUsed += nSizeInBytes;
GDALRasterBlock *poTarget = poOldest;
while( nCacheUsed > nCurCacheMax )
{
while( poTarget != NULL )
{
if( CPLAtomicCompareAndExchange(
&(poTarget->nLockCount), 0, -1) )
break;
poTarget = poTarget->poPrevious;
}
if( poTarget != NULL )
{
if( bSleepsForBockCacheDebug )
CPLSleep(CPLAtof(
CPLGetConfigOption(
"GDAL_RB_INTERNALIZE_SLEEP_AFTER_DROP_LOCK",
"0")));
GDALRasterBlock* _poPrevious = poTarget->poPrevious;
poTarget->Detach_unlocked();
poTarget->GetBand()->UnreferenceBlock(poTarget);
apoBlocksToFree[nBlocksToFree++] = poTarget;
if( poTarget->GetDirty() )
{
// Only free one dirty block at a time so that
// other dirty blocks of other bands with the same
// coordinates can be found with TryGetLockedBlock()
bLoopAgain = nCacheUsed > nCurCacheMax;
break;
}
if( nBlocksToFree == 64 )
{
bLoopAgain = ( nCacheUsed > nCurCacheMax );
break;
}
poTarget = _poPrevious;
}
else
{
break;
}
}
/* ------------------------------------------------------------------ */
/* Add this block to the list. */
/* ------------------------------------------------------------------ */
if( !bLoopAgain )
Touch_unlocked();
}
bFirstIter = false;
// Now free blocks we have detached and removed from their band.
for( int i = 0; i < nBlocksToFree; ++i)
{
GDALRasterBlock * const poBlock = apoBlocksToFree[i];
if( poBlock->GetDirty() )
{
CPLErr eErr = poBlock->Write();
if( eErr != CE_None )
{
// Save the error for later reporting.
poBlock->GetBand()->SetFlushBlockErr(eErr);
}
}
// Try to recycle the data of an existing block.
void* pDataBlock = poBlock->pData;
if( pNewData == NULL && pDataBlock != NULL &&
poBlock->GetBlockSize() == nSizeInBytes )
{
pNewData = pDataBlock;
}
else
{
VSIFree(poBlock->pData);
}
poBlock->pData = NULL;
poBlock->GetBand()->AddBlockToFreeList(poBlock);
}
}
while(bLoopAgain);
if( pNewData == NULL )
{
pNewData = VSI_MALLOC_VERBOSE( nSizeInBytes );
if( pNewData == NULL )
{
return( CE_Failure );
}
}
pData = pNewData;
return CE_None;
}
CPLErr GDALWMSRasterBand::ReadBlockFromFile(int x, int y, const char *file_name, int to_buffer_band, void *buffer, int advise_read) {
CPLErr ret = CE_None;
GDALDataset *ds = 0;
GByte *color_table = NULL;
int i;
//CPLDebug("WMS", "ReadBlockFromFile: to_buffer_band=%d, (x,y)=(%d, %d)", to_buffer_band, x, y);
/* expected size */
const int esx = MIN(MAX(0, (x + 1) * nBlockXSize), nRasterXSize) - MIN(MAX(0, x * nBlockXSize), nRasterXSize);
const int esy = MIN(MAX(0, (y + 1) * nBlockYSize), nRasterYSize) - MIN(MAX(0, y * nBlockYSize), nRasterYSize);
ds = reinterpret_cast<GDALDataset*>(GDALOpen(file_name, GA_ReadOnly));
if (ds != NULL) {
int sx = ds->GetRasterXSize();
int sy = ds->GetRasterYSize();
bool accepted_as_no_alpha = false; // if the request is for 4 bands but the wms returns 3
/* Allow bigger than expected so pre-tiled constant size images work on corners */
if ((sx > nBlockXSize) || (sy > nBlockYSize) || (sx < esx) || (sy < esy)) {
CPLError(CE_Failure, CPLE_AppDefined, "GDALWMS: Incorrect size %d x %d of downloaded block, expected %d x %d, max %d x %d.",
sx, sy, esx, esy, nBlockXSize, nBlockYSize);
ret = CE_Failure;
}
if (ret == CE_None) {
int nDSRasterCount = ds->GetRasterCount();
if (nDSRasterCount != m_parent_dataset->nBands) {
/* Maybe its an image with color table */
bool accepted_as_ct = false;
if ((eDataType == GDT_Byte) && (ds->GetRasterCount() == 1)) {
GDALRasterBand *rb = ds->GetRasterBand(1);
if (rb->GetRasterDataType() == GDT_Byte) {
GDALColorTable *ct = rb->GetColorTable();
if (ct != NULL) {
accepted_as_ct = true;
if (!advise_read) {
color_table = new GByte[256 * 4];
const int count = MIN(256, ct->GetColorEntryCount());
for (i = 0; i < count; ++i) {
GDALColorEntry ce;
ct->GetColorEntryAsRGB(i, &ce);
color_table[i] = static_cast<GByte>(ce.c1);
color_table[i + 256] = static_cast<GByte>(ce.c2);
color_table[i + 512] = static_cast<GByte>(ce.c3);
color_table[i + 768] = static_cast<GByte>(ce.c4);
}
for (i = count; i < 256; ++i) {
color_table[i] = 0;
color_table[i + 256] = 0;
color_table[i + 512] = 0;
color_table[i + 768] = 0;
}
}
}
}
}
if (nDSRasterCount == 4 && m_parent_dataset->nBands == 3)
{
/* metacarta TMS service sometimes return a 4 band PNG instead of the expected 3 band... */
}
else if (!accepted_as_ct) {
if (ds->GetRasterCount()==3 && m_parent_dataset->nBands == 4 && (eDataType == GDT_Byte))
{ // WMS returned a file with no alpha so we will fill the alpha band with "opaque"
accepted_as_no_alpha = true;
}
else
{
CPLError(CE_Failure, CPLE_AppDefined, "GDALWMS: Incorrect bands count %d in downloaded block, expected %d.",
nDSRasterCount, m_parent_dataset->nBands);
ret = CE_Failure;
}
}
}
}
if (!advise_read) {
for (int ib = 1; ib <= m_parent_dataset->nBands; ++ib) {
if (ret == CE_None) {
void *p = NULL;
GDALRasterBlock *b = NULL;
if ((buffer != NULL) && (ib == to_buffer_band)) {
p = buffer;
} else {
GDALWMSRasterBand *band = static_cast<GDALWMSRasterBand *>(m_parent_dataset->GetRasterBand(ib));
if (m_overview >= 0) band = static_cast<GDALWMSRasterBand *>(band->GetOverview(m_overview));
if (!band->IsBlockInCache(x, y)) {
b = band->GetLockedBlockRef(x, y, true);
if (b != NULL) {
p = b->GetDataRef();
if (p == NULL) {
CPLError(CE_Failure, CPLE_AppDefined, "GDALWMS: GetDataRef returned NULL.");
ret = CE_Failure;
}
}
}
else
{
//CPLDebug("WMS", "Band %d, block (x,y)=(%d, %d) already in cache", band->GetBand(), x, y);
}
}
if (p != NULL) {
int pixel_space = GDALGetDataTypeSize(eDataType) / 8;
int line_space = pixel_space * nBlockXSize;
if (color_table == NULL) {
if( ib <= ds->GetRasterCount()) {
if (ds->RasterIO(GF_Read, 0, 0, sx, sy, p, sx, sy, eDataType, 1, &ib, pixel_space, line_space, 0) != CE_None) {
CPLError(CE_Failure, CPLE_AppDefined, "GDALWMS: RasterIO failed on downloaded block.");
ret = CE_Failure;
}
}
else
{ // parent expects 4 bands but file only has 3 so generate a all "opaque" 4th band
if (accepted_as_no_alpha)
{
// the file had 3 bands and we are reading band 4 (Alpha) so fill with 255 (no alpha)
GByte *byte_buffer = reinterpret_cast<GByte *>(p);
for (int y = 0; y < sy; ++y) {
for (int x = 0; x < sx; ++x) {
const int offset = x + y * line_space;
byte_buffer[offset] = 255; // fill with opaque
}
}
}
else
{ // we should never get here because this case was caught above
CPLError(CE_Failure, CPLE_AppDefined, "GDALWMS: Incorrect bands count %d in downloaded block, expected %d.",
ds->GetRasterCount(), m_parent_dataset->nBands);
ret = CE_Failure;
}
}
} else if (ib <= 4) {
if (ds->RasterIO(GF_Read, 0, 0, sx, sy, p, sx, sy, eDataType, 1, NULL, pixel_space, line_space, 0) != CE_None) {
CPLError(CE_Failure, CPLE_AppDefined, "GDALWMS: RasterIO failed on downloaded block.");
ret = CE_Failure;
}
if (ret == CE_None) {
GByte *band_color_table = color_table + 256 * (ib - 1);
GByte *byte_buffer = reinterpret_cast<GByte *>(p);
for (int y = 0; y < sy; ++y) {
for (int x = 0; x < sx; ++x) {
const int offset = x + y * line_space;
byte_buffer[offset] = band_color_table[byte_buffer[offset]];
}
}
}
} else {
CPLError(CE_Failure, CPLE_AppDefined, "GDALWMS: Color table supports at most 4 components.");
ret = CE_Failure;
}
}
if (b != NULL) {
b->DropLock();
}
}
}
}
GDALClose(ds);
} else {
CPLError(CE_Failure, CPLE_AppDefined, "GDALWMS: Unable to open downloaded block.");
ret = CE_Failure;
}
if (color_table != NULL) {
delete[] color_table;
}
return ret;
}
CPLErr JP2LuraRasterBand::IReadBlock(int nBlockXOff, int nBlockYOff,
void * pImage)
{
JP2LuraDataset *poGDS = reinterpret_cast<JP2LuraDataset *>(poDS);
#ifdef DEBUG_VERBOSE
CPLDebug("JP2Lura", "IReadBlock(nBand=%d,nLevel=%d %d,%d)",
nBand, poGDS->iLevel, nBlockXOff, nBlockYOff);
#endif
int nXOff = nBlockXOff * nBlockXSize;
int nYOff = nBlockYOff * nBlockYSize;
int nXSize = nBlockXSize;
int nYSize = nBlockYSize;
if( nXOff + nXSize > nRasterXSize )
nXSize = nRasterXSize - nXOff;
if( nYOff + nYSize > nRasterYSize )
nYSize = nRasterYSize - nYOff;
GDALRasterIOExtraArg sExtraArgs;
INIT_RASTERIO_EXTRA_ARG(sExtraArgs);
const int nDTSizeBytes = GDALGetDataTypeSizeBytes(eDataType);
CPLErr eErr = IRasterIO(GF_Read, nXOff, nYOff, nXSize, nYSize,
pImage, nXSize, nYSize,
eDataType,
nDTSizeBytes,
nDTSizeBytes * nXSize,
&sExtraArgs);
// Unpack previously packed buffer if needed
if( eErr == CE_None && nXSize < nBlockXSize )
{
GByte* pabyData = reinterpret_cast<GByte*>(pImage);
for( int j = nYSize - 1; j >= 0; --j )
{
memmove( pabyData + j * nBlockXSize * nDTSizeBytes,
pabyData + j * nXSize * nDTSizeBytes,
nXSize * nDTSizeBytes );
}
}
// Caching other bands while we have the cached buffer valid
for( int iBand = 1;eErr == CE_None && iBand <= poGDS->nBands; iBand++ )
{
if( iBand == nBand )
continue;
JP2LuraRasterBand* poOtherBand =
reinterpret_cast<JP2LuraRasterBand*>(poGDS->GetRasterBand(iBand));
GDALRasterBlock* poBlock = poOtherBand->
TryGetLockedBlockRef(nBlockXOff,nBlockYOff);
if (poBlock != nullptr)
{
poBlock->DropLock();
continue;
}
poBlock = poOtherBand->
GetLockedBlockRef(nBlockXOff,nBlockYOff, TRUE);
if (poBlock == nullptr)
{
continue;
}
GByte* pabyData = reinterpret_cast<GByte*>(poBlock->GetDataRef());
eErr = poOtherBand->IRasterIO(GF_Read, nXOff, nYOff, nXSize, nYSize,
pabyData, nXSize, nYSize,
eDataType,
nDTSizeBytes,
nDTSizeBytes * nXSize,
&sExtraArgs);
// Unpack previously packed buffer if needed
if( eErr == CE_None && nXSize < nBlockXSize )
{
for( int j = nYSize - 1; j >= 0; --j )
{
memmove( pabyData + j * nBlockXSize * nDTSizeBytes,
pabyData + j * nXSize * nDTSizeBytes,
nXSize * nDTSizeBytes );
}
}
poBlock->DropLock();
}
return eErr;
}
