int main(int argc, char* argv[]) { int i; int nThreads = CPLGetNumCPUs(); std::vector<CPLJoinableThread*> apsThreads; Strategy eStrategy = STRATEGY_RANDOM; int bNewDatasetOption = FALSE; int nXSize = 5000; int nYSize = 5000; int nBands = 4; char** papszOptions = NULL; int bOnDisk = FALSE; std::vector<ThreadDescription> asThreadDescription; int bMemDriver = FALSE; GDALDataset* poMEMDS = NULL; int bMigrate = FALSE; int nMaxRequests = -1; argc = GDALGeneralCmdLineProcessor( argc, &argv, 0 ); GDALAllRegister(); for(i = 1; i < argc; i++) { if( EQUAL(argv[i], "-threads") && i + 1 < argc) { i ++; nThreads = atoi(argv[i]); } else if( EQUAL(argv[i], "-loops") && i + 1 < argc) { i ++; nLoops = atoi(argv[i]); if( nLoops <= 0 ) nLoops = INT_MAX; } else if( EQUAL(argv[i], "-max_requests") && i + 1 < argc) { i ++; nMaxRequests = atoi(argv[i]); } else if( EQUAL(argv[i], "-strategy") && i + 1 < argc) { i ++; if( EQUAL(argv[i], "random") ) eStrategy = STRATEGY_RANDOM; else if( EQUAL(argv[i], "line") ) eStrategy = STRATEGY_LINE; else if( EQUAL(argv[i], "block") ) eStrategy = STRATEGY_BLOCK; else Usage(); } else if( EQUAL(argv[i], "-xsize") && i + 1 < argc) { i ++; nXSize = atoi(argv[i]); bNewDatasetOption = TRUE; } else if( EQUAL(argv[i], "-ysize") && i + 1 < argc) { i ++; nYSize = atoi(argv[i]); bNewDatasetOption = TRUE; } else if( EQUAL(argv[i], "-bands") && i + 1 < argc) { i ++; nBands = atoi(argv[i]); bNewDatasetOption = TRUE; } else if( EQUAL(argv[i], "-co") && i + 1 < argc) { i ++; papszOptions = CSLAddString(papszOptions, argv[i]); bNewDatasetOption = TRUE; } else if( EQUAL(argv[i], "-ondisk")) { bOnDisk = TRUE; bNewDatasetOption = TRUE; } else if( EQUAL(argv[i], "-check")) { bCheck = TRUE; bNewDatasetOption = TRUE; } else if( EQUAL(argv[i], "-memdriver")) { bMemDriver = TRUE; bNewDatasetOption = TRUE; } else if( EQUAL(argv[i], "-migrate")) bMigrate = TRUE; else if( argv[i][0] == '-' ) Usage(); else if( pszDataset == NULL ) pszDataset = argv[i]; else { Usage(); } } if( pszDataset != NULL && bNewDatasetOption ) Usage(); CPLDebug("TEST", "Using %d threads", nThreads); int bCreatedDataset = FALSE; if( pszDataset == NULL ) { bCreatedDataset = TRUE; if( bOnDisk ) pszDataset = "/tmp/tmp.tif"; else pszDataset = "/vsimem/tmp.tif"; GDALDataset* poDS = ((GDALDriver*)GDALGetDriverByName((bMemDriver) ? "MEM" : "GTiff"))->Create(pszDataset, nXSize, nYSize, nBands, GDT_Byte, papszOptions); if( bCheck ) { GByte* pabyLine = (GByte*) VSIMalloc(nBands * nXSize); for(int iY=0;iY<nYSize;iY++) { for(int iX=0;iX<nXSize;iX++) { for(int iBand=0;iBand<nBands;iBand++) { unsigned long seed = iBand * nXSize * nYSize + iY * nXSize + iX; pabyLine[iBand * nXSize + iX] = (GByte)(myrand_r(&seed) & 0xff); } } CPL_IGNORE_RET_VAL(poDS->RasterIO(GF_Write, 0, iY, nXSize, 1, pabyLine, nXSize, 1, GDT_Byte, nBands, NULL, 0, 0, 0 #ifdef GDAL_COMPILATION , NULL #endif )); } VSIFree(pabyLine); } if( bMemDriver ) poMEMDS = poDS; else GDALClose(poDS); } else { bCheck = FALSE; } CSLDestroy(papszOptions); papszOptions = NULL; Request* psGlobalRequestLast = NULL; for(i = 0; i < nThreads; i++ ) { GDALDataset* poDS; // Since GDAL 2.0, the MEM driver is thread-safe, i.e. does not use the block // cache, but only for operations not involving resampling, which is // the case here if( poMEMDS ) poDS = poMEMDS; else { poDS = (GDALDataset*)GDALOpen(pszDataset, GA_ReadOnly); if( poDS == NULL ) exit(1); } if( bMigrate ) { Resource* psResource = (Resource*)CPLMalloc(sizeof(Resource)); psResource->poDS = poDS; int nBufferSize; if( eStrategy == STRATEGY_RANDOM ) nBufferSize = CreateRandomStrategyRequests( poDS, nMaxRequests, psGlobalRequestList, psGlobalRequestLast); else if( eStrategy == STRATEGY_LINE ) nBufferSize = CreateLineStrategyRequests( poDS, nMaxRequests, psGlobalRequestList, psGlobalRequestLast); else nBufferSize = CreateBlockStrategyRequests( poDS, nMaxRequests, psGlobalRequestList, psGlobalRequestLast); psResource->pBuffer = CPLMalloc(nBufferSize); PutResourceAtEnd(psResource); } else { ThreadDescription sThreadDescription; sThreadDescription.poDS = poDS; sThreadDescription.psRequestList = NULL; Request* psRequestLast = NULL; if( eStrategy == STRATEGY_RANDOM ) sThreadDescription.nBufferSize = CreateRandomStrategyRequests( poDS, nMaxRequests, sThreadDescription.psRequestList, psRequestLast); else if( eStrategy == STRATEGY_LINE ) sThreadDescription.nBufferSize = CreateLineStrategyRequests( poDS, nMaxRequests, sThreadDescription.psRequestList, psRequestLast); else sThreadDescription.nBufferSize = CreateBlockStrategyRequests( poDS, nMaxRequests, sThreadDescription.psRequestList, psRequestLast); asThreadDescription.push_back(sThreadDescription); } } if( bCreatedDataset && poMEMDS == NULL && bOnDisk ) { CPLPushErrorHandler(CPLQuietErrorHandler); VSIUnlink(pszDataset); CPLPopErrorHandler(); } if( bMigrate ) { psLock = CPLCreateLock(LOCK_SPIN); } for(i = 0; i < nThreads; i++ ) { CPLJoinableThread* pThread; if( bMigrate ) pThread = CPLCreateJoinableThread(ThreadFuncWithMigration, NULL); else pThread = CPLCreateJoinableThread(ThreadFuncDedicatedDataset, &(asThreadDescription[i])); apsThreads.push_back(pThread); } for(i = 0; i < nThreads; i++ ) { CPLJoinThread(apsThreads[i]); if( !bMigrate && poMEMDS == NULL ) GDALClose(asThreadDescription[i].poDS); } while( psGlobalResourceList != NULL ) { CPLFree( psGlobalResourceList->pBuffer); if( poMEMDS == NULL ) GDALClose(psGlobalResourceList->poDS); Resource* psNext = psGlobalResourceList->psNext; CPLFree( psGlobalResourceList ); psGlobalResourceList = psNext; } if( psLock ) { CPLDestroyLock( psLock ); } if( bCreatedDataset && poMEMDS == NULL ) { CPLPushErrorHandler(CPLQuietErrorHandler); VSIUnlink(pszDataset); CPLPopErrorHandler(); } if( poMEMDS ) GDALClose(poMEMDS); assert( GDALGetCacheUsed64() == 0 ); GDALDestroyDriverManager(); CSLDestroy( argv ); return 0; }
void *VSIMalloc( size_t nSize ) { if (nMaxPeakAllocSize < 0) { char* pszMaxPeakAllocSize = getenv("CPL_MAX_PEAK_ALLOC_SIZE"); nMaxPeakAllocSize = (pszMaxPeakAllocSize) ? atoi(pszMaxPeakAllocSize) : 0; char* pszMaxCumulAllocSize = getenv("CPL_MAX_CUMUL_ALLOC_SIZE"); nMaxCumulAllocSize = (pszMaxCumulAllocSize) ? atoi(pszMaxCumulAllocSize) : 0; } if (nMaxPeakAllocSize > 0 && (GIntBig)nSize > nMaxPeakAllocSize) return NULL; #ifdef DEBUG_VSIMALLOC_STATS if (nMaxCumulAllocSize > 0 && (GIntBig)nCurrentTotalAllocs + (GIntBig)nSize > nMaxCumulAllocSize) return NULL; #endif // DEBUG_VSIMALLOC_STATS #ifdef DEBUG_VSIMALLOC_MPROTECT char* ptr = NULL; size_t nPageSize = getpagesize(); posix_memalign((void**)&ptr, nPageSize, (3 * sizeof(void*) + nSize + nPageSize - 1) & ~(nPageSize - 1)); #else char* ptr = (char*) malloc(3 * sizeof(void*) + nSize); #endif // DEBUG_VSIMALLOC_MPROTECT if (ptr == NULL) return NULL; ptr[0] = 'V'; ptr[1] = 'S'; ptr[2] = 'I'; ptr[3] = 'M'; memcpy(ptr + sizeof(void*), &nSize, sizeof(void*)); ptr[2 * sizeof(void*) + nSize + 0] = 'E'; ptr[2 * sizeof(void*) + nSize + 1] = 'V'; ptr[2 * sizeof(void*) + nSize + 2] = 'S'; ptr[2 * sizeof(void*) + nSize + 3] = 'I'; #if defined(DEBUG_VSIMALLOC_STATS) || defined(DEBUG_VSIMALLOC_VERBOSE) { CPLMutexHolderD(&hMemStatMutex); #ifdef DEBUG_VSIMALLOC_VERBOSE if( nSize > THRESHOLD_PRINT ) { fprintf(stderr, "Thread[%p] VSIMalloc(%d) = %p" #ifdef DEBUG_VSIMALLOC_STATS ", current_cumul = " CPL_FRMT_GUIB #ifdef DEBUG_BLOCK_CACHE_USE ", block_cache_used = " CPL_FRMT_GIB #endif ", mal+cal-free = %d" #endif "\n", (void*)CPLGetPID(), (int)nSize, ptr + 2 * sizeof(void*) #ifdef DEBUG_VSIMALLOC_STATS , (GUIntBig)(nCurrentTotalAllocs + nSize), #ifdef DEBUG_BLOCK_CACHE_USE , GDALGetCacheUsed64() #endif ,(int)(nVSIMallocs + nVSICallocs - nVSIFrees) #endif ); } #endif // DEBUG_VSIMALLOC_VERBOSE #ifdef DEBUG_VSIMALLOC_STATS nVSIMallocs ++; if (nMaxTotalAllocs == 0) atexit(VSIShowMemStats); nCurrentTotalAllocs += nSize; if (nCurrentTotalAllocs > nMaxTotalAllocs) nMaxTotalAllocs = nCurrentTotalAllocs; #endif // DEBUG_VSIMALLOC_STATS }