static int CreateRandomStrategyRequests(GDALDataset* poDS,
int nMaxRequests,
Request*& psRequestList,
Request*& psRequestLast)
{
unsigned long seed = 1;
int nXSize = poDS->GetRasterXSize();
int nYSize = poDS->GetRasterYSize();
int nMaxXWin = MIN(1000, nXSize/10+1);
int nMaxYWin = MIN(1000, nYSize/10+1);
int nQueriedBands = MIN(4, poDS->GetRasterCount());
int nAverageIterationsToReadWholeFile =
((nXSize + nMaxXWin/2-1) / (nMaxXWin/2)) * ((nYSize + nMaxYWin/2-1) / (nMaxYWin/2));
int nLocalLoops = nLoops * nAverageIterationsToReadWholeFile;
for(int iLoop=0;iLoop<nLocalLoops;iLoop++)
{
if( nMaxRequests > 0 && iLoop == nMaxRequests )
break;
int nXOff = (int)((GIntBig)myrand_r(&seed) * (nXSize-1) / MYRAND_MAX);
int nYOff = (int)((GIntBig)myrand_r(&seed) * (nYSize-1) / MYRAND_MAX);
int nXWin = 1+(int)((GIntBig)myrand_r(&seed) * nMaxXWin / MYRAND_MAX);
int nYWin = 1+(int)((GIntBig)myrand_r(&seed) * nMaxYWin / MYRAND_MAX);
if( nXOff + nXWin > nXSize )
nXWin = nXSize - nXOff;
if( nYOff + nYWin > nYSize )
nYWin = nYSize - nYOff;
AddRequest(psRequestList, psRequestLast, nXOff, nYOff, nXWin, nYWin, nQueriedBands);
}
return nQueriedBands * nMaxXWin * nMaxYWin;
}
static void Check(GByte* pBuffer, int nXSize, int nYSize, int nBands,
int nXOff, int nYOff, int nXWin, int nYWin)
{
for(int iBand=0;iBand<nBands;iBand++)
{
for(int iY=0;iY<nYWin;iY++)
{
for(int iX=0;iX<nXWin;iX++)
{
unsigned long seed = iBand * nXSize * nYSize + (iY + nYOff) * nXSize + iX + nXOff;
GByte expected = (GByte)myrand_r(&seed);
assert( pBuffer[iBand * nXWin * nYWin + iY * nXWin + iX] == expected );
}
}
}
}
void *LDARandomGroupCVModel(void *arg_)
{
size_t i, j, k, n, g;
lda_rgcv_th_arg *arg;
matrix *gid; /* randomization and storing id for each random group into a matrix */
/* Matrix for compute the PLS models for groups */
matrix *subX;
uivector *subY;
LDAMODEL *subm;
/*matrix to predict*/
matrix *predictX;
uivector *realY;
dvector *sens, *spec, *ppv, *npv, *acc;
initDVector(&sens);
initDVector(&spec);
initDVector(&ppv);
initDVector(&npv);
initDVector(&acc);
arg = (lda_rgcv_th_arg*) arg_;
NewMatrix(&gid, arg->group, (size_t)ceil(arg->mx->row/(double)arg->group));
/* Divide in group all the Dataset */
MatrixSet(gid, -1);
/* step 1 generate the random groups */
k = 0;
for(i = 0; i < gid->row; i++){
for(j = 0; j < gid->col; j++){
do{
/*n = randInt(0, arg->mx->row);*/
n = (size_t)myrand_r(&arg->srand_init) % (arg->mx->row);
} while(ValInMatrix(gid, n) == 1 && k < (arg->mx->row));
if(k < arg->mx->row){
gid->data[i][j] = n;
k++;
}
else
continue;
}
}
/*
puts("Gid Matrix");
PrintMatrix(gid);
*/
/*
printf("Excuded the group number %u\n", (unsigned int)g);
puts("Sub Model\nX:");
PrintArray(subX);
puts("Y:");
PrintArray(subY);
puts("\n\nPredict Group\nX:");
PrintArray(predictX);
puts("RealY:");
PrintArray(realY);
*/
/*step 2*/
for(g = 0; g < gid->row; g++){ /*For aeach group */
/* Estimate how many objects are inside the sub model without the group "g" */
n = 0;
for(i = 0; i < gid->row; i++){
if(i != g){
for(j = 0; j < gid->col; j++){
if((int)gid->data[i][j] != -1)
n++;
else
continue;
}
}
else
continue;
}
/*Allocate the submodel*/
NewMatrix(&subX, n, arg->mx->col);
NewUIVector(&subY, n);
/* Estimate how many objects are inside the group "g" to predict*/
n = 0;
for(j = 0; j < gid->col; j++){
if((int)gid->data[g][j] != -1)
n++;
else
continue;
}
/*Allocate the */
NewMatrix(&predictX, n, arg->mx->col);
NewUIVector(&realY, n);
/* copy the submodel values */
for(i = 0, k = 0; i < gid->row; i++){
if(i != g){
for(j = 0; j < gid->col; j++){
size_t a = (size_t)gid->data[i][j]; /* get the row index */
if(a != -1){
for(n = 0; n < arg->mx->col; n++){
/*setMatrixValue(subX, k, n, getMatrixValue(arg->mx, a, n));*/
subX->data[k][n] = arg->mx->data[a][n];
}
subY->data[k] = arg->my->data[a];
k++;
}
else{
continue;
}
}
}
else{
continue;
}
}
/* copy the objects to predict into predictmx*/
for(j = 0, k = 0; j < gid->col; j++){
size_t a = (size_t)gid->data[g][j];
if(a != -1){
for(n = 0; n < arg->mx->col; n++){
predictX->data[k][n] = arg->mx->data[a][n];
}
realY->data[k] = arg->my->data[a];
k++;
}
else{
continue;
}
}
NewLDAModel(&subm);
LDA(subX, subY, subm);
LDAError(predictX, realY, subm, &sens, &spec, &ppv, &npv, &acc);
if(arg->sens->size == 0){
DVectorCopy(sens, &arg->sens);
}
else{
for(i = 0; i < sens->size; i++){
arg->sens->data[i] += sens->data[i];
}
}
if(arg->spec->size == 0){
DVectorCopy(spec, &arg->spec);
}
else{
for(i = 0; i < spec->size; i++){
arg->spec->data[i] += spec->data[i];
}
}
if(arg->ppv->size == 0){
DVectorCopy(ppv, &arg->ppv);
}
else{
for(i = 0; i < ppv->size; i++){
arg->ppv->data[i] += ppv->data[i];
}
}
if(arg->npv->size == 0){
DVectorCopy(npv, &arg->npv);
}
else{
for(i = 0; i < npv->size; i++){
arg->npv->data[i] += npv->data[i];
}
}
if(arg->acc->size == 0){
DVectorCopy(acc, &arg->acc);
}
else{
for(i = 0; i < acc->size; i++){
arg->acc->data[i] += acc->data[i];
}
}
DelLDAModel(&subm);
DelMatrix(&subX);
DelUIVector(&subY);
DelMatrix(&predictX);
DelUIVector(&realY);
}
DelDVector(&acc);
DelDVector(&sens);
DelDVector(&spec);
DelDVector(&ppv);
DelDVector(&npv);
DelMatrix(&gid);
return 0;
}
int main(int argc, char* argv[])
{
int i;
int nThreads = CPLGetNumCPUs();
std::vector<CPLJoinableThread*> apsThreads;
Strategy eStrategy = STRATEGY_RANDOM;
int bNewDatasetOption = FALSE;
int nXSize = 5000;
int nYSize = 5000;
int nBands = 4;
char** papszOptions = NULL;
int bOnDisk = FALSE;
std::vector<ThreadDescription> asThreadDescription;
int bMemDriver = FALSE;
GDALDataset* poMEMDS = NULL;
int bMigrate = FALSE;
int nMaxRequests = -1;
argc = GDALGeneralCmdLineProcessor( argc, &argv, 0 );
GDALAllRegister();
for(i = 1; i < argc; i++)
{
if( EQUAL(argv[i], "-threads") && i + 1 < argc)
{
i ++;
nThreads = atoi(argv[i]);
}
else if( EQUAL(argv[i], "-loops") && i + 1 < argc)
{
i ++;
nLoops = atoi(argv[i]);
if( nLoops <= 0 )
nLoops = INT_MAX;
}
else if( EQUAL(argv[i], "-max_requests") && i + 1 < argc)
{
i ++;
nMaxRequests = atoi(argv[i]);
}
else if( EQUAL(argv[i], "-strategy") && i + 1 < argc)
{
i ++;
if( EQUAL(argv[i], "random") )
eStrategy = STRATEGY_RANDOM;
else if( EQUAL(argv[i], "line") )
eStrategy = STRATEGY_LINE;
else if( EQUAL(argv[i], "block") )
eStrategy = STRATEGY_BLOCK;
else
Usage();
}
else if( EQUAL(argv[i], "-xsize") && i + 1 < argc)
{
i ++;
nXSize = atoi(argv[i]);
bNewDatasetOption = TRUE;
}
else if( EQUAL(argv[i], "-ysize") && i + 1 < argc)
{
i ++;
nYSize = atoi(argv[i]);
bNewDatasetOption = TRUE;
}
else if( EQUAL(argv[i], "-bands") && i + 1 < argc)
{
i ++;
nBands = atoi(argv[i]);
bNewDatasetOption = TRUE;
}
else if( EQUAL(argv[i], "-co") && i + 1 < argc)
{
i ++;
papszOptions = CSLAddString(papszOptions, argv[i]);
bNewDatasetOption = TRUE;
}
else if( EQUAL(argv[i], "-ondisk"))
{
bOnDisk = TRUE;
bNewDatasetOption = TRUE;
}
else if( EQUAL(argv[i], "-check"))
{
bCheck = TRUE;
bNewDatasetOption = TRUE;
}
else if( EQUAL(argv[i], "-memdriver"))
{
bMemDriver = TRUE;
bNewDatasetOption = TRUE;
}
else if( EQUAL(argv[i], "-migrate"))
bMigrate = TRUE;
else if( argv[i][0] == '-' )
Usage();
else if( pszDataset == NULL )
pszDataset = argv[i];
else
{
Usage();
}
}
if( pszDataset != NULL && bNewDatasetOption )
Usage();
CPLDebug("TEST", "Using %d threads", nThreads);
int bCreatedDataset = FALSE;
if( pszDataset == NULL )
{
bCreatedDataset = TRUE;
if( bOnDisk )
pszDataset = "/tmp/tmp.tif";
else
pszDataset = "/vsimem/tmp.tif";
GDALDataset* poDS = ((GDALDriver*)GDALGetDriverByName((bMemDriver) ? "MEM" : "GTiff"))->Create(pszDataset,
nXSize, nYSize, nBands, GDT_Byte, papszOptions);
if( bCheck )
{
GByte* pabyLine = (GByte*) VSIMalloc(nBands * nXSize);
for(int iY=0;iY<nYSize;iY++)
{
for(int iX=0;iX<nXSize;iX++)
{
for(int iBand=0;iBand<nBands;iBand++)
{
unsigned long seed = iBand * nXSize * nYSize + iY * nXSize + iX;
pabyLine[iBand * nXSize + iX] = (GByte)(myrand_r(&seed) & 0xff);
}
}
CPL_IGNORE_RET_VAL(poDS->RasterIO(GF_Write, 0, iY, nXSize, 1,
pabyLine, nXSize, 1,
GDT_Byte,
nBands, NULL,
0, 0, 0
#ifdef GDAL_COMPILATION
, NULL
#endif
));
}
VSIFree(pabyLine);
}
if( bMemDriver )
poMEMDS = poDS;
else
GDALClose(poDS);
}
else
{
bCheck = FALSE;
}
CSLDestroy(papszOptions);
papszOptions = NULL;
Request* psGlobalRequestLast = NULL;
for(i = 0; i < nThreads; i++ )
{
GDALDataset* poDS;
// Since GDAL 2.0, the MEM driver is thread-safe, i.e. does not use the block
// cache, but only for operations not involving resampling, which is
// the case here
if( poMEMDS )
poDS = poMEMDS;
else
{
poDS = (GDALDataset*)GDALOpen(pszDataset, GA_ReadOnly);
if( poDS == NULL )
exit(1);
}
if( bMigrate )
{
Resource* psResource = (Resource*)CPLMalloc(sizeof(Resource));
psResource->poDS = poDS;
int nBufferSize;
if( eStrategy == STRATEGY_RANDOM )
nBufferSize = CreateRandomStrategyRequests(
poDS, nMaxRequests, psGlobalRequestList, psGlobalRequestLast);
else if( eStrategy == STRATEGY_LINE )
nBufferSize = CreateLineStrategyRequests(
poDS, nMaxRequests, psGlobalRequestList, psGlobalRequestLast);
else
nBufferSize = CreateBlockStrategyRequests(
poDS, nMaxRequests, psGlobalRequestList, psGlobalRequestLast);
psResource->pBuffer = CPLMalloc(nBufferSize);
PutResourceAtEnd(psResource);
}
else
{
ThreadDescription sThreadDescription;
sThreadDescription.poDS = poDS;
sThreadDescription.psRequestList = NULL;
Request* psRequestLast = NULL;
if( eStrategy == STRATEGY_RANDOM )
sThreadDescription.nBufferSize = CreateRandomStrategyRequests(
poDS, nMaxRequests, sThreadDescription.psRequestList, psRequestLast);
else if( eStrategy == STRATEGY_LINE )
sThreadDescription.nBufferSize = CreateLineStrategyRequests(
poDS, nMaxRequests, sThreadDescription.psRequestList, psRequestLast);
else
sThreadDescription.nBufferSize = CreateBlockStrategyRequests(
poDS, nMaxRequests, sThreadDescription.psRequestList, psRequestLast);
asThreadDescription.push_back(sThreadDescription);
}
}
if( bCreatedDataset && poMEMDS == NULL && bOnDisk )
{
CPLPushErrorHandler(CPLQuietErrorHandler);
VSIUnlink(pszDataset);
CPLPopErrorHandler();
}
if( bMigrate )
{
psLock = CPLCreateLock(LOCK_SPIN);
}
for(i = 0; i < nThreads; i++ )
{
CPLJoinableThread* pThread;
if( bMigrate )
pThread = CPLCreateJoinableThread(ThreadFuncWithMigration, NULL);
else
pThread = CPLCreateJoinableThread(ThreadFuncDedicatedDataset,
&(asThreadDescription[i]));
apsThreads.push_back(pThread);
}
for(i = 0; i < nThreads; i++ )
{
CPLJoinThread(apsThreads[i]);
if( !bMigrate && poMEMDS == NULL )
GDALClose(asThreadDescription[i].poDS);
}
while( psGlobalResourceList != NULL )
{
CPLFree( psGlobalResourceList->pBuffer);
if( poMEMDS == NULL )
GDALClose(psGlobalResourceList->poDS);
Resource* psNext = psGlobalResourceList->psNext;
CPLFree( psGlobalResourceList );
psGlobalResourceList = psNext;
}
if( psLock )
{
CPLDestroyLock( psLock );
}
if( bCreatedDataset && poMEMDS == NULL )
{
CPLPushErrorHandler(CPLQuietErrorHandler);
VSIUnlink(pszDataset);
CPLPopErrorHandler();
}
if( poMEMDS )
GDALClose(poMEMDS);
assert( GDALGetCacheUsed64() == 0 );
GDALDestroyDriverManager();
CSLDestroy( argv );
return 0;
}
