int OutCoreInterp::update(double data_x, double data_y, double data_z)
{
int fileNum;
//fileNum = upper_grid_y / local_grid_size_y;
if((fileNum = findFileNum(data_y)) < 0)
{
cout << "OutCoreInterp::update() findFileNum() error!" << endl;
cout << "data_x: " << data_x << " data_y: " << data_y << " grid_y: " << (int)(data_y/GRID_DIST_Y) << " fileNum: " << fileNum << " open file: " << openFile << endl;
return -1;
}
if(openFile == fileNum)
{
// write into memory;
updateInterpArray(fileNum, data_x, data_y, data_z);
}else{
UpdateInfo ui(data_x, data_y, data_z);
qlist[fileNum].push_back(ui);
if(qlist[fileNum].size() == QUEUE_LIMIT)
{
//cout << "erase" << endl;
if(openFile != -1)
{
// if we use mmap and write directly into disk, no need this step.
// munmap would be enough for this step..
// write back to disk
gridMap[openFile]->getGridFile()->unmap();
openFile = -1;
}
// upload from disk to memory
gridMap[fileNum]->getGridFile()->map();
openFile = fileNum;
// pop every update information
list<UpdateInfo>::const_iterator iter;
for(iter = qlist[openFile].begin(); iter!= qlist[openFile].end(); iter++){
updateInterpArray(openFile, (*iter).data_x, (*iter).data_y, (*iter).data_z);
}
// flush
qlist[openFile].erase(qlist[openFile].begin(), qlist[openFile].end());
}
}
return 0;
}
int OutCoreInterp::finish(char *outputName, int outputFormat, unsigned int outputType)
{
int i, j;
GridPoint *p;
GridFile *gf;
int len_y;
int offset;
//struct tms tbuf;
clock_t t0, t1;
/*
// managing overlap.
read adjacent blocks and store ghost cells and update the cells
merging multiple files
*/
if(openFile != -1){
gridMap[openFile]->getGridFile()->unmap();
openFile = -1;
}
for(i = 0; i < numFiles; i++)
{
if(qlist[i].size() != 0)
{
if((gf = gridMap[i]->getGridFile()) == NULL)
{
cout << "OutCoreInterp::finish() getGridFile() NULL" << endl;
return -1;
}
gf->map();
openFile = i;
// flush UpdateInfo queue
// pop every update information
list<UpdateInfo>::const_iterator iter;
for(iter = qlist[i].begin(); iter!= qlist[i].end(); iter++){
updateInterpArray(i, (*iter).data_x, (*iter).data_y, (*iter).data_z);
}
qlist[i].erase(qlist[i].begin(), qlist[i].end());
gf->unmap();
openFile = -1;
gf = NULL;
}
}
for(i = 0; i < numFiles - 1; i++)
{
// read the upside overlap
if((gf = gridMap[i]->getGridFile()) == NULL)
{
cout << "OutCoreInterp::finish() getGridFile() NULL" << endl;
return -1;
}
if(gf->map() != -1)
{
openFile = i;
// Sriram's edit to copy over DEM values to overlap also
len_y = 2 * (gridMap[i]->getOverlapUpperBound() - gridMap[i]->getUpperBound() - 1);
if((p = (GridPoint *)malloc(sizeof(GridPoint) * len_y * GRID_SIZE_X)) == NULL)
{
cout << "OutCoreInterp::finish() malloc error" << endl;
return -1;
}
int start = (gridMap[i]->getOverlapUpperBound() - gridMap[i]->getOverlapLowerBound() - len_y) * GRID_SIZE_X;
cout << "copy from " << start << " to " << (start + len_y * GRID_SIZE_X) << endl;
memcpy(p, &(gf->interp[start]), sizeof(GridPoint) * len_y * (GRID_SIZE_X) );
gf->unmap();
gf = NULL;
openFile = -1;
} else {
cout << "OutCoreInterp::finish() gf->map() error" << endl;
return -1;
}
// update (i+1)th component
if((gf = gridMap[i+1]->getGridFile()) == NULL)
{
cout << "OutCoreInterp::finish() getGridFile() NULL" << endl;
return -1;
}
if(gf->map() != -1)
{
offset = 0;
openFile = i - 1;
for(j = 0; j < len_y * GRID_SIZE_X; j++)
{
if(gf->interp[j + offset].Zmin > p[j].Zmin)
gf->interp[j + offset].Zmin = p[j].Zmin;
if(gf->interp[j + offset].Zmax < p[j].Zmax)
gf->interp[j + offset].Zmax = p[j].Zmax;
gf->interp[j + offset].Zmean += p[j].Zmean;
gf->interp[j + offset].count += p[j].count;
if (p[j].sum != -1) {
gf->interp[j + offset].Zidw += p[j].Zidw;
gf->interp[j + offset].sum += p[j].sum;
} else {
gf->interp[j + offset].Zidw = p[j].Zidw;
gf->interp[j + offset].sum = p[j].sum;
}
}
if(p != NULL){
free(p);
p = NULL;
}
gf->unmap();
gf = NULL;
openFile = -1;
} else {
cout << "OutCoreInterp::finish() gf->map() error" << endl;
return -1;
}
}
for(i = numFiles -1; i > 0; i--)
{
// read the downside overlap
if((gf = gridMap[i]->getGridFile()) == NULL)
{
cout << "GridFile is NULL" << endl;
return -1;
}
if(gf->map() != -1)
{
openFile = i;
len_y = 2 * (gridMap[i]->getLowerBound() - gridMap[i]->getOverlapLowerBound());
if((p = (GridPoint *)malloc(sizeof(GridPoint) * len_y * GRID_SIZE_X)) == NULL)
{
cout << "OutCoreInterp::finish() malloc error" << endl;
return -1;
}
memcpy(p, &(gf->interp[0]), len_y * sizeof(GridPoint) * GRID_SIZE_X);
//finalize();
gf->unmap();
gf = NULL;
openFile = -1;
} else {
cout << "OutCoreInterp::finish gf->map() error" << endl;
return -1;
}
// update (i-1)th component
if((gf = gridMap[i-1]->getGridFile()) == NULL)
{
cout << "GridFile is NULL" << endl;
return -1;
}
if(gf->map() != -1)
{
offset = (gridMap[i-1]->getOverlapUpperBound() - gridMap[i-1]->getOverlapLowerBound() - len_y) * GRID_SIZE_X;
openFile = i - 1;
for(j = 0; j < len_y * GRID_SIZE_X; j++)
{
// Sriram - the overlap already contains the correct values
gf->interp[j + offset].Zmin = p[j].Zmin;
gf->interp[j + offset].Zmax = p[j].Zmax;
gf->interp[j + offset].Zmean = p[j].Zmean;
gf->interp[j + offset].count = p[j].count;
gf->interp[j + offset].Zidw = p[j].Zidw;
gf->interp[j + offset].sum = p[j].sum;
}
//if(i - 1 == 0)
//finalize();
if(p != NULL){
free(p);
p = NULL;
}
gf->unmap();
gf = NULL;
openFile = -1;
}
}
for(i = 0; i < numFiles; i++)
{
gridMap[i]->getGridFile()->map();
openFile = i;
finalize();
gridMap[i]->getGridFile()->unmap();
openFile = -1;
}
t0 = clock();
// merge pieces into one file
if(outputFile(outputName, outputFormat, outputType) < 0)
{
cout << "OutCoreInterp::finish outputFile error" << endl;
return -1;
}
t1 = clock();
printf("Output Execution time: %10.2f\n", (double)(t1 - t0)/CLOCKS_PER_SEC);
return 0;
}
int OutCoreInterp::update(double data_x, double data_y, double data_z)
{
// update()
// push the point info into an appropriate queue
// if the length of the queue exceeds a limit,
// then
// if another file is loaded, write back (lazy update)
// if the file we need has not opened,
// read a file from disk
// update the file (PROC1)
//
// find which file should be updated
/*
if(data_x < 0){
cout << "4. !!!! " << data_x << endl;
return -1;
}
*/
int fileNum;
//fileNum = upper_grid_y / local_grid_size_y;
if((fileNum = findFileNum(data_y)) < 0)
{
cerr << "OutCoreInterp::update() findFileNum() error!" << endl;
cerr << "data_x: " << data_x << " data_y: " << data_y << " grid_y: " << (int)(data_y/GRID_DIST_Y) << " fileNum: " << fileNum << " open file: " << openFile << endl;
return -1;
}
if(openFile == fileNum)
{
// write into memory;
updateInterpArray(fileNum, data_x, data_y, data_z);
} else {
UpdateInfo ui(data_x, data_y, data_z);
qlist[fileNum].push_back(ui);
if(qlist[fileNum].size() == QUEUE_LIMIT)
{
//cout << "erase" << endl;
if(openFile != -1)
{
// if we use mmap and write directly into disk, no need this step.
// munmap would be enough for this step..
// write back to disk
gridMap[openFile]->getGridFile()->unmap();
openFile = -1;
}
// upload from disk to memory
gridMap[fileNum]->getGridFile()->map();
openFile = fileNum;
// pop every update information
list<UpdateInfo>::const_iterator iter;
for(iter = qlist[openFile].begin(); iter!= qlist[openFile].end(); iter++) {
updateInterpArray(openFile, (*iter).data_x, (*iter).data_y, (*iter).data_z);
}
// flush
qlist[openFile].erase(qlist[openFile].begin(), qlist[openFile].end());
}
}
return 0;
}
int OutCoreInterp::finish(const std::string& outputName, int outputFormat, unsigned int outputType, double *adfGeoTransform, const char* wkt)
{
int i, j;
GridPoint *p;
GridFile *gf;
int len_y;
int offset;
//struct tms tbuf;
clock_t t0, t1;
/*
// managing overlap.
read adjacent blocks and store ghost cells and update the cells
merging multiple files
*/
if(openFile != -1) {
gridMap[openFile]->getGridFile()->unmap();
openFile = -1;
}
////////////////////////////////////////////////////////////
// flushing
// can be moved inside the communications
for(i = 0; i < numFiles; i++)
{
if(qlist[i].size() != 0)
{
if((gf = gridMap[i]->getGridFile()) == NULL)
{
cerr << "OutCoreInterp::finish() getGridFile() NULL" << endl;
return -1;
}
gf->map();
openFile = i;
// flush UpdateInfo queue
// pop every update information
list<UpdateInfo>::const_iterator iter;
for(iter = qlist[i].begin(); iter!= qlist[i].end(); iter++) {
updateInterpArray(i, (*iter).data_x, (*iter).data_y, (*iter).data_z);
}
qlist[i].erase(qlist[i].begin(), qlist[i].end());
gf->unmap();
openFile = -1;
gf = NULL;
}
}
////////////////////////////////////////////////////////////
for(i = 0; i < numFiles - 1; i++)
{
//////////////////////////////////////////////////////////////
// read the upside overlap
if((gf = gridMap[i]->getGridFile()) == NULL)
{
cerr << "OutCoreInterp::finish() getGridFile() NULL" << endl;
return -1;
}
if(gf->map() != -1)
{
openFile = i;
// Sriram's edit to copy over DEM values to overlap also
len_y = 2 * (gridMap[i]->getOverlapUpperBound() - gridMap[i]->getUpperBound() - 1);
if((p = (GridPoint *)malloc(sizeof(GridPoint) * len_y * GRID_SIZE_X)) == NULL)
{
cerr << "OutCoreInterp::finish() malloc error" << endl;
return -1;
}
int start = (gridMap[i]->getOverlapUpperBound() - gridMap[i]->getOverlapLowerBound() - len_y) * GRID_SIZE_X;
cerr << "copy from " << start << " to " << (start + len_y * GRID_SIZE_X) << endl;
memcpy(p, &(gf->interp[start]), sizeof(GridPoint) * len_y * (GRID_SIZE_X) );
gf->unmap();
gf = NULL;
openFile = -1;
} else {
cerr << "OutCoreInterp::finish() gf->map() error" << endl;
return -1;
}
//////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////
// update (i+1)th component
if((gf = gridMap[i+1]->getGridFile()) == NULL)
{
cerr << "OutCoreInterp::finish() getGridFile() NULL" << endl;
return -1;
}
if(gf->map() != -1)
{
offset = 0;
openFile = i - 1;
for(j = 0; j < len_y * GRID_SIZE_X; j++)
{
if(gf->interp[j + offset].Zmin > p[j].Zmin)
gf->interp[j + offset].Zmin = p[j].Zmin;
if(gf->interp[j + offset].Zmax < p[j].Zmax)
gf->interp[j + offset].Zmax = p[j].Zmax;
gf->interp[j + offset].Zmean += p[j].Zmean;
gf->interp[j + offset].count += p[j].count;
/*
// https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Online_algorithm
double delta = p[j].Zstd_tmp - p[j].Zstd_tmp;
gf->interp[j + offset].Zstd_tmp += delta / gf->interp[j + offset].count;
gf->interp[j + offset].Zstd += delta * (
interp[x][y].Zstd += delta * (data_z-interp[x][y].Zstd_tmp);
gf->interp[j + offset].Zstd += p[j].Zstd;
gf->interp[j + offset].Zstd_tmp += p[j].Zstd_tmp;
*/
if (p[j].sum != -1) {
gf->interp[j + offset].Zidw += p[j].Zidw;
gf->interp[j + offset].sum += p[j].sum;
} else {
gf->interp[j + offset].Zidw = p[j].Zidw;
gf->interp[j + offset].sum = p[j].sum;
}
}
if(p != NULL) {
free(p);
p = NULL;
}
gf->unmap();
gf = NULL;
openFile = -1;
} else {
cerr << "OutCoreInterp::finish() gf->map() error" << endl;
return -1;
}
//////////////////////////////////////////////////////////////
}
for(i = numFiles -1; i > 0; i--)
{
//////////////////////////////////////////////////////////////
// read the downside overlap
if((gf = gridMap[i]->getGridFile()) == NULL)
{
cerr << "GridFile is NULL" << endl;
return -1;
}
if(gf->map() != -1)
{
openFile = i;
len_y = 2 * (gridMap[i]->getLowerBound() - gridMap[i]->getOverlapLowerBound());
if((p = (GridPoint *)malloc(sizeof(GridPoint) * len_y * GRID_SIZE_X)) == NULL)
{
cerr << "OutCoreInterp::finish() malloc error" << endl;
return -1;
}
memcpy(p, &(gf->interp[0]), len_y * sizeof(GridPoint) * GRID_SIZE_X);
//finalize();
gf->unmap();
gf = NULL;
openFile = -1;
} else {
cerr << "OutCoreInterp::finish gf->map() error" << endl;
return -1;
}
//////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////
// update (i-1)th component
if((gf = gridMap[i-1]->getGridFile()) == NULL)
{
cerr << "GridFile is NULL" << endl;
return -1;
}
if(gf->map() != -1)
{
offset = (gridMap[i-1]->getOverlapUpperBound() - gridMap[i-1]->getOverlapLowerBound() - len_y) * GRID_SIZE_X;
openFile = i - 1;
for(j = 0; j < len_y * GRID_SIZE_X; j++)
{
// Sriram - the overlap already contains the correct values
gf->interp[j + offset].Zmin = p[j].Zmin;
gf->interp[j + offset].Zmax = p[j].Zmax;
gf->interp[j + offset].Zmean = p[j].Zmean;
gf->interp[j + offset].count = p[j].count;
gf->interp[j + offset].Zidw = p[j].Zidw;
gf->interp[j + offset].sum = p[j].sum;
//gf->interp[j + offset].Zstd = p[j].Zstd;
//gf->interp[j + offset].Zstd_tmp = p[j].Zstd_tmp;
}
//if(i - 1 == 0)
//finalize();
if(p != NULL) {
free(p);
p = NULL;
}
gf->unmap();
gf = NULL;
openFile = -1;
}
//////////////////////////////////////////////////////////////
}
for(i = 0; i < numFiles; i++)
{
gridMap[i]->getGridFile()->map();
openFile = i;
finalize();
gridMap[i]->getGridFile()->unmap();
openFile = -1;
}
t0 = clock();
//t0 = times(&tbuf);
// merge pieces into one file
if(outputFile(outputName, outputFormat, outputType, adfGeoTransform, wkt) < 0)
{
cerr << "OutCoreInterp::finish outputFile error" << endl;
return -1;
}
t1 = clock();
//t1 = times(&tbuf);
cerr << "Output Execution time: " << (double)(t1 - t0)/CLOCKS_PER_SEC << std::endl;
return 0;
}
