int OutCoreInterp::update(double data_x, double data_y, double data_z) { int fileNum; //fileNum = upper_grid_y / local_grid_size_y; if((fileNum = findFileNum(data_y)) < 0) { cout << "OutCoreInterp::update() findFileNum() error!" << endl; cout << "data_x: " << data_x << " data_y: " << data_y << " grid_y: " << (int)(data_y/GRID_DIST_Y) << " fileNum: " << fileNum << " open file: " << openFile << endl; return -1; } if(openFile == fileNum) { // write into memory; updateInterpArray(fileNum, data_x, data_y, data_z); }else{ UpdateInfo ui(data_x, data_y, data_z); qlist[fileNum].push_back(ui); if(qlist[fileNum].size() == QUEUE_LIMIT) { //cout << "erase" << endl; if(openFile != -1) { // if we use mmap and write directly into disk, no need this step. // munmap would be enough for this step.. // write back to disk gridMap[openFile]->getGridFile()->unmap(); openFile = -1; } // upload from disk to memory gridMap[fileNum]->getGridFile()->map(); openFile = fileNum; // pop every update information list<UpdateInfo>::const_iterator iter; for(iter = qlist[openFile].begin(); iter!= qlist[openFile].end(); iter++){ updateInterpArray(openFile, (*iter).data_x, (*iter).data_y, (*iter).data_z); } // flush qlist[openFile].erase(qlist[openFile].begin(), qlist[openFile].end()); } } return 0; }
int OutCoreInterp::finish(char *outputName, int outputFormat, unsigned int outputType) { int i, j; GridPoint *p; GridFile *gf; int len_y; int offset; //struct tms tbuf; clock_t t0, t1; /* // managing overlap. read adjacent blocks and store ghost cells and update the cells merging multiple files */ if(openFile != -1){ gridMap[openFile]->getGridFile()->unmap(); openFile = -1; } for(i = 0; i < numFiles; i++) { if(qlist[i].size() != 0) { if((gf = gridMap[i]->getGridFile()) == NULL) { cout << "OutCoreInterp::finish() getGridFile() NULL" << endl; return -1; } gf->map(); openFile = i; // flush UpdateInfo queue // pop every update information list<UpdateInfo>::const_iterator iter; for(iter = qlist[i].begin(); iter!= qlist[i].end(); iter++){ updateInterpArray(i, (*iter).data_x, (*iter).data_y, (*iter).data_z); } qlist[i].erase(qlist[i].begin(), qlist[i].end()); gf->unmap(); openFile = -1; gf = NULL; } } for(i = 0; i < numFiles - 1; i++) { // read the upside overlap if((gf = gridMap[i]->getGridFile()) == NULL) { cout << "OutCoreInterp::finish() getGridFile() NULL" << endl; return -1; } if(gf->map() != -1) { openFile = i; // Sriram's edit to copy over DEM values to overlap also len_y = 2 * (gridMap[i]->getOverlapUpperBound() - gridMap[i]->getUpperBound() - 1); if((p = (GridPoint *)malloc(sizeof(GridPoint) * len_y * GRID_SIZE_X)) == NULL) { cout << "OutCoreInterp::finish() malloc error" << endl; return -1; } int start = (gridMap[i]->getOverlapUpperBound() - gridMap[i]->getOverlapLowerBound() - len_y) * GRID_SIZE_X; cout << "copy from " << start << " to " << (start + len_y * GRID_SIZE_X) << endl; memcpy(p, &(gf->interp[start]), sizeof(GridPoint) * len_y * (GRID_SIZE_X) ); gf->unmap(); gf = NULL; openFile = -1; } else { cout << "OutCoreInterp::finish() gf->map() error" << endl; return -1; } // update (i+1)th component if((gf = gridMap[i+1]->getGridFile()) == NULL) { cout << "OutCoreInterp::finish() getGridFile() NULL" << endl; return -1; } if(gf->map() != -1) { offset = 0; openFile = i - 1; for(j = 0; j < len_y * GRID_SIZE_X; j++) { if(gf->interp[j + offset].Zmin > p[j].Zmin) gf->interp[j + offset].Zmin = p[j].Zmin; if(gf->interp[j + offset].Zmax < p[j].Zmax) gf->interp[j + offset].Zmax = p[j].Zmax; gf->interp[j + offset].Zmean += p[j].Zmean; gf->interp[j + offset].count += p[j].count; if (p[j].sum != -1) { gf->interp[j + offset].Zidw += p[j].Zidw; gf->interp[j + offset].sum += p[j].sum; } else { gf->interp[j + offset].Zidw = p[j].Zidw; gf->interp[j + offset].sum = p[j].sum; } } if(p != NULL){ free(p); p = NULL; } gf->unmap(); gf = NULL; openFile = -1; } else { cout << "OutCoreInterp::finish() gf->map() error" << endl; return -1; } } for(i = numFiles -1; i > 0; i--) { // read the downside overlap if((gf = gridMap[i]->getGridFile()) == NULL) { cout << "GridFile is NULL" << endl; return -1; } if(gf->map() != -1) { openFile = i; len_y = 2 * (gridMap[i]->getLowerBound() - gridMap[i]->getOverlapLowerBound()); if((p = (GridPoint *)malloc(sizeof(GridPoint) * len_y * GRID_SIZE_X)) == NULL) { cout << "OutCoreInterp::finish() malloc error" << endl; return -1; } memcpy(p, &(gf->interp[0]), len_y * sizeof(GridPoint) * GRID_SIZE_X); //finalize(); gf->unmap(); gf = NULL; openFile = -1; } else { cout << "OutCoreInterp::finish gf->map() error" << endl; return -1; } // update (i-1)th component if((gf = gridMap[i-1]->getGridFile()) == NULL) { cout << "GridFile is NULL" << endl; return -1; } if(gf->map() != -1) { offset = (gridMap[i-1]->getOverlapUpperBound() - gridMap[i-1]->getOverlapLowerBound() - len_y) * GRID_SIZE_X; openFile = i - 1; for(j = 0; j < len_y * GRID_SIZE_X; j++) { // Sriram - the overlap already contains the correct values gf->interp[j + offset].Zmin = p[j].Zmin; gf->interp[j + offset].Zmax = p[j].Zmax; gf->interp[j + offset].Zmean = p[j].Zmean; gf->interp[j + offset].count = p[j].count; gf->interp[j + offset].Zidw = p[j].Zidw; gf->interp[j + offset].sum = p[j].sum; } //if(i - 1 == 0) //finalize(); if(p != NULL){ free(p); p = NULL; } gf->unmap(); gf = NULL; openFile = -1; } } for(i = 0; i < numFiles; i++) { gridMap[i]->getGridFile()->map(); openFile = i; finalize(); gridMap[i]->getGridFile()->unmap(); openFile = -1; } t0 = clock(); // merge pieces into one file if(outputFile(outputName, outputFormat, outputType) < 0) { cout << "OutCoreInterp::finish outputFile error" << endl; return -1; } t1 = clock(); printf("Output Execution time: %10.2f\n", (double)(t1 - t0)/CLOCKS_PER_SEC); return 0; }
int OutCoreInterp::update(double data_x, double data_y, double data_z) { // update() // push the point info into an appropriate queue // if the length of the queue exceeds a limit, // then // if another file is loaded, write back (lazy update) // if the file we need has not opened, // read a file from disk // update the file (PROC1) // // find which file should be updated /* if(data_x < 0){ cout << "4. !!!! " << data_x << endl; return -1; } */ int fileNum; //fileNum = upper_grid_y / local_grid_size_y; if((fileNum = findFileNum(data_y)) < 0) { cerr << "OutCoreInterp::update() findFileNum() error!" << endl; cerr << "data_x: " << data_x << " data_y: " << data_y << " grid_y: " << (int)(data_y/GRID_DIST_Y) << " fileNum: " << fileNum << " open file: " << openFile << endl; return -1; } if(openFile == fileNum) { // write into memory; updateInterpArray(fileNum, data_x, data_y, data_z); } else { UpdateInfo ui(data_x, data_y, data_z); qlist[fileNum].push_back(ui); if(qlist[fileNum].size() == QUEUE_LIMIT) { //cout << "erase" << endl; if(openFile != -1) { // if we use mmap and write directly into disk, no need this step. // munmap would be enough for this step.. // write back to disk gridMap[openFile]->getGridFile()->unmap(); openFile = -1; } // upload from disk to memory gridMap[fileNum]->getGridFile()->map(); openFile = fileNum; // pop every update information list<UpdateInfo>::const_iterator iter; for(iter = qlist[openFile].begin(); iter!= qlist[openFile].end(); iter++) { updateInterpArray(openFile, (*iter).data_x, (*iter).data_y, (*iter).data_z); } // flush qlist[openFile].erase(qlist[openFile].begin(), qlist[openFile].end()); } } return 0; }
int OutCoreInterp::finish(const std::string& outputName, int outputFormat, unsigned int outputType, double *adfGeoTransform, const char* wkt) { int i, j; GridPoint *p; GridFile *gf; int len_y; int offset; //struct tms tbuf; clock_t t0, t1; /* // managing overlap. read adjacent blocks and store ghost cells and update the cells merging multiple files */ if(openFile != -1) { gridMap[openFile]->getGridFile()->unmap(); openFile = -1; } //////////////////////////////////////////////////////////// // flushing // can be moved inside the communications for(i = 0; i < numFiles; i++) { if(qlist[i].size() != 0) { if((gf = gridMap[i]->getGridFile()) == NULL) { cerr << "OutCoreInterp::finish() getGridFile() NULL" << endl; return -1; } gf->map(); openFile = i; // flush UpdateInfo queue // pop every update information list<UpdateInfo>::const_iterator iter; for(iter = qlist[i].begin(); iter!= qlist[i].end(); iter++) { updateInterpArray(i, (*iter).data_x, (*iter).data_y, (*iter).data_z); } qlist[i].erase(qlist[i].begin(), qlist[i].end()); gf->unmap(); openFile = -1; gf = NULL; } } //////////////////////////////////////////////////////////// for(i = 0; i < numFiles - 1; i++) { ////////////////////////////////////////////////////////////// // read the upside overlap if((gf = gridMap[i]->getGridFile()) == NULL) { cerr << "OutCoreInterp::finish() getGridFile() NULL" << endl; return -1; } if(gf->map() != -1) { openFile = i; // Sriram's edit to copy over DEM values to overlap also len_y = 2 * (gridMap[i]->getOverlapUpperBound() - gridMap[i]->getUpperBound() - 1); if((p = (GridPoint *)malloc(sizeof(GridPoint) * len_y * GRID_SIZE_X)) == NULL) { cerr << "OutCoreInterp::finish() malloc error" << endl; return -1; } int start = (gridMap[i]->getOverlapUpperBound() - gridMap[i]->getOverlapLowerBound() - len_y) * GRID_SIZE_X; cerr << "copy from " << start << " to " << (start + len_y * GRID_SIZE_X) << endl; memcpy(p, &(gf->interp[start]), sizeof(GridPoint) * len_y * (GRID_SIZE_X) ); gf->unmap(); gf = NULL; openFile = -1; } else { cerr << "OutCoreInterp::finish() gf->map() error" << endl; return -1; } ////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////// // update (i+1)th component if((gf = gridMap[i+1]->getGridFile()) == NULL) { cerr << "OutCoreInterp::finish() getGridFile() NULL" << endl; return -1; } if(gf->map() != -1) { offset = 0; openFile = i - 1; for(j = 0; j < len_y * GRID_SIZE_X; j++) { if(gf->interp[j + offset].Zmin > p[j].Zmin) gf->interp[j + offset].Zmin = p[j].Zmin; if(gf->interp[j + offset].Zmax < p[j].Zmax) gf->interp[j + offset].Zmax = p[j].Zmax; gf->interp[j + offset].Zmean += p[j].Zmean; gf->interp[j + offset].count += p[j].count; /* // https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Online_algorithm double delta = p[j].Zstd_tmp - p[j].Zstd_tmp; gf->interp[j + offset].Zstd_tmp += delta / gf->interp[j + offset].count; gf->interp[j + offset].Zstd += delta * ( interp[x][y].Zstd += delta * (data_z-interp[x][y].Zstd_tmp); gf->interp[j + offset].Zstd += p[j].Zstd; gf->interp[j + offset].Zstd_tmp += p[j].Zstd_tmp; */ if (p[j].sum != -1) { gf->interp[j + offset].Zidw += p[j].Zidw; gf->interp[j + offset].sum += p[j].sum; } else { gf->interp[j + offset].Zidw = p[j].Zidw; gf->interp[j + offset].sum = p[j].sum; } } if(p != NULL) { free(p); p = NULL; } gf->unmap(); gf = NULL; openFile = -1; } else { cerr << "OutCoreInterp::finish() gf->map() error" << endl; return -1; } ////////////////////////////////////////////////////////////// } for(i = numFiles -1; i > 0; i--) { ////////////////////////////////////////////////////////////// // read the downside overlap if((gf = gridMap[i]->getGridFile()) == NULL) { cerr << "GridFile is NULL" << endl; return -1; } if(gf->map() != -1) { openFile = i; len_y = 2 * (gridMap[i]->getLowerBound() - gridMap[i]->getOverlapLowerBound()); if((p = (GridPoint *)malloc(sizeof(GridPoint) * len_y * GRID_SIZE_X)) == NULL) { cerr << "OutCoreInterp::finish() malloc error" << endl; return -1; } memcpy(p, &(gf->interp[0]), len_y * sizeof(GridPoint) * GRID_SIZE_X); //finalize(); gf->unmap(); gf = NULL; openFile = -1; } else { cerr << "OutCoreInterp::finish gf->map() error" << endl; return -1; } ////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////// // update (i-1)th component if((gf = gridMap[i-1]->getGridFile()) == NULL) { cerr << "GridFile is NULL" << endl; return -1; } if(gf->map() != -1) { offset = (gridMap[i-1]->getOverlapUpperBound() - gridMap[i-1]->getOverlapLowerBound() - len_y) * GRID_SIZE_X; openFile = i - 1; for(j = 0; j < len_y * GRID_SIZE_X; j++) { // Sriram - the overlap already contains the correct values gf->interp[j + offset].Zmin = p[j].Zmin; gf->interp[j + offset].Zmax = p[j].Zmax; gf->interp[j + offset].Zmean = p[j].Zmean; gf->interp[j + offset].count = p[j].count; gf->interp[j + offset].Zidw = p[j].Zidw; gf->interp[j + offset].sum = p[j].sum; //gf->interp[j + offset].Zstd = p[j].Zstd; //gf->interp[j + offset].Zstd_tmp = p[j].Zstd_tmp; } //if(i - 1 == 0) //finalize(); if(p != NULL) { free(p); p = NULL; } gf->unmap(); gf = NULL; openFile = -1; } ////////////////////////////////////////////////////////////// } for(i = 0; i < numFiles; i++) { gridMap[i]->getGridFile()->map(); openFile = i; finalize(); gridMap[i]->getGridFile()->unmap(); openFile = -1; } t0 = clock(); //t0 = times(&tbuf); // merge pieces into one file if(outputFile(outputName, outputFormat, outputType, adfGeoTransform, wkt) < 0) { cerr << "OutCoreInterp::finish outputFile error" << endl; return -1; } t1 = clock(); //t1 = times(&tbuf); cerr << "Output Execution time: " << (double)(t1 - t0)/CLOCKS_PER_SEC << std::endl; return 0; }