void SurroundednessTreeGenerator::merge(const comp_ptr &comp1, const comp_ptr &comp2) { assert(comp1 != comp2); if (comp1->isAncestorOf(comp2)) { mergeInto(comp2, comp1); } else { assert(comp1->getParent() == comp2->getParent() || comp2->isAncestorOf(comp1)); mergeInto(comp1, comp2); } }
MojErr MojDb::merge(const MojDbQuery& query, const MojObject& props, MojUInt32& countOut, MojUInt32 flags, MojDbReqRef req) { MojLogTrace(s_log); countOut = 0; MojErr err = beginReq(req); MojErrCheck(err); MojDbCursor cursor; err = findImpl(query, cursor, NULL, req, OpUpdate); MojErrCheck(err); MojAssert(cursor.txn()); MojUInt32 count = 0; MojUInt32 warns = 0; bool found = false; MojObject prev; for (;;) { // get prev rev from cursor MojDbStorageItem* prevItem = NULL; err = cursor.get(prevItem, found); if (err == MojErrInternalIndexOnFind) { warns++; continue; } MojErrCheck(err); if (!found) break; err = prevItem->toObject(prev, m_kindEngine); MojErrCheck(err); // merge obj into prev MojObject merged; err = mergeInto(merged, props, prev); MojErrCheck(err); // and update the db const MojObject& id = prevItem->id(); err = putObj(id, merged, &prev, prevItem, req, OpUpdate); MojErrCheck(err); ++count; } if (warns > 0) MojLogWarning(s_log, _T("Merge index_warnings: %s; count: %d\n"), query.from().data(), warns); err = cursor.close(); MojErrCheck(err); err = req->end(); MojErrCheck(err); countOut = count; return MojErrNone; }
int main() { std::cout << "Enter the first array (I'll sort it, don't worry ;) ):" << std::endl; std::vector<int> A = readVector(std::cin); std::cout << "Enter the second array:" << std::endl; std::vector<int> B = readVector(std::cin); std::sort(A.begin(), A.end()); std::sort(B.begin(), B.end()); A.resize(A.size() + B.size(), 0); mergeInto(A, B); std::cout << "Merged:" << std::endl; for (int x : A) std::cout << x << ", "; std::cout << "\b\b " << std::endl; return 0; }
bool GetByIdVariant::attemptToMerge(const GetByIdVariant& other) { if (m_alternateBase != other.m_alternateBase) return false; if (m_offset != other.m_offset) return false; if (m_callLinkStatus || other.m_callLinkStatus) return false; if (!areCompatible(m_constantChecks, other.m_constantChecks)) return false; mergeInto(other.m_constantChecks, m_constantChecks); m_structureSet.merge(other.m_structureSet); return true; }
MojErr MojDb::mergeInto(MojObject& dest, const MojObject& obj, const MojObject& prev) { MojLogTrace(s_log); // TODO: support field deletion // TODO: move merge fn out of db MojErr err; MojObject::ConstIterator objIter = obj.begin(); MojObject::ConstIterator prevIter = prev.begin(); while (objIter != obj.end() || prevIter != prev.end()) { // compare keys from two iters int comp; if (objIter == obj.end()) { comp = 1; } else if (prevIter == prev.end()) { comp = -1; } else { comp = objIter.key().compare(prevIter.key()); } // put the appropriate value into dest if (comp > 0) { err = dest.put(prevIter.key(), prevIter.value()); MojErrCheck(err); ++prevIter; } else if (comp < 0) { err = dest.put(objIter.key(), objIter.value()); MojErrCheck(err); ++objIter; } else { MojObject::Type newType = objIter.value().type(); MojObject::Type prevType = prevIter.value().type(); if (newType == MojObject::TypeObject && prevType == MojObject::TypeObject) { MojObject merged(MojObject::TypeObject); err = mergeInto(merged, objIter.value(), prevIter.value()); MojErrCheck(err); err = dest.put(objIter.key(), merged); MojErrCheck(err); } else { err = dest.put(objIter.key(), objIter.value()); MojErrCheck(err); } ++prevIter; ++objIter; } } return MojErrNone; }
MojErr MojDb::putImpl(MojObject& obj, MojUInt32 flags, MojDbReq& req, bool checkSchema) { MojLogTrace(s_log); MojRefCountedPtr<MojDbStorageItem> prevItem; MojObject id; if (obj.get(IdKey, id)) { // get previous revision if we have an id MojErr err = m_objDb->get(id, req.txn(), true, prevItem); MojErrCheck(err); } if (MojFlagGet(flags, FlagIgnoreMissing) && MojFlagGet(flags, FlagMerge)) { if (!prevItem.get()) { MojErr err = obj.putBool(MojDb::IgnoreIdKey, true); // so that we can drop it in output MojErrCheck(err); return MojErrNone; } } MojDbOp op = OpCreate; MojObject* prevPtr = NULL; MojObject prev; if (prevItem.get()) { // deal with prev, if it exists op = OpUpdate; prevPtr = &prev; MojErr err = prevItem->toObject(prev, m_kindEngine); MojErrCheck(err); if (MojFlagGet(flags, FlagMerge)) { // do merge MojObject merged; err = mergeInto(merged, obj, prev); MojErrCheck(err); obj = merged; } else if (!MojFlagGet(flags, FlagForce)) { // if the force flag is not set, throw error if we are updating // an existing, non-deleted object and no rev was specified MojInt64 rev; if (obj.get(RevKey, rev) == false) { bool deleted = false; if (!prev.get(DelKey, deleted) || !deleted) MojErrThrow(MojErrDbRevNotSpecified); } } } // if the new object has a rev and it doesn't match the old rev, don't do the update MojInt64 newRev; if (prevPtr != NULL && obj.get(RevKey, newRev)) { MojInt64 oldRev; MojErr err = prevPtr->getRequired(RevKey, oldRev); MojErrCheck(err); if (!MojFlagGet(flags, FlagForce) && newRev != oldRev) MojErrThrowMsg(MojErrDbRevisionMismatch, _T("db: revision mismatch - expected %lld, got %lld"), oldRev, newRev); } // save it MojErr err = putObj(id, obj, prevPtr, prevItem.get(), req, op, checkSchema); MojErrCheck(err); if (prevItem.get()) { err = prevItem->close(); MojErrCheck(err); } return MojErrNone; }
void QRegExpEngine::addCatTransitions () { mergeInto (&s[2].outs); }
int main(int argc, char ** argv) { int rank, size; MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); if(argc != 11) { if(rank == 0) { printf("ERROR: Incorrect input parameters, to run the code:\nMultiGPUKDE InputFile OutputFile xMin xMax yMin yMax cellSize bandwidth numberOfRegionsInX numberOfRegionsInY\n"); } MPI_Finalize(); exit(1); } float xMin = atof(argv[3]); float xMax = atof(argv[4]); float yMin = atof(argv[5]); float yMax = atof(argv[6]); float cellSize = atof(argv[7]); float bandwidth = atof(argv[8]); int nXRegion = atoi(argv[9]); int nYRegion = atoi(argv[10]); // printf("Size: %d\n", size); if(nXRegion * nYRegion > size) { if(rank == 0) { printf("ERROR: The number of regions %d(x) * %d(y) should be no bigger than the number of processors %d\n", nXRegion, nYRegion, size); } MPI_Finalize(); exit(1); } FILE * file; int allPoints = 0; float * allX; float * allY; int * allBlockID; int * allPointsInB; int * allPointsIndex; int * allRegions; int nXCell = ceil((xMax - xMin)/cellSize); int nYCell = ceil((yMax - yMin)/cellSize); xMax = xMin + nXCell * cellSize; yMax = yMin + nYCell * cellSize; int gridX = ceil((float) nXCell / BLOCKSIZE); int gridY = ceil((float) nYCell / BLOCKSIZE); float blockSizeDis = BLOCKSIZE * cellSize; int blockBandwidth = ceil(bandwidth / blockSizeDis); int dataGridX = gridX + 2 * blockBandwidth; int dataGridY = gridY + 2 * blockBandwidth; int myPoints; float * myX; float * myY; int * myIndexTable; float * myDensity; int myRegion[4]; int myCellXMin, myCellXMax, myCellYMin, myCellYMax; float myXMin, myXMax, myYMin, myYMax; int myXCell, myYCell, myGridX, myGridY, myDataGridX, myDataGridY; struct timeval tBegin; struct timeval tEnd; if(rank == 0) { printf("**********************************************\nMulti-GPU Kernel Density Estimation\nCyberInfrastructure and Geospatial Information\n(CIGI) Labortory(http://www.cigi.illinois.edu)\nDeveloped by Yizhao Gao ([email protected])\n**********************************************\n"); printf("Running on %d (%d * %d) nodes\n", nXRegion * nYRegion, nXRegion, nYRegion); printf("Study area:\n\txMin: %f, xMax: %f, yMin: %f, yMax %f\n", xMin, xMax, yMin, yMax); printf("\tOutput raster size: %d(row) * %d(column)\n", nYCell, nXCell); gettimeofday(&tBegin, NULL); if(NULL == (file = fopen(argv[1], "r"))) { printf("ERROR: Can't open the input file.\n"); //MPI_Abort(MPI_COMM_WORLD, MPI_ERR_FILE); exit(1); } allPoints = getNumPoints(file); printf("Number of input points: %d\n", allPoints); if(NULL == (allX = (float *)malloc(sizeof(float) * allPoints))) { printf("ERROR: Out of memory at line %d in file %s\n", __LINE__, __FILE__); exit(1); } if(NULL == (allY = (float *)malloc(sizeof(float) * allPoints))) { printf("ERROR: Out of memory at line %d in file %s\n", __LINE__, __FILE__); exit(1); } if(NULL == (allBlockID = (int *)malloc(sizeof(int) * allPoints))) { printf("ERROR: Out of memory at line %d in file %s\n", __LINE__, __FILE__); exit(1); } allPointsInB = readFile(file, allX, allY, allBlockID, allPoints, xMin - blockSizeDis, yMax + blockSizeDis, blockSizeDis, dataGridX, dataGridY); fclose(file); gettimeofday(&tEnd, NULL); printf("Time - Input:\t%lfms\n", ((&tEnd)->tv_sec - (&tBegin)->tv_sec) * 1000 + (double)((&tEnd)->tv_usec - (&tBegin)->tv_usec) / 1000); tBegin = tEnd; allPointsIndex = indexPoints(allX, allY, allBlockID, allPoints, dataGridX, dataGridY, allPointsInB); free(allBlockID); gettimeofday(&tEnd, NULL); printf("Time - Spatial index:\t%lfms\n", ((&tEnd)->tv_sec - (&tBegin)->tv_sec) * 1000 + (double)((&tEnd)->tv_usec - (&tBegin)->tv_usec) / 1000); tBegin = tEnd; allRegions = spatialDomainDecom(nXRegion, nYRegion, gridX, gridY, blockBandwidth, allPointsInB); /* for(int i = 0; i < nXRegion * nYRegion; i++) { printf("Region %d:\t%d\t%d\t%d\t%d\n", i, allRegions[4 * i], allRegions[4 * i + 1], allRegions[4 * i + 2], allRegions[4 * i + 3]); } */ for(int i = 1; i < nXRegion * nYRegion; i++) { // myPoints = getNPoints(allPointsInB, dataGridX, dataGridY, blockBandwidth, allRegions + 4 * i); myDataGridX = allRegions[4 * i + 1] - allRegions[4 * i] + 2 * blockBandwidth; myDataGridY = allRegions[4 * i + 3] - allRegions[4 * i + 2] + 2 * blockBandwidth; myPoints = getDataInRegion(allPointsInB, allPointsIndex, allX, allY, dataGridX, dataGridY, blockBandwidth, allRegions + 4 * i, myX, myY, myIndexTable); MPI_Send(&myPoints, 1, MPI_INT, i, 0, MPI_COMM_WORLD); MPI_Send(allRegions + i * 4, 4, MPI_INT, i, 1, MPI_COMM_WORLD); MPI_Send(myX, myPoints, MPI_FLOAT, i, 2, MPI_COMM_WORLD); MPI_Send(myY, myPoints, MPI_FLOAT, i, 3, MPI_COMM_WORLD); MPI_Send(myIndexTable, myDataGridX * myDataGridY, MPI_INT, i, 4, MPI_COMM_WORLD); free(myX); free(myY); free(myIndexTable); } myRegion[0] = allRegions[0]; myRegion[1] = allRegions[1]; myRegion[2] = allRegions[2]; myRegion[3] = allRegions[3]; myDataGridX = myRegion[1] - myRegion[0] + 2 * blockBandwidth; myDataGridY = myRegion[3] - myRegion[2] + 2 * blockBandwidth; myPoints = getDataInRegion(allPointsInB, allPointsIndex, allX, allY, dataGridX, dataGridY, blockBandwidth, allRegions, myX, myY, myIndexTable); //writeGeoTiff("/gpfs_scratch/tsccsj/KDE/allInB.tif", allPointsInB, dataGridX, dataGridY, xMin - blockSizeDis, yMax + blockSizeDis, blockSizeDis); gettimeofday(&tEnd, NULL); printf("Time - spliting data and computation to nodes:\t%lfms\n", ((&tEnd)->tv_sec - (&tBegin)->tv_sec) * 1000 + (double)((&tEnd)->tv_usec - (&tBegin)->tv_usec) / 1000); free(allPointsInB); free(allPointsIndex); free(allX); free(allY); } else if(rank < nXRegion * nYRegion) { MPI_Recv(&myPoints, 1, MPI_INT, 0, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); /* if(myPoints < 0) { MPI_Finalize(); exit(1); } */ MPI_Recv(&myRegion, 4, MPI_INT, 0, 1, MPI_COMM_WORLD, MPI_STATUS_IGNORE); myDataGridX = myRegion[1] - myRegion[0] + 2 * blockBandwidth; myDataGridY = myRegion[3] - myRegion[2] + 2 * blockBandwidth; if(NULL == (myIndexTable = (int *)malloc(sizeof(int) * myDataGridX * myDataGridY))) { printf("ERROR: Out of memory at line %d in file %s\n", __LINE__, __FILE__); exit(1); } if(NULL == (myX = (float *)malloc(sizeof(float) * myPoints))) { printf("ERROR: Out of memory at line %d in file %s\n", __LINE__, __FILE__); exit(1); } if(NULL == (myY = (float *)malloc(sizeof(float) * myPoints))) { printf("ERROR: Out of memory at line %d in file %s\n", __LINE__, __FILE__); exit(1); } MPI_Recv(myX, myPoints, MPI_FLOAT, 0, 2, MPI_COMM_WORLD, MPI_STATUS_IGNORE); MPI_Recv(myY, myPoints, MPI_FLOAT, 0, 3, MPI_COMM_WORLD, MPI_STATUS_IGNORE); MPI_Recv(myIndexTable, myDataGridX * myDataGridY, MPI_INT, 0, 4, MPI_COMM_WORLD, MPI_STATUS_IGNORE); } if(rank < nXRegion * nYRegion) { myCellXMin = myRegion[0] * BLOCKSIZE; myCellXMax = myRegion[1] * BLOCKSIZE; if(myCellXMax > nXCell) { myCellXMax = nXCell; } myCellYMin = myRegion[2] * BLOCKSIZE; myCellYMax = myRegion[3] * BLOCKSIZE; if(myCellYMax > nYCell) { myCellYMax = nYCell; } myXMin = xMin + myCellXMin * cellSize; myXMax = xMin + myCellXMax * cellSize; myYMin = yMax - myCellYMax * cellSize; myYMax = yMax - myCellYMin * cellSize; myXCell = myCellXMax - myCellXMin; myYCell = myCellYMax - myCellYMin; myGridX = myRegion[1] - myRegion[0]; myGridY = myRegion[3] - myRegion[2]; /* if(rank == 0) { printf("Rank %d: x: %f to %f\ty: %f to %f\n", rank, myXMin, myXMax, myYMin, myYMax); for(int i = 0; i < 1000; i ++) { printf("%f\t%f\n", myX[i], myY[i]); } } */ // printf("Region %d\tThe region I get: x:\t%d - \t%d\t y\t%d - \t%d\n", rank, myCellXMin, myCellXMax, myCellYMin, myCellYMax); // printf("Region %d\tThe region I get: x:\t%f - \t%f\t y\t%f - \t%f\n", rank, myXMin, myXMax, myYMin, myYMax); // printf("Rank %d: x: %f to %f\ty: %f to %f\n", rank, myXMin, myXMax, myYMin, myYMax); gettimeofday(&tBegin, NULL); myDensity = performKDE(myX, myY, myPoints, myIndexTable, myGridX, myGridY, blockBandwidth, myXCell, myYCell, myXMin, myYMax, cellSize, bandwidth); gettimeofday(&tEnd, NULL); printf("****************************************\nNode %d:\n\tStudy area:\txMin: %f, xMax: %f, yMin: %f, yMax %f\n\tOutput raster size: %d(row) * %d(column)\n\tNumber of input points: %d\n Time - KDE on this node:\t%lfms\n", rank, myXMin, myXMax, myYMin, myYMax, myYCell, myXCell, myPoints, ((&tEnd)->tv_sec - (&tBegin)->tv_sec) * 1000 + (double)((&tEnd)->tv_usec - (&tBegin)->tv_usec) / 1000); free(myIndexTable); free(myX); free(myY); } if(rank == 0) { float * fullDensity; float * densityBuffer; if(NULL == (fullDensity = (float *)malloc(sizeof(float) * nXCell * nYCell))) { printf("ERROR: Out of memory at line %d in file %s\n", __LINE__, __FILE__); exit(1); } if(NULL == (densityBuffer = (float *)malloc(sizeof(float) * nXCell * nYCell))) { printf("ERROR: Out of memory at line %d in file %s\n", __LINE__, __FILE__); exit(1); } gettimeofday(&tBegin, NULL); mergeInto(fullDensity, nXCell, myDensity, myCellXMin, myCellYMin, myXCell, myYCell); free(myDensity); for(int i = 1; i < nXRegion * nYRegion; i++) { myCellXMin = allRegions[4 * i] * BLOCKSIZE; myCellXMax = allRegions[4 * i + 1] * BLOCKSIZE; if(myCellXMax > nXCell) { myCellXMax = nXCell; } myCellYMin = allRegions[4 * i + 2] * BLOCKSIZE; myCellYMax = allRegions[4 * i + 3] * BLOCKSIZE; if(myCellYMax > nYCell) { myCellYMax = nYCell; } myXCell = myCellXMax - myCellXMin; myYCell = myCellYMax - myCellYMin; // printf("Rank 0 is about to receive %d from Rank %d\n", myXCell * myYCell, i); MPI_Recv(densityBuffer, myXCell * myYCell, MPI_FLOAT, i, 5, MPI_COMM_WORLD, MPI_STATUS_IGNORE); mergeInto(fullDensity, nXCell, densityBuffer, myCellXMin, myCellYMin, myXCell, myYCell); } free(allRegions); free(densityBuffer); gettimeofday(&tEnd, NULL); printf("****************************************\nTime - Merge result:\t%lfms\n", ((&tEnd)->tv_sec - (&tBegin)->tv_sec) * 1000 + (double)((&tEnd)->tv_usec - (&tBegin)->tv_usec) / 1000); tBegin = tEnd; /* for(int i = 0; i < nYCell; i++) { for(int j = 0; j < nXCell; j++) { printf("\t%f", fullDensity[i * nXCell + j]); } printf("\n"); } */ //Output writeGeoTiff(argv[2], fullDensity, nXCell, nYCell, xMin, yMax, cellSize); gettimeofday(&tEnd, NULL); printf("Time - Output GeoTiff:\t%lfms\n", ((&tEnd)->tv_sec - (&tBegin)->tv_sec) * 1000 + (double)((&tEnd)->tv_usec - (&tBegin)->tv_usec) / 1000); free(fullDensity); printf("Finished !\n"); } else if(rank < nXRegion * nYRegion) { // printf("Rank %d is about to send %d to Rank 0\n", rank, myXCell * myYCell); MPI_Send(myDensity, myXCell * myYCell, MPI_FLOAT, 0, 5, MPI_COMM_WORLD); free(myDensity); } MPI_Finalize(); return 0; }