init_fat(){
/* enable everdrive card */
clearFATBuffers();
ed_begin();
put_string("Initialise", 0, 1);
everdrive_error = disk_init();
put_number(everdrive_error, 3, 0, 2);
everdrive_error = getMBR(0);
put_number(everdrive_error, 3, 4, 2);
everdrive_error = getFATVol();
put_number(everdrive_error, 3, 8, 2);
everdrive_error = getFATFS();
put_number(everdrive_error, 3, 12, 2);
/* NOTE: You should normally check that each call to the above 4 functions
has returned correctly before proceeding! */
}
std::unique_ptr<te::sam::rtree::Index<size_t, 8> > terrama2::core::createRTreeFromSeries(const terrama2::core::DataSetSeries& dataSetSeries)
{
auto dataSet = dataSetSeries.syncDataSet;
auto dataSetType = dataSetSeries.teDataSetType;
std::unique_ptr<te::sam::rtree::Index<size_t, 8> > rtree(new te::sam::rtree::Index<size_t, 8>);
std::size_t geomPropertyPos = te::da::GetFirstPropertyPos(dataSet->dataset().get(), te::dt::GEOMETRY_TYPE);
if(geomPropertyPos == std::numeric_limits<std::size_t>::max())
{
QString errMsg(QObject::tr("Could not find a geometry property in the indexed dataset"));
TERRAMA2_LOG_ERROR() << errMsg;
throw terrama2::InvalidArgumentException() << terrama2::ErrorDescription(errMsg);
}
// Creates a rtree with all geometries
for(unsigned int i = 0; i < dataSet->size(); ++i)
{
auto geometry = dataSet->getGeometry(i, geomPropertyPos);
geometry->transform(4326);
rtree->insert(*geometry->getMBR(), i);
}
return rtree;
}
double terrama2::services::analysis::core::grid::zonal::forecast::accum::operatorImpl(terrama2::services::analysis::core::StatisticOperation statisticOperation,
const std::string& dataSeriesName, const std::string& dateDiscardBefore, const std::string& dateDiscardAfter, const size_t band, terrama2::services::analysis::core::Buffer buffer)
{
OperatorCache cache;
terrama2::services::analysis::core::python::readInfoFromDict(cache);
// After the operator lock is released it's not allowed to return any value because it doesn' have the interpreter lock.
// In case an exception is thrown, we need to set this boolean. Once the code left the lock is acquired we should return NAN.
bool exceptionOccurred = false;
auto& contextManager = ContextManager::getInstance();
auto analysis = cache.analysisPtr;
try
{
terrama2::core::verify::analysisMonitoredObject(analysis);
}
catch(const terrama2::core::VerifyException&)
{
contextManager.addError(cache.analysisHashCode, QObject::tr("Use of invalid operator for analysis %1.").arg(analysis->id).toStdString());
return NAN;
}
terrama2::services::analysis::core::MonitoredObjectContextPtr context;
try
{
context = ContextManager::getInstance().getMonitoredObjectContext(cache.analysisHashCode);
}
catch(const terrama2::Exception& e)
{
TERRAMA2_LOG_ERROR() << boost::get_error_info<terrama2::ErrorDescription>(e)->toStdString();
return NAN;
}
try
{
// In case an error has already occurred, there is nothing to be done
if(!context->getErrors().empty())
return NAN;
bool hasData = false;
auto dataManagerPtr = context->getDataManager().lock();
if(!dataManagerPtr)
{
QString errMsg(QObject::tr("Invalid data manager."));
throw terrama2::core::InvalidDataManagerException() << terrama2::ErrorDescription(errMsg);
}
std::shared_ptr<ContextDataSeries> moDsContext = context->getMonitoredObjectContextDataSeries(dataManagerPtr);
if(!moDsContext)
{
QString errMsg(QObject::tr("Could not recover monitored object data series."));
throw InvalidDataSeriesException() << terrama2::ErrorDescription(errMsg);
}
if(moDsContext->series.syncDataSet->size() == 0)
{
QString errMsg(QObject::tr("Could not recover monitored object data series."));
throw InvalidDataSeriesException() << terrama2::ErrorDescription(errMsg);
}
auto moGeom = moDsContext->series.syncDataSet->getGeometry(cache.index, moDsContext->geometryPos);
if(!moGeom.get())
{
QString errMsg(QObject::tr("Could not recover monitored object geometry."));
throw InvalidDataSetException() << terrama2::ErrorDescription(errMsg);
}
auto geomResult = createBuffer(buffer, moGeom);
auto dataSeries = context->findDataSeries(dataSeriesName);
/////////////////////////////////////////////////////////////////
//map of sum of values for each pixel
std::unordered_map<std::pair<int, int>, std::pair<double, int>, boost::hash<std::pair<int, int> > > valuesMap;
auto datasets = dataSeries->datasetList;
for(const auto& dataset : datasets)
{
auto rasterList = context->getRasterList(dataSeries, dataset->id, dateDiscardBefore, dateDiscardAfter);
//sanity check, if no date range only the last raster should be returned
if(dateDiscardBefore.empty() && rasterList.size() > 1)
{
QString errMsg(QObject::tr("Invalid list of raster for dataset: %1").arg(dataset->id));
throw terrama2::InvalidArgumentException() << terrama2::ErrorDescription(errMsg);
}
if(rasterList.empty())
{
QString errMsg(QObject::tr("Invalid raster for dataset: %1").arg(dataset->id));
throw terrama2::InvalidArgumentException() << terrama2::ErrorDescription(errMsg);
}
auto firstRaster = rasterList.front();
//no intersection between the raster and the object geometry
if(!firstRaster->getExtent()->intersects(*geomResult->getMBR()))
continue;
geomResult->transform(firstRaster->getSRID());
prec::appendValues(rasterList, band, geomResult.get() , valuesMap);
if(!valuesMap.empty())
{
hasData = true;
break;
}
}
if(exceptionOccurred)
return NAN;
if(!hasData && statisticOperation != StatisticOperation::COUNT)
{
return NAN;
}
std::vector<double> values;
values.reserve(valuesMap.size());
for(const auto& pair : valuesMap)
values.push_back(pair.second.first);
terrama2::services::analysis::core::calculateStatistics(values, cache);
return terrama2::services::analysis::core::getOperationResult(cache, statisticOperation);
}
catch(const terrama2::Exception& e)
{
context->addError(boost::get_error_info<terrama2::ErrorDescription>(e)->toStdString());
return NAN;
}
catch(const std::exception& e)
{
context->addError(e.what());
return NAN;
}
catch(...)
{
QString errMsg = QObject::tr("An unknown exception occurred.");
context->addError(errMsg.toStdString());
return NAN;
}
}
int main(int argc, char** argv)
{
uint32_t dim = 2;
uint32_t n = 8;
uint32_t N = 100;
/*read static data set*/
vector <Point> P;
P = genPoints(dim, N, 1, 0);
// displayPset(P);
try {
IStorageManager* memfile = StorageManager::createNewMemoryStorageManager();
StorageManager::IBuffer* file = StorageManager::createNewRandomEvictionsBuffer(*memfile, 10, false);
id_type indexIdentifier;
ISpatialIndex* tree = RTree::createNewRTree(*file, 0.7, CAPACITY, CAPACITY, dim, SpatialIndex::RTree::RV_RSTAR, indexIdentifier);
id_type id = 0;
for(uint32_t i = 0; i < N; ++i)
{
std::ostringstream os;
os << P[i];
std::string data = os.str();
tree->insertData(data.size() + 1, reinterpret_cast<const byte*>(data.c_str()), P[i], id);
id++;
}
for(uint32_t loop = 1; loop <= LOOPNUM; ++loop)
{
cout << "/**************** BEGIN " << loop << " ***************/" << endl;
/*generate query set*/
vector <Point> Q;
Q = genPoints(dim, n, 1, loop);
/*double pdata1[] = {0, 0};
double pdata2[] = {0, 1};
double pdata3[] = {1, 1};
Point q1(pdata1, dim), q2(pdata2, dim), q3(pdata3, dim);
Q.push_back(q1);
Q.push_back(q2);
Q.push_back(q3);
displayPset(Q); */
/*************** BEGIN BF MBM method ******************/
/* MBM method for finding ANN of Q */
CATCH mbmcost;
mbmcost.catch_time();
Region M = getMBR(Q, dim, n);
MyQueryStrategy qs(M, Q, FUN);
tree->queryStrategy(qs);
mbmcost.catch_time();
cout << "MBM: cpu cost is " << mbmcost.get_cost(2) << " millisecond(s)" << endl;
cout << "MBM: best_dist is " << qs.best_dist << endl;
cout << "MBM: best_NN is ";
displayCoordinates(qs.best_NN);
cout << "MBM: leafIO = " << qs.mbm_leafIO << "; indexIO = " << qs.mbm_indexIO << endl << endl;
/*************** END BF MBM method *******************/
cout << "/**************** END " << loop << " ****************/" << endl << endl;
} // end loop
delete tree;
delete file;
delete memfile;
}
catch(Tools::Exception& e)
{
cerr << "*********ERROR**********" << endl;
std::string s = e.what();
cerr << s << endl;
return -1;
}
catch(...)
{
cerr << "**********ERROR********" << endl;
return -1;
}
return 1;
}
int main(int argc, char** argv)
{
if (argc != 4)
{
cerr << "Usage: " << argv[0] << " dim n area." << endl;
return -1;
}
int dim = atol(argv[1]);
int n = atol(argv[2]);
double area = atof(argv[3]);
if(dim <= 0)
{
cerr << "Dimension should be larger than 0." << endl;
return -1;
}
if(n <= 0)
{
cerr << "The number of query points should be larger than 0." << endl;
return -1;
}
if(area <= 0 || area > 1)
{
cerr << "the area of query points should be in (0, 1]." << endl;
return -1;
}
/*read static data set*/
vector <Point> P;
ifstream in("../data.ini");
if(!in)
{
cerr << "Cannot open file data.ini.\n";
return -1;
}
P = readPoints(in, dim);
uint32_t N = P.size();
try {
IStorageManager* memfile = StorageManager::createNewMemoryStorageManager();
StorageManager::IBuffer* file = StorageManager::createNewRandomEvictionsBuffer(*memfile, 10, false);
id_type indexIdentifier;
ISpatialIndex* tree = RTree::createNewRTree(*file, 0.7, CAPACITY, CAPACITY, dim, SpatialIndex::RTree::RV_RSTAR, indexIdentifier);
id_type id = 0;
for(uint32_t i = 0; i < N; ++i)
{
std::ostringstream os;
os << P[i];
std::string data = os.str();
tree->insertData(data.size() + 1, reinterpret_cast<const byte*>(data.c_str()), P[i], id);
id++;
}
/*std::cerr << "Operations: " << N << std::endl;
std::cerr << *tree;
std::cerr << "Buffer hits: " << file->getHits() << std::endl;
std::cerr << "Index ID: " << indexIdentifier << std::endl;
bool ret = tree->isIndexValid();
if (ret == false) std::cerr << "ERROR: Structure is invalid!" << std::endl;
else std::cerr << "The stucture seems O.K." << std::endl; */
for(uint32_t loop = 1; loop <= LOOPNUM; ++loop)
{
cout << "/**************** BEGIN " << loop << " ***************/" << endl;
/*generate query set*/
vector <Point> Q;
//Q = genPoints(dim, n, area, loop);
stringstream ss;
ss << "../query/n" << n << "M" << area << "/loop" << loop;
cout << ss.str().c_str() << endl;
ifstream qin(ss.str().c_str());
if(!qin)
{
cerr << "Cannot open query file";
return -1;
}
Q = readPoints(qin, dim);
/*************** BEGIN MQM method ******************/
MQM(tree, Q, n, FUN); // MQM method for finding ANN of Q
/*************** END MQM method *******************/
/*************** BEGIN ADM method ******************/
CATCH cost1;
cost1.catch_time();
vector <uint32_t> nnIDs = nearestNeighborSet(tree, Q, n); // find the NN for every qi in Q as qi'
vector <Point> newQ;
for(uint32_t i = 0; i < n; ++i)
{
newQ.push_back(P[nnIDs[i]]);
}
cost1.catch_time();
cout << "proposal method: cpu cost for finding NNs of Q as Q' is " << cost1.get_cost(2) << " millisecond(s)" << endl;
/***** read dist-matrix index for Q' ********/
uint32_t maxK = P.size() / RATIO; // the length of dist-matrix index
uint32_t * dmindex[n];
for(uint32_t i = 0; i < n; ++i)
{ // read the dist-matrix index of qi'
dmindex[i] = readDMIndex(nnIDs[i], maxK);
if (dmindex[i] == NULL)
{
cerr << "error for loading Dist-Matrix Index." << endl;
return -1;
}
}
double minadist = 0;
/* ADM method for finding approxiamte ANN */
Point adm_ANN = ADM(newQ, n, P, N, dmindex, maxK, FUN, minadist, ERROR_RATE);
cout << "ADM: best_dist is " << getAdist(adm_ANN, Q, n, FUN) << endl << endl;
/*************** END ADM method *******************/
/*************** BEGIN approxiamte vp-ANN method ******************/
/* approximate vp-ANN method for finding ANN of Q'*/
CATCH cost2;
cost2.catch_time();
Point centroid = findCentroid(Q, dim, n);
minadist = getAdist(centroid, Q, n, FUN);
uint32_t vpID = nearestNeighbor(tree, centroid);
uint32_t best_id_Q = epsilonANN(Q, n, P, N, vpID, dmindex, maxK, FUN);
cost2.catch_time();
cout << "approxiamte vp-ANN method: cpu cost is " << cost2.get_cost(2) << " millisecond(s)" << endl;
cout << "approximate vp-ANN method: best_dist is " << getAdist(P[best_id_Q], Q, n, FUN) << endl;
cout << "approxiamte vp-ANN method: best_NN is ";
displayCoordinates(P[best_id_Q]);
cout << endl;
/*************** END approxiamte vp-ANN method *******************/
/*************** BEGIN BF MBM method ******************/
/* MBM method for finding ANN of Q */
CATCH mbmcost;
mbmcost.catch_time();
Region M = getMBR(Q, dim, n);
MyQueryStrategy qs = MyQueryStrategy(M, Q, FUN);
tree->queryStrategy(qs);
mbmcost.catch_time();
cout << "MBM: cpu cost is " << mbmcost.get_cost(2) << " millisecond(s)" << endl;
cout << "MBM: best_dist is " << qs.best_dist << endl;
cout << "MBM: best_NN is ";
displayCoordinates(qs.best_NN);
cout << "MBM: leafIO = " << qs.mbm_leafIO << "; indexIO = " << qs.mbm_indexIO << endl << endl;
/*************** END BF MBM method *******************/
/*************** BEGIN brute method ******************/
/* brute method for finding ANN of Q*/
CATCH brute_cost;
brute_cost.catch_time();
uint32_t ANNid = brute_ANN(Q, n, P, N, FUN);
brute_cost.catch_time();
cout << "brute method: cpu cost is " << brute_cost.get_cost(2) << " millisecond(s)" << endl;
double adist = getAdist(P[ANNid], Q, n, FUN);
cout << "brute method: best_dist is " << adist << endl;
cout << "brute method: best_NN is ";
displayCoordinates(P[ANNid]);
/*************** END brute method *******************/
cout << "/**************** END " << loop << " ****************/" << endl << endl;
} // end loop
delete tree;
delete file;
delete memfile;
}
catch(Tools::Exception& e)
{
cerr << "*********ERROR**********" << endl;
std::string s = e.what();
cerr << s << endl;
return -1;
}
catch(...)
{
cerr << "**********ERROR********" << endl;
return -1;
}
return 1;
}
void terrama2::services::analysis::core::Context::addDataSeries(const AnalysisHashCode analysisHashCode,
terrama2::core::DataSeriesPtr dataSeries,
std::shared_ptr<te::gm::Geometry> envelope,
const std::string& dateFilter, bool createSpatialIndex)
{
std::lock_guard<std::recursive_mutex> lock(mutex_);
bool needToAdd = false;
for(auto dataset : dataSeries->datasetList)
{
if(!exists(analysisHashCode, dataset->id, dateFilter))
{
needToAdd = true;
break;
}
}
auto analysis = Context::getInstance().getAnalysis(analysisHashCode);
if(!needToAdd)
return;
time_t ts = 0;
struct tm t;
char buf[16];
::localtime_r(&ts, &t);
::strftime(buf, sizeof(buf), "%Z", &t);
boost::local_time::time_zone_ptr zone(new boost::local_time::posix_time_zone(buf));
boost::local_time::local_date_time ldt = boost::local_time::local_microsec_clock::local_time(zone);
auto dataManagerPtr = dataManager_.lock();
if(!dataManagerPtr)
{
QString errMsg(QObject::tr("Invalid data manager."));
throw terrama2::core::InvalidDataManagerException() << terrama2::ErrorDescription(errMsg);
}
auto dataProvider = dataManagerPtr->findDataProvider(dataSeries->dataProviderId);
terrama2::core::Filter filter;
filter.discardAfter = analysisStartTime_[analysisHashCode];
if(!dateFilter.empty())
{
double seconds = terrama2::core::TimeUtils::convertTimeString(dateFilter, "SECOND", "h");
ldt -= boost::posix_time::seconds(seconds);
std::unique_ptr<te::dt::TimeInstantTZ> titz(new te::dt::TimeInstantTZ(ldt));
filter.discardBefore = std::move(titz);
}
//accessing data
terrama2::core::DataAccessorPtr accessor = terrama2::core::DataAccessorFactory::getInstance().make(dataProvider, dataSeries, filter);
std::unordered_map<terrama2::core::DataSetPtr, terrama2::core::DataSetSeries > seriesMap = accessor->getSeries(filter);
if(seriesMap.empty())
{
QString errMsg(QObject::tr("The data series %1 does not contain data").arg(dataSeries->id));
throw EmptyDataSeriesException() << terrama2::ErrorDescription(errMsg);
}
for(auto mapItem : seriesMap)
{
auto series = mapItem.second;
std::shared_ptr<ContextDataSeries> dataSeriesContext(new ContextDataSeries);
std::size_t geomPropertyPosition = te::da::GetFirstPropertyPos(series.syncDataSet->dataset().get(), te::dt::GEOMETRY_TYPE);
dataSeriesContext->series = series;
dataSeriesContext->geometryPos = geomPropertyPosition;
if(createSpatialIndex)
{
int size = series.syncDataSet->size();
for(std::size_t i = 0; i < size; ++i)
{
auto geom = series.syncDataSet->getGeometry(i, geomPropertyPosition);
dataSeriesContext->rtree.insert(*geom->getMBR(), i);
}
}
ContextKey key;
key.datasetId_ = series.dataSet->id;
key.analysisHashCode_ = analysisHashCode;
key.dateFilter_ = dateFilter;
datasetMap_[key] = dataSeriesContext;
}
}
