void addInput(std::shared_ptr<terrama2::services::collector::core::DataManager> dataManager)
{
///////////////////////////////////////////////
// input
// DataProvider information
terrama2::core::DataProvider* dataProvider = new terrama2::core::DataProvider();
terrama2::core::DataProviderPtr dataProviderPtr(dataProvider);
dataProvider->uri = "file://"+TERRAMA2_DATA_DIR+"/fire_system";
dataProvider->intent = terrama2::core::DataProviderIntent::COLLECTOR_INTENT;
dataProvider->active = true;
dataProvider->id = 1;
dataProvider->name = "DataProvider queimadas local";
dataProvider->dataProviderType = "FILE";
dataManager->add(dataProviderPtr);
auto& semanticsManager = terrama2::core::SemanticsManager::getInstance();
// DataSeries information
terrama2::core::DataSeries* dataSeries = new terrama2::core::DataSeries();
terrama2::core::DataSeriesPtr dataSeriesPtr(dataSeries);
dataSeries->id = 1;
dataSeries->name = "DataProvider queimadas local";
dataSeries->semantics = semanticsManager.getSemantics("OCCURRENCE-wfp");
dataSeries->dataProviderId = dataProviderPtr->id;
terrama2::core::DataSetOccurrence* dataSet = new terrama2::core::DataSetOccurrence();
dataSet->id = 1;
dataSet->active = true;
dataSet->format.emplace("mask", "exporta_%YYYY%MM%DD_%hh%mm.csv");
dataSeries->datasetList.emplace_back(dataSet);
dataManager->add(dataSeriesPtr);
}
int main(int argc, char* argv[])
{
terrama2::core::TerraMA2Init terramaRaii("example", 0);
{
//DataProvider information
terrama2::core::DataProvider* dataProvider = new terrama2::core::DataProvider();
terrama2::core::DataProviderPtr dataProviderPtr(dataProvider);
dataProvider->uri = "file://";
dataProvider->uri += TERRAMA2_DATA_DIR;
dataProvider->uri += "/geotiff";
dataProvider->intent = terrama2::core::DataProviderIntent::COLLECTOR_INTENT;
dataProvider->dataProviderType = "FILE";
dataProvider->active = true;
auto& semanticsManager = terrama2::core::SemanticsManager::getInstance();
//DataSeries information
terrama2::core::DataSeries* dataSeries = new terrama2::core::DataSeries();
terrama2::core::DataSeriesPtr dataSeriesPtr(dataSeries);
dataSeries->semantics = semanticsManager.getSemantics("GRID-geotiff");
terrama2::core::DataSetGrid* dataSet = new terrama2::core::DataSetGrid();
dataSet->active = true;
dataSet->format.emplace("mask", "L5219076_07620040908_r3g2b1.tif");
dataSeries->datasetList.emplace_back(dataSet);
//empty filter
terrama2::core::Filter filter;
//accessing data
terrama2::core::DataAccessorGeoTiff accessor(dataProviderPtr, dataSeriesPtr);
auto remover = std::make_shared<terrama2::core::FileRemover>();
terrama2::core::GridSeriesPtr gridSeries = accessor.getGridSeries(filter, remover);
assert(gridSeries->gridMap().size() == 1);
auto raster = gridSeries->gridMap().begin()->second;
assert(raster->getSRID() != 0);
std::cout << "SRID: " << raster->getSRID() << std::endl;
std::string output = TERRAMA2_DATA_DIR;
output+="/grid_output.tif";
te::rp::Copy2DiskRaster(*raster, output);
std::cout << "Tiff file copied to " << output << std::endl;
}
return 0;
}
terrama2::core::DataProviderPtr outputDataProvider()
{
terrama2::core::DataProvider* dataProvider = new terrama2::core::DataProvider();
terrama2::core::DataProviderPtr dataProviderPtr(dataProvider);
dataProvider->uri = "file://" + TERRAMA2_DATA_DIR + "/";
dataProvider->intent = terrama2::core::DataProviderIntent::COLLECTOR_INTENT;
dataProvider->active = true;
dataProvider->id = 1;
dataProvider->name = "DataProvider queimadas local";
dataProvider->dataProviderType = "FILE";
return dataProviderPtr;
}
terrama2::core::DataProviderPtr additionalDataProvider()
{
//DataProvider information
terrama2::core::DataProvider* dataProvider = new terrama2::core::DataProvider();
terrama2::core::DataProviderPtr dataProviderPtr(dataProvider);
dataProvider->name = "Shapefiles provider";
dataProvider->uri = "file://"+ TERRAMA2_DATA_DIR+"/shapefile";
dataProvider->intent = terrama2::core::DataProviderIntent::PROCESS_INTENT;
dataProvider->dataProviderType = "FILE";
dataProvider->active = true;
dataProvider->id = 2;
return dataProviderPtr;
}
int main(int argc, char* argv[])
{
terrama2::core::TerraMA2Init terramaRaii("example", 0);
{
//DataProvider information
terrama2::core::DataProvider* dataProvider = new terrama2::core::DataProvider();
terrama2::core::DataProviderPtr dataProviderPtr(dataProvider);
dataProvider->uri = "file://";
dataProvider->uri+=TERRAMA2_DATA_DIR;
dataProvider->uri+="/PCD_serrmar_INPE";
dataProvider->intent = terrama2::core::DataProviderIntent::COLLECTOR_INTENT;
dataProvider->dataProviderType = "FILE";
dataProvider->active = true;
//DataSeries information
terrama2::core::DataSeries* dataSeries = new terrama2::core::DataSeries();
terrama2::core::DataSeriesPtr dataSeriesPtr(dataSeries);
auto& semanticsManager = terrama2::core::SemanticsManager::getInstance();
dataSeries->semantics = semanticsManager.getSemantics("DCP-inpe");
terrama2::core::DataSetDcp* dataSet = new terrama2::core::DataSetDcp();
dataSet->active = true;
dataSet->format.emplace("mask", "30885.txt");
dataSet->format.emplace("timezone", "+00");
dataSeries->datasetList.emplace_back(dataSet);
//empty filter
terrama2::core::Filter filter;
filter.lastValue = true;
//accessing data
terrama2::core::DataAccessorDcpInpe accessor(dataProviderPtr, dataSeriesPtr);
auto remover = std::make_shared<terrama2::core::FileRemover>();
terrama2::core::DcpSeriesPtr dcpSeries = accessor.getDcpSeries(filter, remover);
assert(dcpSeries->dcpSeriesMap().size() == 1);
auto series = dcpSeries->getSeries().begin()->second;
auto teDataSet = series.syncDataSet->dataset();
std::cout << "dataset size: " << teDataSet->size() << std::endl;
}
return 0;
}
terrama2::core::DataProviderPtr buildInputProvider()
{
terrama2::core::DataProvider* dataProvider = new terrama2::core::DataProvider();
terrama2::core::DataProviderPtr dataProviderPtr(dataProvider);
QString uri = QString("file://%1/PCD_serrmar_INPE").arg(QString::fromStdString(TERRAMA2_DATA_DIR));
dataProvider->id = 1;
dataProvider->projectId = 1;
dataProvider->name = "Provider";
dataProvider->description = "Testing provider";
dataProvider->intent = terrama2::core::DataProviderIntent::COLLECTOR_INTENT;
dataProvider->uri = uri.toStdString();
dataProvider->active = true;
dataProvider->dataProviderType = "FILE";
return dataProviderPtr;
}
void addInput(std::shared_ptr<terrama2::services::collector::core::DataManager> dataManager)
{
///////////////////////////////////////////////
// input
// DataProvider information
terrama2::core::DataProvider* dataProvider = new terrama2::core::DataProvider();
terrama2::core::DataProviderPtr dataProviderPtr(dataProvider);
dataProvider->uri = "ftp://*****:*****@server-ftpdsa.cptec.inpe.br";
dataProvider->intent = terrama2::core::DataProviderIntent::COLLECTOR_INTENT;
dataProvider->active = true;
dataProvider->id = 1;
dataProvider->name = "Dados DSA curso";
dataProvider->dataProviderType = "FTP";
dataManager->add(dataProviderPtr);
auto& semanticsManager = terrama2::core::SemanticsManager::getInstance();
// DataSeries information
terrama2::core::DataSeries* dataSeries = new terrama2::core::DataSeries();
terrama2::core::DataSeriesPtr dataSeriesPtr(dataSeries);
dataSeries->id = 1;
dataSeries->name = "Hidroestimador_input";
dataSeries->semantics = semanticsManager.getSemantics("GRID-grads");
dataSeries->dataProviderId = dataProviderPtr->id;
terrama2::core::DataSetOccurrence* dataSet = new terrama2::core::DataSetOccurrence();
dataSet->id = 1;
dataSet->active = true;
dataSet->format.emplace("ctl_filename", "racc.ctl");
dataSet->format.emplace("folder", "hidro");
dataSet->format.emplace("srid", "4326");
dataSet->format.emplace("timezone", "UTC+00");
dataSet->format.emplace("data_type", "INT16");
dataSet->format.emplace("number_of_bands", "1");
dataSet->format.emplace("bytes_before", "0");
dataSet->format.emplace("bytes_after", "0");
dataSeries->datasetList.emplace_back(dataSet);
dataManager->add(dataSeriesPtr);
}
void addStaticDataSeries(std::shared_ptr<terrama2::services::collector::core::DataManager> dataManager)
{
terrama2::core::DataProvider* dataProvider = new terrama2::core::DataProvider();
std::shared_ptr<const terrama2::core::DataProvider> dataProviderPtr(dataProvider);
dataProvider->name = "Provider";
dataProvider->uri += TERRAMA2_DATA_DIR;
dataProvider->uri += "/shapefile";
dataProvider->intent = terrama2::core::DataProviderIntent::COLLECTOR_INTENT;
dataProvider->dataProviderType = "FILE";
dataProvider->active = true;
dataProvider->id = 3;
dataManager->add(dataProviderPtr);
auto& semanticsManager = terrama2::core::SemanticsManager::getInstance();
terrama2::core::DataSeries* dataSeries = new terrama2::core::DataSeries();
terrama2::core::DataSeriesPtr dataSeriesPtr(dataSeries);
dataSeries->dataProviderId = dataProvider->id;
dataSeries->semantics = semanticsManager.getSemantics("STATIC_DATA-ogr");
dataSeries->semantics.dataSeriesType = terrama2::core::DataSeriesType::STATIC;
dataSeries->name = "States Brazil";
dataSeries->id = 3;
dataSeries->dataProviderId = dataProvider->id;
//DataSet information
terrama2::core::DataSet* dataSet = new terrama2::core::DataSet;
terrama2::core::DataSetPtr dataSetPtr(dataSet);
dataSet->active = true;
dataSet->format.emplace("mask", "estados_2010.shp");
dataSet->format.emplace("srid", "4326");
dataSet->format.emplace("identifier", "nome");
dataSet->id = 1;
dataSet->dataSeriesId = dataSeries->id;
dataSeries->datasetList.push_back(dataSetPtr);
dataManager->add(dataSeriesPtr);
}
terrama2::core::DataProviderPtr inputDataProvider()
{
QUrl uri;
uri.setScheme("postgis");
uri.setHost(QString::fromStdString(TERRAMA2_DATABASE_HOST));
uri.setPort(std::stoi(TERRAMA2_DATABASE_PORT));
uri.setUserName(QString::fromStdString(TERRAMA2_DATABASE_USERNAME));
uri.setPassword(QString::fromStdString(TERRAMA2_DATABASE_PASSWORD));
uri.setPath(QString::fromStdString("/"+TERRAMA2_DATABASE_DBNAME));
//DataProvider information
terrama2::core::DataProvider* dataProvider = new terrama2::core::DataProvider();
terrama2::core::DataProviderPtr dataProviderPtr(dataProvider);
dataProvider->name = "PostGIS provider";
dataProvider->uri = uri.url().toStdString();
dataProvider->intent = terrama2::core::DataProviderIntent::COLLECTOR_INTENT;
dataProvider->dataProviderType = "POSTGIS";
dataProvider->active = true;
dataProvider->id = 1;
return dataProviderPtr;
}
void addGridSeries(std::shared_ptr<terrama2::services::collector::core::DataManager> dataManager)
{
//DataProvider information
terrama2::core::DataProvider* dataProvider = new terrama2::core::DataProvider();
terrama2::core::DataProviderPtr dataProviderPtr(dataProvider);
dataProvider->uri = "file://"+TERRAMA2_DATA_DIR;
dataProvider->intent = terrama2::core::DataProviderIntent::COLLECTOR_INTENT;
dataProvider->dataProviderType = "FILE";
dataProvider->active = true;
dataProvider->id = 3;
dataProvider->name = "Local Geotiff";
dataManager->add(dataProviderPtr);
auto& semanticsManager = terrama2::core::SemanticsManager::getInstance();
//DataSeries information
terrama2::core::DataSeries* dataSeries = new terrama2::core::DataSeries();
terrama2::core::DataSeriesPtr dataSeriesPtr(dataSeries);
dataSeries->semantics = semanticsManager.getSemantics("GRID-static_geotiff");
dataSeries->name = "geotiff";
dataSeries->id = 3;
dataSeries->dataProviderId = 3;
terrama2::core::DataSetGrid* dataSet = new terrama2::core::DataSetGrid();
dataSet->id = 3;
dataSet->active = true;
dataSet->format.emplace("mask", "cbers2b_rgb342_crop.tif");
dataSet->format.emplace("folder", "geotiff");
dataSeries->datasetList.emplace_back(dataSet);
dataManager->add(dataSeriesPtr);
}
int main(int argc, char* argv[])
{
terrama2::core::initializeTerraMA();
{
QUrl uri;
uri.setScheme("postgis");
uri.setHost(QString::fromStdString(TERRAMA2_DATABASE_HOST));
uri.setPort(std::stoi(TERRAMA2_DATABASE_PORT));
uri.setUserName(QString::fromStdString(TERRAMA2_DATABASE_USERNAME));
uri.setPassword(QString::fromStdString(TERRAMA2_DATABASE_PASSWORD));
uri.setPath(QString::fromStdString("/"+TERRAMA2_DATABASE_DBNAME));
//DataProvider information
terrama2::core::DataProvider* dataProvider = new terrama2::core::DataProvider();
terrama2::core::DataProviderPtr dataProviderPtr(dataProvider);
dataProvider->uri = uri.url().toStdString();
dataProvider->intent = terrama2::core::DataProviderIntent::COLLECTOR_INTENT;
dataProvider->dataProviderType = "POSTGIS";
dataProvider->active = true;
//DataSeries information
terrama2::core::DataSeries* dataSeries = new terrama2::core::DataSeries();
terrama2::core::DataSeriesPtr dataSeriesPtr(dataSeries);
auto& semanticsManager = terrama2::core::SemanticsManager::getInstance();
dataSeries->semantics = semanticsManager.getSemantics("OCCURRENCE-postgis");
//DataSet information
terrama2::core::DataSetOccurrence* dataSet = new terrama2::core::DataSetOccurrence();
dataSet->active = true;
dataSet->format.emplace("table_name", "fires");
dataSet->format.emplace("timestamp_property", "data_pas");
dataSet->format.emplace("geometry_property", "geom");
dataSeries->datasetList.emplace_back(dataSet);
//accessing data
terrama2::core::DataAccessorOccurrencePostGis accessor(dataProviderPtr, dataSeriesPtr);
//empty filter
terrama2::core::Filter filter;
boost::local_time::time_zone_ptr zone(new boost::local_time::posix_time_zone("+00"));
filter.lastValue = true;
std::string boundingBoxWkt = "POLYGON((-51.11 -17.74, -41.11 -17.74, -41.11 -20.83, -51.11 -20.83, -51.11 -17.74))";
te::gm::Geometry* geometry = te::gm::WKTReader::read(boundingBoxWkt.c_str());
geometry->setSRID(4326);
filter.region = std::shared_ptr<te::gm::Geometry>(geometry);
terrama2::core::OccurrenceSeriesPtr occurrenceSeries = accessor.getOccurrenceSeries(filter);
assert(occurrenceSeries->occurrencesMap().size() == 1);
auto teDataSet = (*occurrenceSeries->occurrencesMap().begin()).second.syncDataSet->dataset();
//Print column names and types (DateTime/Double)
int dateColumn = -1;
int geomColumn = -1;
std::string names, types;
for(int i = 0; i < teDataSet->getNumProperties(); ++i)
{
std::string name = teDataSet->getPropertyName(i);
names+= name + "\t";
if(name == "data_pas")
{
types+= "DataTime\t";
dateColumn = i;
}
else if(name == "geom")
{
types+= "Geometry\t";
geomColumn = i;
}
else
types+= "Int\t";
}
std::cout << names << std::endl;
std::cout << types << std::endl;
//Print values
teDataSet->moveBeforeFirst();
while(teDataSet->moveNext())
{
for(int i = 0; i < teDataSet->getNumProperties(); ++i)
{
if(teDataSet->isNull(i))
{
std::cout << "NULL";
continue;
}
if(i == dateColumn)
{
std::shared_ptr<te::dt::DateTime> dateTime = teDataSet->getDateTime(i);
std::cout << dateTime->toString();
}
else if(i == geomColumn)
{
std::cout << "<<Geometry>>";
}
else
{
std::cout << teDataSet->getInt32(i);
}
std::cout << "\t";
}
std::cout << std::endl;
}
std::cout << "\ndataset size: " << teDataSet->size() << std::endl;
}
terrama2::core::finalizeTerraMA();
return 0;
}
int main(int argc, char* argv[])
{
terrama2::core::TerraMA2Init terramaRaii("example", 0);
{
//DataProvider information
terrama2::core::DataProvider* dataProvider = new terrama2::core::DataProvider();
terrama2::core::DataProviderPtr dataProviderPtr(dataProvider);
dataProvider->uri = "file://";
dataProvider->uri += TERRAMA2_DATA_DIR;
dataProvider->uri += "/fire_system";
dataProvider->intent = terrama2::core::DataProviderIntent::COLLECTOR_INTENT;
dataProvider->dataProviderType = "FILE";
dataProvider->active = true;
//DataSeries information
terrama2::core::DataSeries* dataSeries = new terrama2::core::DataSeries();
terrama2::core::DataSeriesPtr dataSeriesPtr(dataSeries);
auto& semanticsManager = terrama2::core::SemanticsManager::getInstance();
dataSeries->semantics = semanticsManager.getSemantics("OCCURRENCE-wfp");
terrama2::core::DataSetOccurrence* dataSet =new terrama2::core::DataSetOccurrence();
dataSet->active = true;
dataSet->format.emplace("mask", "exporta_%YYYY%MM%DD_%hh%mm.csv");
dataSet->format.emplace("srid", "4326");
dataSeries->datasetList.emplace_back(dataSet);
//empty filter
terrama2::core::Filter filter;
//accessing data
terrama2::core::DataAccessorOccurrenceWfp accessor(dataProviderPtr, dataSeriesPtr);
auto remover = std::make_shared<terrama2::core::FileRemover>();
terrama2::core::OccurrenceSeriesPtr occurrenceSeries = accessor.getOccurrenceSeries(filter, remover);
std::cout << "Last data timestamp: " << accessor.lastDateTime()->toString() << std::endl;
assert(occurrenceSeries->occurrencesMap().size() == 1);
std::shared_ptr<te::da::DataSet> teDataSet = (*occurrenceSeries->occurrencesMap().begin()).second.syncDataSet->dataset();
//Print column names and types (DateTime/Double)
int dateColumn = -1;
int geomColumn = -1;
std::string names, types;
for(int i = 0; i < teDataSet->getNumProperties(); ++i)
{
std::string name = teDataSet->getPropertyName(i);
names+= name + "\t";
if(name == "data_pas")
{
types+= "DataTime\t";
dateColumn = i;
}
else if(name == "position")
{
types+= "Geometry\t";
geomColumn = i;
}
else
types+= "String\t";
}
std::cout << names << std::endl;
std::cout << types << std::endl;
//Print values
teDataSet->moveBeforeFirst();
while(teDataSet->moveNext())
{
for(int i = 0; i < teDataSet->getNumProperties(); ++i)
{
std::cout << teDataSet->getAsString(i) << "\t";
}
std::cout << std::endl;
}
std::cout << "\nDataSet size: " << teDataSet->size() << std::endl;
}
return 0;
}
void TsDataAccessorFile::testGetFoldersList()
{
// Valid folders
QTemporaryDir dir(QDir::tempPath() + QString::fromStdString("/2016"));
QTemporaryDir subdir1(dir.path() + QString::fromStdString("/10"));
QTemporaryDir subdir2(dir.path() + QString::fromStdString("/11"));
QTemporaryDir subdir3(dir.path() + QString::fromStdString("/12"));
QTemporaryDir subdir4(subdir1.path() + QString::fromStdString("/03"));
QTemporaryDir subdir5(subdir3.path() + QString::fromStdString("/11"));
QTemporaryDir subdir6(subdir4.path() + QString::fromStdString("/final"));
QTemporaryDir subdir7(subdir4.path() + QString::fromStdString("/final"));
QTemporaryDir subdir8(subdir5.path() + QString::fromStdString("/final"));
// Invalid folders
QTemporaryDir dir2(QDir::tempPath() + QString::fromStdString("/2016"));
QTemporaryDir subdir9(dir2.path() + QString::fromStdString("/03"));
QTemporaryDir subdir10(subdir9.path() + QString::fromStdString("/aa"));
QTemporaryDir dir3(QDir::tempPath() + QString::fromStdString("/2020"));
QTemporaryDir dir4(QDir::tempPath() + QString::fromStdString("/folder"));
terrama2::core::DataProvider* dataProvider = new terrama2::core::DataProvider();
terrama2::core::DataProviderPtr dataProviderPtr(dataProvider);
terrama2::core::DataSeries* dataSeries = new terrama2::core::DataSeries();
terrama2::core::DataSeriesPtr dataSeriesPtr(dataSeries);
TestDataAccessorFile da(dataProviderPtr, dataSeriesPtr);
std::vector<std::string> fileList;
fileList.push_back(QDir::tempPath().toStdString());
{
auto foldersList = da.getFoldersList(fileList, "/%YYYY*/%MM*/%DD*/final*");
if(foldersList.size() < 3)
QFAIL("Wrong number of folders matched!");
}
{
auto foldersList = da.getFoldersList(fileList, "%YYYY*/%MM*/%DD*/final*");
if(foldersList.size() < 3)
QFAIL("Wrong number of folders matched!");
}
// empty mask
{
auto foldersList = da.getFoldersList(fileList, "");
if(foldersList.size() != 1)
QFAIL("Wrong number of folders matched!");
}
// considered as empty mask
{
auto foldersList = da.getFoldersList(fileList, "/");
if(foldersList.size() != 1)
QFAIL("Wrong number of folders matched!");
}
// considered as empty mask
{
auto foldersList = da.getFoldersList(fileList, "//");
if(foldersList.size() != 1)
QFAIL("Wrong number of folders matched!");
}
// ignore the extras '/'
{
auto foldersList = da.getFoldersList(fileList, "%YYYY*//%MM*/%DD*//final*");
if(foldersList.size() < 3)
QFAIL("Wrong number of folders matched!");
}
// the folders doesn't match
{
auto foldersList = da.getFoldersList(fileList, "%YYYY*/nonexistent/%DD*");
if(!foldersList.empty())
QFAIL("Wrong number of folders matched!");
}
}
void TsDataAccessorFile::testGetFilesList()
{
// Valid folders
QTemporaryDir dir(QDir::tempPath() + QString::fromStdString("/2016"));
QTemporaryDir subdir1(dir.path() + QString::fromStdString("/10"));
QTemporaryDir subdir2(dir.path() + QString::fromStdString("/11"));
QTemporaryDir subdir3(dir.path() + QString::fromStdString("/12"));
QTemporaryDir subdir4(subdir1.path() + QString::fromStdString("/03"));
QTemporaryDir subdir5(subdir3.path() + QString::fromStdString("/11"));
QTemporaryDir subdir6(subdir2.path() + QString::fromStdString("/final"));
QTemporaryDir subdir7(subdir4.path() + QString::fromStdString("/final"));
QTemporaryDir subdir8(subdir5.path() + QString::fromStdString("/final"));
// Invalid folders
QTemporaryDir subdir9(dir.path() + QString::fromStdString("/03"));
QTemporaryDir subdir10(subdir9.path() + QString::fromStdString("/aa"));
QTemporaryDir dir3(QDir::tempPath() + QString::fromStdString("/2020"));
QTemporaryDir dir4(QDir::tempPath() + QString::fromStdString("/folder"));
terrama2::core::DataProvider* dataProvider = new terrama2::core::DataProvider();
terrama2::core::DataProviderPtr dataProviderPtr(dataProvider);
terrama2::core::DataSeries* dataSeries = new terrama2::core::DataSeries();
terrama2::core::DataSeriesPtr dataSeriesPtr(dataSeries);
TestDataAccessorFile da(dataProviderPtr, dataSeriesPtr);
std::vector<std::string> baseDirList;
baseDirList.push_back(QDir::tempPath().toStdString());
QTemporaryFile file1(subdir7.path() + QString::fromStdString("/file_2017_06_01_14:40_XXXXXX.file"));
QTemporaryFile file2(subdir7.path() + QString::fromStdString("/file_2017_06_01_14:41_XXXXXX.file"));
QTemporaryFile file3(subdir8.path() + QString::fromStdString("/file_2017_06_02_12:00_XXXXXX.file"));
file1.open();
file2.open();
file3.open();
{
std::string fileMask = "file_%YYYY_%MM_%DD_%hh:%mm_*.file";
std::string foldermask = "/%YYYY*/%MM*/%DD*/final*/";
terrama2::core::Filter filter;
std::string timezone = "03";
auto remover = std::make_shared<terrama2::core::FileRemover>();
QFileInfoList fileList;
for(auto& baseDir : baseDirList)
fileList.append(da.getFilesList(baseDir, fileMask, foldermask, filter, timezone, remover));
if(fileList.size() < 3)
QFAIL("Wrong number of folders matched!");
}
}
int main(int argc, char* argv[])
{
terrama2::core::initializeTerraMA();
std::cout << "NOT WORKING" << std::endl;
return 1;
{
QUrl uri;
uri.setScheme("postgis");
uri.setHost(QString::fromStdString(TERRAMA2_DATABASE_HOST));
uri.setPort(std::stoi(TERRAMA2_DATABASE_PORT));
uri.setUserName(QString::fromStdString(TERRAMA2_DATABASE_USERNAME));
uri.setPassword(QString::fromStdString(TERRAMA2_DATABASE_PASSWORD));
uri.setPath(QString::fromStdString("/"+TERRAMA2_DATABASE_DBNAME));
//DataProvider information
terrama2::core::DataProvider* dataProvider = new terrama2::core::DataProvider();
terrama2::core::DataProviderPtr dataProviderPtr(dataProvider);
dataProvider->uri = uri.url().toStdString();
dataProvider->intent = terrama2::core::DataProviderIntent::COLLECTOR_INTENT;
dataProvider->dataProviderType = "POSTGIS";
dataProvider->active = true;
//DataSeries information
terrama2::core::DataSeries* dataSeries = new terrama2::core::DataSeries();
terrama2::core::DataSeriesPtr dataSeriesPtr(dataSeries);
auto& semanticsManager = terrama2::core::SemanticsManager::getInstance();
dataSeries->semantics = semanticsManager.getSemantics("DCP-postgis");
//DataSet information
terrama2::core::DataSetDcp* dataSet = new terrama2::core::DataSetDcp();
dataSet->active = true;
dataSet->format.emplace("table_name", "inpe");
dataSet->format.emplace("timestamp_property", "datetime");
dataSeries->datasetList.emplace_back(dataSet);
//accessing data
terrama2::core::DataAccessorDcpPostGIS accessor(dataProviderPtr, dataSeriesPtr);
//empty filter
terrama2::core::Filter filter;
filter.lastValue = true;
terrama2::core::DcpSeriesPtr dcpSeries = accessor.getDcpSeries(filter);
std::cout << "\nLast data timestamp: " << accessor.lastDateTime()->toString() << std::endl;
assert(dcpSeries->dcpSeriesMap().size() == 1);
auto datasetSeries = (*dcpSeries->dcpSeriesMap().begin()).second;
std::shared_ptr<te::da::DataSet> teDataSet = datasetSeries.syncDataSet->dataset();
std::string tableName = dataSet->format["table_name"];
// creates a DataSource to the data and filters the dataset,
// also joins if the DCP comes from separated files
std::shared_ptr<te::da::DataSource> datasource(te::da::DataSourceFactory::make("POSTGIS"));
std::map<std::string, std::string> connInfo {{"PG_HOST", uri.host().toStdString()},
{"PG_PORT", std::to_string(uri.port())},
{"PG_USER", uri.userName().toStdString()},
{"PG_PASSWORD", uri.password().toStdString()},
{"PG_DB_NAME", uri.path().section("/", 1, 1).toStdString()},
{"PG_CONNECT_TIMEOUT", "4"},
{"PG_CLIENT_ENCODING", "UTF-8"}
};
datasource->setConnectionInfo(connInfo);
// RAII for open/closing the datasource
terrama2::core::OpenClose<std::shared_ptr<te::da::DataSource>> openClose(datasource);
// get a transactor to interact to the data source
std::shared_ptr<te::da::DataSourceTransactor> transactor(datasource->getTransactor());
auto primaryKey = datasetSeries.teDataSetType->getPrimaryKey();
assert(primaryKey);
assert(!teDataSet->isEmpty());
auto pkName = primaryKey->getName();
std::string sql("DELETE FROM " +tableName+" WHERE "+pkName+" NOT IN (");
auto pos = datasetSeries.teDataSetType->getPropertyPosition(pkName);
teDataSet->moveBeforeFirst();
while (teDataSet->moveNext())
{
sql+=std::to_string(teDataSet->getInt32(pos))+",";
}
sql.pop_back();
sql+=")";
transactor->execute(sql);
std::cout << "dataset size: " << teDataSet->size() << std::endl;
}
terrama2::core::finalizeTerraMA();
return 0;
}
int main(int argc, char* argv[])
{
terrama2::core::initializeTerraMA();
{
//DataProvider information
terrama2::core::DataProvider* dataProvider = new terrama2::core::DataProvider();
terrama2::core::DataProviderPtr dataProviderPtr(dataProvider);
dataProvider->uri = "file://";
dataProvider->uri += TERRAMA2_DATA_DIR;
dataProvider->intent = terrama2::core::DataProviderIntent::COLLECTOR_INTENT;
dataProvider->dataProviderType = "FILE";
dataProvider->active = true;
auto& semanticsManager = terrama2::core::SemanticsManager::getInstance();
//DataSeries information
terrama2::core::DataSeries* dataSeries = new terrama2::core::DataSeries();
terrama2::core::DataSeriesPtr dataSeriesPtr(dataSeries);
dataSeries->semantics = semanticsManager.getSemantics("OCCURRENCE-wfp");
terrama2::core::DataSetOccurrence* dataSet =new terrama2::core::DataSetOccurrence();
dataSet->active = true;
dataSet->format.emplace("mask", "exporta_yyyyMMdd_hhmm.csv");
dataSet->format.emplace("folder", "fire_system");
dataSeries->datasetList.emplace_back(dataSet);
terrama2::core::Filter filter;
filter.discardBefore = terrama2::core::TimeUtils::stringToTimestamp("2016-05-01 08:29:00UTM+00", "%Y-%m-%d %H:%M:%S%ZP");
filter.discardAfter = terrama2::core::TimeUtils::stringToTimestamp("2016-05-01 08:31:00UTM+00", "%Y-%m-%d %H:%M:%S%ZP");
//accessing data
terrama2::core::DataAccessorOccurrenceWfp accessor(dataProviderPtr, dataSeriesPtr);
terrama2::core::OccurrenceSeriesPtr occurrenceSeries = accessor.getOccurrenceSeries(filter);
std::cout << "Last data timestamp: " << accessor.lastDateTime()->toString() << std::endl;
assert(occurrenceSeries->occurrencesMap().size() == 1);
std::shared_ptr<te::da::DataSet> teDataSet = (*occurrenceSeries->occurrencesMap().begin()).second.syncDataSet->dataset();
//Print column names and types (DateTime/Double)
int dateColumn = -1;
int geomColumn = -1;
std::string names, types;
for(int i = 0; i < teDataSet->getNumProperties(); ++i)
{
std::string name = teDataSet->getPropertyName(i);
names+= name + "\t";
if(name == "data_pas")
{
types+= "DataTime\t";
dateColumn = i;
}
else if(name == "position")
{
types+= "Geometry\t";
geomColumn = i;
}
else
types+= "String\t";
}
std::cout << names << std::endl;
std::cout << types << std::endl;
//Print values
teDataSet->moveBeforeFirst();
while(teDataSet->moveNext())
{
for(int i = 0; i < teDataSet->getNumProperties(); ++i)
{
std::cout << teDataSet->getAsString(i) << "\t";
}
std::cout << std::endl;
}
std::cout << "\nDataSet size: " << teDataSet->size() << std::endl;
}
terrama2::core::finalizeTerraMA();
return 0;
}
int main(int, char**)
{
terrama2::core::TerraMA2Init terramaRaii("example", 0);
terrama2::core::registerFactories();
//DataProvider information
terrama2::core::DataProvider* dataProvider = new terrama2::core::DataProvider();
terrama2::core::DataProviderPtr dataProviderPtr(dataProvider);
dataProvider->uri = "file://";
dataProvider->uri+=TERRAMA2_DATA_DIR;
dataProvider->intent = terrama2::core::DataProviderIntent::COLLECTOR_INTENT;
dataProvider->dataProviderType = "FILE";
dataProvider->active = true;
//DataSeries information
terrama2::core::DataSeries* dataSeries = new terrama2::core::DataSeries();
terrama2::core::DataSeriesPtr dataSeriesPtr(dataSeries);
auto& semanticsManager = terrama2::core::SemanticsManager::getInstance();
dataSeries->semantics = semanticsManager.getSemantics("DCP-generic");
terrama2::core::DataSetDcp* dataSet = new terrama2::core::DataSetDcp();
dataSet->active = true;
dataSet->format.emplace("folder", "/pcd_toa5/CPV/");
dataSet->format.emplace("mask", "CPV_slow_%YYYY_%MM_%DD_%hh%mm.dat");
dataSet->format.emplace("timezone", "+00");
dataSet->format.emplace("srid", "4326");
dataSet->format.emplace("header_size", "4");
dataSet->format.emplace("properties_names_line", "2");
dataSet->format.emplace("convert_all", "true");
dataSet->format.emplace("default_type", "TEXT");
QJsonArray fields;
{
QJsonObject obj;
obj.insert("property_name", QString("TIMESTAMP"));
obj.insert("alias", QString("datetime"));
obj.insert("type", QString("DATETIME"));
obj.insert("format", QString("%YYYY-%MM-%DD %hh:%mm:%ss"));
fields.push_back(obj);
}
{
QJsonObject obj;
obj.insert("property_name", QString("RECORD"));
obj.insert("alias", QString("record"));
obj.insert("type", QString("INTEGER"));
fields.push_back(obj);
}
{
QJsonObject obj;
obj.insert("property_name", QString("E_field_Avg"));
obj.insert("alias", QString("efield"));
obj.insert("type", QString("INTEGER"));
fields.push_back(obj);
}
{
QJsonObject obj;
obj.insert("property_name", QString("panel_temp_Avg"));
obj.insert("alias", QString("panel"));
obj.insert("type", QString("INTEGER"));
fields.push_back(obj);
}
{
QJsonObject obj;
obj.insert("property_name", QString("leakage_cur_Avg"));
obj.insert("alias", QString("leak"));
obj.insert("type", QString("FLOAT"));
fields.push_back(obj);
}
{
QJsonObject obj;
obj.insert("property_name", QString("battery_volt_Avg"));
obj.insert("alias", QString("bateria"));
obj.insert("type", QString("FLOAT"));
fields.push_back(obj);
}
{
QJsonObject obj;
obj.insert("property_name", QString("internal_RH_Avg"));
obj.insert("alias", QString("interval"));
obj.insert("type", QString("TEXT"));
fields.push_back(obj);
}
QJsonObject obj;
obj.insert("fields", fields);
QJsonDocument doc(obj);
dataSet->format.emplace("fields", QString(doc.toJson(QJsonDocument::Compact)).toStdString());
dataSeries->datasetList.emplace_back(dataSet);
//empty filter
terrama2::core::Filter filter;
//accessing data
auto accessor = terrama2::core::DataAccessorFactory::getInstance().make(dataProviderPtr, dataSeriesPtr);
auto remover = std::make_shared<terrama2::core::FileRemover>();
auto uriMap = accessor->getFiles(filter, remover);
auto dataMap = accessor->getSeries(uriMap, filter, remover);
terrama2::core::DcpSeriesPtr dcpSeries = std::make_shared<terrama2::core::DcpSeries>();
dcpSeries->addDcpSeries(dataMap);
assert(dcpSeries->dcpSeriesMap().size() == 1);
QUrl uri;
uri.setScheme("postgis");
uri.setHost("localhost");
uri.setPort(5432);
uri.setUserName("postgres");
uri.setPassword("postgres");
uri.setPath("/terrama2");
//DataProvider information
terrama2::core::DataProvider* dataProviderPostGIS = new terrama2::core::DataProvider();
terrama2::core::DataProviderPtr dataProviderPostGISPtr(dataProviderPostGIS);
dataProviderPostGIS->uri = uri.url().toStdString();
dataProviderPostGIS->intent = terrama2::core::DataProviderIntent::PROCESS_INTENT;
dataProviderPostGIS->dataProviderType = "POSTGIS";
dataProviderPostGIS->active = true;
//DataSeries information
terrama2::core::DataSeries* outputDataSeries = new terrama2::core::DataSeries();
terrama2::core::DataSeriesPtr outputDataSeriesPtr(outputDataSeries);
outputDataSeries->semantics = semanticsManager.getSemantics("DCP-postgis");
terrama2::core::DataSetDcp* dataSetOutput = new terrama2::core::DataSetDcp();
terrama2::core::DataSetPtr dataSetOutputPtr(dataSetOutput);
dataSetOutput->active = true;
dataSetOutput->format.emplace("table_name", "dcp_generic");
dataSetOutput->format.emplace("timestamp_column", "datetime");
auto dataStorager = terrama2::core::DataStoragerFactory::getInstance().make(outputDataSeriesPtr, dataProviderPostGISPtr);
for(auto& item : dataMap)
{
dataStorager->store( item.second, dataSetOutputPtr);
}
return 0;
}
int main(int, char**)
{
terrama2::core::TerraMA2Init terramaRaii("example", 0);
terrama2::core::registerFactories();
{
//DataProvider information
terrama2::core::DataProvider* dataProvider = new terrama2::core::DataProvider();
terrama2::core::DataProviderPtr dataProviderPtr(dataProvider);
dataProvider->uri = "file://";
dataProvider->uri += TERRAMA2_DATA_DIR;
dataProvider->uri += "/grib";
dataProvider->intent = terrama2::core::DataProviderIntent::COLLECTOR_INTENT;
dataProvider->dataProviderType = "FILE";
dataProvider->active = true;
//DataSeries information
terrama2::core::DataSeries* dataSeries = new terrama2::core::DataSeries();
terrama2::core::DataSeriesPtr dataSeriesPtr(dataSeries);
auto& semanticsManager = terrama2::core::SemanticsManager::getInstance();
dataSeries->semantics = semanticsManager.getSemantics("GRID-grib");
terrama2::core::DataSetGrid* dataSet = new terrama2::core::DataSetGrid();
dataSet->active = true;
dataSet->format.emplace("mask", "WRF_ams_09km_2016012900_2016012900.grb2");
dataSet->format.emplace("timezone", "+00");
dataSet->format.emplace("bands", "UGRD;VGRD");
dataSet->format.emplace("iso_surface", "10000");
dataSeries->datasetList.emplace_back(dataSet);
//empty filter
terrama2::core::Filter filter;
//accessing data
terrama2::core::DataAccessorGrib accessor(dataProviderPtr, dataSeriesPtr);
auto remover = std::make_shared<terrama2::core::FileRemover>();
terrama2::core::GridSeriesPtr gridSeries = accessor.getGridSeries(filter,remover);
assert(gridSeries->gridMap().size() == 1);
//DataProvider information
terrama2::core::DataProvider* outputDataProvider = new terrama2::core::DataProvider();
terrama2::core::DataProviderPtr outputDataProviderPtr(outputDataProvider);
outputDataProvider->uri = "file://"+TERRAMA2_DATA_DIR+"/saida";
terrama2::core::DataSeries* outputDataSeries = new terrama2::core::DataSeries();
terrama2::core::DataSeriesPtr outputDataSeriesPtr(outputDataSeries);
outputDataSeries->semantics = semanticsManager.getSemantics("GRID-gdal");
terrama2::core::DataStoragerTiff dataStorager(outputDataSeriesPtr, outputDataProviderPtr);
terrama2::core::DataSetGrid* outputDataSet = new terrama2::core::DataSetGrid();
terrama2::core::DataSetGridPtr outputDataSetPtr(outputDataSet);
outputDataSet->active = true;
outputDataSet->format.emplace("mask", "ventos.tif");
outputDataSeries->datasetList.push_back(outputDataSetPtr);
auto seriesMap = gridSeries->getSeries();
auto series = seriesMap.begin()->second;
dataStorager.store(series, outputDataSetPtr);
}
return 0;
}
int main(int argc, char* argv[])
{
terrama2::core::initializeTerraMA();
//DataProvider information
terrama2::core::DataProvider* dataProvider = new terrama2::core::DataProvider();
terrama2::core::DataProviderPtr dataProviderPtr(dataProvider);
dataProvider->uri = "file://"+TERRAMA2_DATA_DIR+"/pcd_toa5";
dataProvider->intent = terrama2::core::DataProviderIntent::COLLECTOR_INTENT;
dataProvider->dataProviderType = "FILE";
dataProvider->active = true;
//DataSeries information
terrama2::core::DataSeries* dataSeries = new terrama2::core::DataSeries();
terrama2::core::DataSeriesPtr dataSeriesPtr(dataSeries);
auto& semanticsManager = terrama2::core::SemanticsManager::getInstance();
dataSeries->semantics = semanticsManager.getSemantics("DCP-toa5");
terrama2::core::DataSetDcp* dataSet = new terrama2::core::DataSetDcp();
dataSet->active = true;
dataSet->format.emplace("mask", "GRM_slow_2014_01_02_1713.dat");
dataSet->format.emplace("timezone", "+00");
dataSet->format.emplace("folder", "GRM");
dataSeries->datasetList.emplace_back(dataSet);
//empty filter
terrama2::core::Filter filter;
//accessing data
terrama2::core::DataAccessorDcpToa5 accessor(dataProviderPtr, dataSeriesPtr);
terrama2::core::DcpSeriesPtr dcpSeries = accessor.getDcpSeries(filter);
assert(dcpSeries->dcpSeriesMap().size() == 1);
std::shared_ptr<te::da::DataSet> teDataSet = (*dcpSeries->dcpSeriesMap().begin()).second.syncDataSet->dataset();
//Print column names and types (DateTime/Int/String/Double)
int dateColumnDateTime = -1;
int dateColumnRecord = -1;
int dateColumnStation = -1;
std::string names, types;
for(int i = 0; i < teDataSet->getNumProperties(); ++i)
{
std::string name = teDataSet->getPropertyName(i);
names+= name + "\t";
if (name == "RECORD")
{
types+= "INT32\t";
dateColumnRecord = i;
}
else if (name == "Estacao_ID")
{
types+= "String\t";
dateColumnStation = i;
}
else if(name == "DateTime")
{
types+= "DataTime\t";
dateColumnDateTime = i;
}
else
{
types+= "Double\t";
}
}
std::cout << names << std::endl;
std::cout << types << std::endl;
//Print values
teDataSet->moveBeforeFirst();
while(teDataSet->moveNext())
{
for(int i = 0; i < teDataSet->getNumProperties(); ++i)
{
if(teDataSet->isNull(i))
{
std::cout << "NULL";
}
else if(i == dateColumnRecord)
{
int value_int = teDataSet->getInt32(i);
std::cout << value_int;
}
else if(i == dateColumnStation)
{
std::string value_str = teDataSet->getString(i);
std::cout << value_str;
}
else if(i == dateColumnDateTime)
{
std::shared_ptr<te::dt::DateTime> dateTime = teDataSet->getDateTime(i);
std::cout << dateTime->toString();
}
else
{
double value = teDataSet->getDouble(i);
std::cout.precision(4);
std::cout << std::fixed << value;
}
std::cout << "\t";
}
std::cout << std::endl;
}
terrama2::core::finalizeTerraMA();
return 0;
}