void terrama2::services::alert::core::AdditionalDataHelper::addAdditionalAttributesColumns(std::shared_ptr<te::da::DataSetType> alertDataSetType) const
{
if(!isDataReady_)
{
throw AdditionalDataException() << ErrorDescription(QObject::tr("Data not loaded.\nCall AdditionalDataHelper::prepareData."));
}
for(const auto& data : dataMap_)
{
const auto& dataSet = data.first;
const auto& dataSetSeries = data.second;
for(const auto& attribute : additionalData_.attributes)
{
try
{
std::string name = dataSeries_->name+"_"+attribute+"_"+std::to_string(dataSet->id);
auto teDataSetType = dataSetSeries.teDataSetType;
auto property = teDataSetType->getProperty(attribute);
auto newProperty = property->clone();
newProperty->setName(name);
alertDataSetType->add(newProperty);
}
catch(...)
{
TERRAMA2_LOG_WARNING() << QObject::tr("Attribute %1 not found in dataset %2").arg(QString::fromStdString(attribute)).arg(dataSet->id);
}
}
}
}
std::set<std::string> terrama2::services::analysis::core::verify::dataAvailable(DataManagerPtr dataManager, AnalysisPtr analysis)
{
std::set<std::string> vecMessages;
for(auto& analysisDataSeries : analysis->analysisDataSeriesList)
{
auto dataSeries = dataManager->findDataSeries(analysisDataSeries.dataSeriesId);
auto dataProvider = dataManager->findDataProvider(dataSeries->dataProviderId);
auto dataAccesor = terrama2::core::DataAccessorFactory::getInstance().make(dataProvider, dataSeries);
try
{
terrama2::core::Filter filter;
auto series = dataAccesor->getSeries(filter, nullptr);
if(series.empty())
{
QString errMsg = QObject::tr("No data available for data series '%1'.").arg(dataSeries->name.c_str());
TERRAMA2_LOG_WARNING() << errMsg;
vecMessages.insert(vecMessages.end(), errMsg.toStdString());
}
}
catch(const terrama2::core::NoDataException&)
{
QString errMsg = QObject::tr("No data available for data series '%1'.").arg(dataSeries->name.c_str());
TERRAMA2_LOG_WARNING() << errMsg;
vecMessages.insert(vecMessages.end(), errMsg.toStdString());
}
catch(const terrama2::Exception& e)
{
std::string errMsg = boost::get_error_info<terrama2::ErrorDescription>(e)->toStdString();
vecMessages.insert(vecMessages.end(), errMsg);
TERRAMA2_LOG_ERROR() << errMsg;
}
}
return vecMessages;
}
std::set<std::string> terrama2::services::analysis::core::verify::inactiveDataSeries(DataManagerPtr dataManager, AnalysisPtr analysis)
{
std::set<std::string> vecMessages;
for(auto& analysisDataSeries : analysis->analysisDataSeriesList)
{
auto dataSeries = dataManager->findDataSeries(analysisDataSeries.dataSeriesId);
if(!dataSeries->active)
{
QString errMsg = QObject::tr("Data series '%1' is inactive.").arg(dataSeries->name.c_str());
TERRAMA2_LOG_WARNING() << errMsg;
vecMessages.insert(errMsg.toStdString());
}
}
return vecMessages;
}
bool terrama2::services::alert::core::AdditionalDataHelper::prepareData(terrama2::core::Filter filter)
{
auto dataAccessor = terrama2::core::DataAccessorFactory::getInstance().make(dataProvider_, dataSeries_);
dataMap_ = dataAccessor->getSeries(filter, remover_);
if(dataMap_.empty())
{
TERRAMA2_LOG_WARNING() << QObject::tr("No data to available in dataseries %1.").arg(additionalData_.id);
return false;
}
for(const auto& data : dataMap_)
{
const auto& dataSetSeries = data.second;
auto mapper = std::make_shared<terrama2::core::DataSetMapper>(dataSetSeries.syncDataSet->dataset(), additionalData_.identifier);
mapperMap_.emplace(data.first, mapper);
}
isDataReady_ = true;
return true;
}
std::set<std::string> terrama2::services::analysis::core::verify::attributeIdentifier(DataManagerPtr dataManager, AnalysisPtr analysis)
{
std::set<std::string> vecMessages;
if(analysis->type == AnalysisType::MONITORED_OBJECT_TYPE)
{
for(auto analysisDataSeries : analysis->analysisDataSeriesList)
{
if(analysisDataSeries.type == AnalysisDataSeriesType::DATASERIES_MONITORED_OBJECT_TYPE)
{
auto dataSeries = dataManager->findDataSeries(analysisDataSeries.dataSeriesId);
auto dataProvider = dataManager->findDataProvider(dataSeries->dataProviderId);
std::string identifier = analysisDataSeries.metadata["identifier"];
auto dataAccesor = terrama2::core::DataAccessorFactory::getInstance().make(dataProvider, dataSeries);
try
{
terrama2::core::Filter filter;
auto series = dataAccesor->getSeries(filter, nullptr);
if(!series.empty())
{
auto it = series.begin();
if(it != series.end())
{
if(!it->second.teDataSetType)
{
QString errMsg = QObject::tr("Could not read dataset type for data series '%1'.").arg(dataSeries->name.c_str());
TERRAMA2_LOG_WARNING() << errMsg;
vecMessages.insert(vecMessages.end(), errMsg.toStdString());
}
auto property = it->second.teDataSetType->getProperty(identifier);
if(property == nullptr)
{
QString errMsg = QObject::tr("Could not find the attribute identifier '%1' in data series '%2'.").arg(QString::fromStdString(identifier), QString::fromStdString(dataSeries->name));
TERRAMA2_LOG_WARNING() << errMsg;
vecMessages.insert(vecMessages.end(), errMsg.toStdString());
}
}
else
{
throw new terrama2::core::NoDataException();
}
}
}
catch(const terrama2::core::NoDataException&)
{
QString errMsg = QObject::tr("No data available for data series '%1'.").arg(dataSeries->name.c_str());
TERRAMA2_LOG_WARNING() << errMsg;
vecMessages.insert(vecMessages.end(), errMsg.toStdString());
}
catch(const terrama2::Exception& e)
{
std::string errMsg = boost::get_error_info<terrama2::ErrorDescription>(e)->toStdString();
vecMessages.insert(vecMessages.end(), errMsg);
TERRAMA2_LOG_ERROR() << errMsg;
}
}
}
}
return vecMessages;
}
void terrama2::services::view::core::Service::removeCompleteView(const ViewPtr& viewPtr, DataSeriesId dataSeriesId, bool removeAll) noexcept
{
ViewId id = viewPtr->id;
try
{
std::lock_guard<std::mutex> lock(mutex_);
TERRAMA2_LOG_INFO() << tr("Trying to remove view %1.").arg(id);
auto it = timers_.find(id);
if(it != timers_.end())
{
auto timer = timers_.at(id);
timer->disconnect();
timers_.erase(id);
}
// remove from queue
processQueue_.erase(std::remove_if(processQueue_.begin(), processQueue_.end(),
[&id](const terrama2::core::ExecutionPackage& executionPackage)
{ return id == executionPackage.processId; }), processQueue_.end());
if (removeAll)
{
terrama2::core::DataProviderPtr inputDataProvider;
// Locking datamanager
{
auto dataManager = dataManager_.lock();
auto lock = dataManager->getLock();
terrama2::core::DataSeriesPtr inputDataSeries = dataManager->findDataSeries(dataSeriesId);
inputDataProvider = dataManager->findDataProvider(inputDataSeries->dataProviderId);
lock.unlock();
}
try
{
// Retrieving Maps server handler
MapsServerPtr mapsServer = MapsServerFactory::getInstance().make(mapsServerUri_, "GEOSERVER");
// removing from geoserver
mapsServer->cleanup(viewPtr, inputDataProvider, logger_);
}
catch(const terrama2::services::view::core::NotFoundGeoserverException& e)
{
// Nothing
TERRAMA2_LOG_DEBUG() << tr("There is no workspace in GeoServer");
}
catch(const terrama2::services::view::core::ViewGeoserverException&)
{
// TODO: Improve validation in order to notify WebApp
TERRAMA2_LOG_WARNING() << tr("Could not perform clean up in GeoServer and database. Please remove it manually");
}
}
waitQueue_.erase(id);
TERRAMA2_LOG_INFO() << tr("View %1 removed successfully.").arg(id);
}
catch(const boost::exception& e)
{
TERRAMA2_LOG_ERROR() << boost::get_error_info<terrama2::ErrorDescription>(e);
TERRAMA2_LOG_INFO() << tr("Could not remove view: %1.").arg(id);
}
catch(const std::exception& e)
{
TERRAMA2_LOG_ERROR() << e.what();
TERRAMA2_LOG_INFO() << tr("Could not remove view: %1.").arg(id);
}
catch(...)
{
TERRAMA2_LOG_ERROR() << tr("Unknown error");
TERRAMA2_LOG_INFO() << tr("Could not remove view: %1.").arg(id);
}
}
void terrama2::core::DataRetrieverFTP::retrieveDataCallback(const std::string& mask,
const Filter& filter,
const std::string& timezone,
std::shared_ptr<terrama2::core::FileRemover> remover,
const std::string& temporaryFolderUri,
const std::string& foldersMask,
std::function<void(const std::string &, const std::string &, const std::string&)> processFile) const
{
try
{
// find valid directories
std::vector< std::string > baseUriList;
baseUriList.push_back(dataProvider_->uri);
auto tz = timezone.empty() ? "UTC+00" : timezone;
if(!foldersMask.empty())
{
auto uriList = getFoldersList(baseUriList, foldersMask, tz, filter);
if(uriList.empty())
{
QString errMsg = QObject::tr("No files found!");
TERRAMA2_LOG_WARNING() << errMsg;
throw terrama2::core::NoDataException() << ErrorDescription(errMsg);
}
baseUriList = uriList;
}
// Build URI to get PATH fragment
te::core::URI dataProviderURI(dataProvider_->uri);
// Set temporary directory. When empty, creates a new pointing to dataProvider Path.
// In this way, we will have something like "temporaryDir/dataProviderPath"
// It is important due the folder may contains temporal mask
std::string temporaryDataDir = temporaryFolderUri;
if (temporaryFolderUri.empty())
temporaryDataDir = getTemporaryFolder(remover, temporaryFolderUri) + dataProviderURI.path();
// flag if there is any files for the dataset
bool hasData = false;
// Get a file listing from server
for(const auto& uri : baseUriList)
{
std::vector<std::string> vectorFiles = curlwrapper_->listFiles(normalizeURI(uri));
std::vector<std::string> vectorNames;
// filter file names that should be downloaded.
for(const std::string& fileName: vectorFiles)
{
// FIXME: use timestamp
std::shared_ptr< te::dt::TimeInstantTZ > timestamp;
if(terrama2::core::isValidDataSetName(mask,filter, tz, fileName,timestamp))
vectorNames.push_back(fileName);
}
if(vectorNames.empty())
{
continue;
}
hasData = true;
te::core::URI u(uri);
std::string uriPath = QString::fromStdString(u.path()).replace(dataProviderURI.path().c_str(), "/").toStdString();
// Performs the download of files in the vectorNames
for(const auto& file: vectorNames)
{
// Create directory struct
QString saveDir(QString::fromStdString(temporaryDataDir+ "/" + uriPath));
QString savePath = QUrl(saveDir).toLocalFile();
QDir dir(savePath);
if(!dir.exists())
dir.mkpath(savePath);
std::string uriOrigin = uri + "/" + file;
std::string filePath = savePath.toStdString() + "/" + file;
remover->addTemporaryFolder(temporaryDataDir);
remover->addTemporaryFile(filePath);
try
{
curlwrapper_->downloadFile(normalizeURI(uriOrigin).uri(), filePath);
TERRAMA2_LOG_WARNING() << QObject::tr("Finished downloading file: %1").arg(QString::fromStdString(file));
processFile(temporaryDataDir, file, uriPath);
}
catch(const te::Exception& e)
{
QString errMsg = QObject::tr("Error during download of file %1.\n").arg(QString::fromStdString(file));
auto errStr = boost::get_error_info<te::ErrorDescription>(e);
if(errStr)
errMsg.append(QString::fromStdString(*errStr));
errMsg.append(e.what());
TERRAMA2_LOG_ERROR() << errMsg;
throw DataRetrieverException() << ErrorDescription(errMsg);
}
}
}
if(!hasData)
{
QString errMsg = QObject::tr("No data in the remote server.");
TERRAMA2_LOG_WARNING() << errMsg;
throw NoDataException() << ErrorDescription(errMsg);
}
}
catch(const NoDataException&)
{
throw;
}
catch(const DataRetrieverException&)
{
throw;
}
catch(const te::Exception& e)
{
QString errMsg = QObject::tr("Error during download.\n");
errMsg.append(boost::get_error_info<terrama2::ErrorDescription>(e));
errMsg.append(e.what());
TERRAMA2_LOG_ERROR() << errMsg;
throw DataRetrieverException() << ErrorDescription(errMsg);
}
catch(const std::exception& e)
{
QString errMsg = QObject::tr("Error during download.\n");
errMsg.append(e.what());
TERRAMA2_LOG_ERROR() << errMsg;
throw DataRetrieverException() << ErrorDescription(errMsg);
}
catch(...)
{
throw DataRetrieverException() << ErrorDescription(QObject::tr("Unknown Error."));
}
}
std::string terrama2::core::DataRetrieverFTP::retrieveData(const std::string& mask,
const Filter& filter,
const std::string& timezone,
std::shared_ptr<terrama2::core::FileRemover> remover,
const std::string& temporaryFolderUri,
const std::string& foldersMask) const
{
std::string downloadBaseFolderUri = getTemporaryFolder(remover, temporaryFolderUri);
try
{
// find valid directories
std::vector< std::string > baseUriList;
baseUriList.push_back(dataProvider_->uri);
if(!foldersMask.empty())
{
auto uriList = getFoldersList(baseUriList, foldersMask);
if(uriList.empty())
{
QString errMsg = QObject::tr("No files found!");
TERRAMA2_LOG_WARNING() << errMsg;
throw terrama2::core::NoDataException() << ErrorDescription(errMsg);
}
baseUriList = uriList;
}
// Get a file listing from server
for(const auto& uri : baseUriList)
{
std::vector<std::string> vectorFiles = curlwrapper_->listFiles(te::core::URI(uri));
std::vector<std::string> vectorNames;
// filter file names that should be downloaded.
for(const std::string& fileName: vectorFiles)
{
// FIXME: use timestamp
std::shared_ptr< te::dt::TimeInstantTZ > timestamp;
if(terrama2::core::isValidDataSetName(mask,filter, timezone.empty() ? "UTC+00" : timezone, fileName,timestamp))
vectorNames.push_back(fileName);
}
if(vectorNames.empty())
{
continue;
}
// Create directory struct
QString saveDir(QString::fromStdString(uri));
saveDir.replace(QString::fromStdString(dataProvider_->uri), QString::fromStdString(downloadBaseFolderUri));
QString savePath = QUrl(saveDir).toString(QUrl::RemoveScheme);
QDir dir(savePath);
if(!dir.exists())
dir.mkpath(savePath);
// Performs the download of files in the vectorNames
for(const auto& file: vectorNames)
{
std::string uriOrigin = uri + "/" + file;
std::string filePath = savePath.toStdString() + "/" + file;
try
{
curlwrapper_->downloadFile(uriOrigin, filePath);
}
catch(const te::Exception& e)
{
QString errMsg = QObject::tr("Error during download of file %1.\n").arg(QString::fromStdString(file));
auto errStr = boost::get_error_info<te::ErrorDescription>(e);
if(errStr)
errMsg.append(QString::fromStdString(*errStr));
errMsg.append(e.what());
TERRAMA2_LOG_ERROR() << errMsg;
throw DataRetrieverException() << ErrorDescription(errMsg);
}
remover->addTemporaryFile(filePath);
}
}
}
catch(const NoDataException&)
{
throw;
}
catch(const DataRetrieverException&)
{
throw;
}
catch(const te::Exception& e)
{
QString errMsg = QObject::tr("Error during download.\n");
errMsg.append(boost::get_error_info<terrama2::ErrorDescription>(e));
errMsg.append(e.what());
TERRAMA2_LOG_ERROR() << errMsg;
throw DataRetrieverException() << ErrorDescription(errMsg);
}
catch(const std::exception& e)
{
QString errMsg = QObject::tr("Error during download.\n");
errMsg.append(e.what());
TERRAMA2_LOG_ERROR() << errMsg;
throw DataRetrieverException() << ErrorDescription(errMsg);
}
catch(...)
{
throw DataRetrieverException() << ErrorDescription(QObject::tr("Unknown Error."));
}
// returns the absolute path of the folder that contains the files that have been made the download.
return downloadBaseFolderUri;
}
