void terrama2::services::alert::core::AdditionalDataHelper::addAdditionalAttributesColumns(std::shared_ptr<te::da::DataSetType> alertDataSetType) const { if(!isDataReady_) { throw AdditionalDataException() << ErrorDescription(QObject::tr("Data not loaded.\nCall AdditionalDataHelper::prepareData.")); } for(const auto& data : dataMap_) { const auto& dataSet = data.first; const auto& dataSetSeries = data.second; for(const auto& attribute : additionalData_.attributes) { try { std::string name = dataSeries_->name+"_"+attribute+"_"+std::to_string(dataSet->id); auto teDataSetType = dataSetSeries.teDataSetType; auto property = teDataSetType->getProperty(attribute); auto newProperty = property->clone(); newProperty->setName(name); alertDataSetType->add(newProperty); } catch(...) { TERRAMA2_LOG_WARNING() << QObject::tr("Attribute %1 not found in dataset %2").arg(QString::fromStdString(attribute)).arg(dataSet->id); } } } }
std::set<std::string> terrama2::services::analysis::core::verify::dataAvailable(DataManagerPtr dataManager, AnalysisPtr analysis) { std::set<std::string> vecMessages; for(auto& analysisDataSeries : analysis->analysisDataSeriesList) { auto dataSeries = dataManager->findDataSeries(analysisDataSeries.dataSeriesId); auto dataProvider = dataManager->findDataProvider(dataSeries->dataProviderId); auto dataAccesor = terrama2::core::DataAccessorFactory::getInstance().make(dataProvider, dataSeries); try { terrama2::core::Filter filter; auto series = dataAccesor->getSeries(filter, nullptr); if(series.empty()) { QString errMsg = QObject::tr("No data available for data series '%1'.").arg(dataSeries->name.c_str()); TERRAMA2_LOG_WARNING() << errMsg; vecMessages.insert(vecMessages.end(), errMsg.toStdString()); } } catch(const terrama2::core::NoDataException&) { QString errMsg = QObject::tr("No data available for data series '%1'.").arg(dataSeries->name.c_str()); TERRAMA2_LOG_WARNING() << errMsg; vecMessages.insert(vecMessages.end(), errMsg.toStdString()); } catch(const terrama2::Exception& e) { std::string errMsg = boost::get_error_info<terrama2::ErrorDescription>(e)->toStdString(); vecMessages.insert(vecMessages.end(), errMsg); TERRAMA2_LOG_ERROR() << errMsg; } } return vecMessages; }
std::set<std::string> terrama2::services::analysis::core::verify::inactiveDataSeries(DataManagerPtr dataManager, AnalysisPtr analysis) { std::set<std::string> vecMessages; for(auto& analysisDataSeries : analysis->analysisDataSeriesList) { auto dataSeries = dataManager->findDataSeries(analysisDataSeries.dataSeriesId); if(!dataSeries->active) { QString errMsg = QObject::tr("Data series '%1' is inactive.").arg(dataSeries->name.c_str()); TERRAMA2_LOG_WARNING() << errMsg; vecMessages.insert(errMsg.toStdString()); } } return vecMessages; }
bool terrama2::services::alert::core::AdditionalDataHelper::prepareData(terrama2::core::Filter filter) { auto dataAccessor = terrama2::core::DataAccessorFactory::getInstance().make(dataProvider_, dataSeries_); dataMap_ = dataAccessor->getSeries(filter, remover_); if(dataMap_.empty()) { TERRAMA2_LOG_WARNING() << QObject::tr("No data to available in dataseries %1.").arg(additionalData_.id); return false; } for(const auto& data : dataMap_) { const auto& dataSetSeries = data.second; auto mapper = std::make_shared<terrama2::core::DataSetMapper>(dataSetSeries.syncDataSet->dataset(), additionalData_.identifier); mapperMap_.emplace(data.first, mapper); } isDataReady_ = true; return true; }
std::set<std::string> terrama2::services::analysis::core::verify::attributeIdentifier(DataManagerPtr dataManager, AnalysisPtr analysis) { std::set<std::string> vecMessages; if(analysis->type == AnalysisType::MONITORED_OBJECT_TYPE) { for(auto analysisDataSeries : analysis->analysisDataSeriesList) { if(analysisDataSeries.type == AnalysisDataSeriesType::DATASERIES_MONITORED_OBJECT_TYPE) { auto dataSeries = dataManager->findDataSeries(analysisDataSeries.dataSeriesId); auto dataProvider = dataManager->findDataProvider(dataSeries->dataProviderId); std::string identifier = analysisDataSeries.metadata["identifier"]; auto dataAccesor = terrama2::core::DataAccessorFactory::getInstance().make(dataProvider, dataSeries); try { terrama2::core::Filter filter; auto series = dataAccesor->getSeries(filter, nullptr); if(!series.empty()) { auto it = series.begin(); if(it != series.end()) { if(!it->second.teDataSetType) { QString errMsg = QObject::tr("Could not read dataset type for data series '%1'.").arg(dataSeries->name.c_str()); TERRAMA2_LOG_WARNING() << errMsg; vecMessages.insert(vecMessages.end(), errMsg.toStdString()); } auto property = it->second.teDataSetType->getProperty(identifier); if(property == nullptr) { QString errMsg = QObject::tr("Could not find the attribute identifier '%1' in data series '%2'.").arg(QString::fromStdString(identifier), QString::fromStdString(dataSeries->name)); TERRAMA2_LOG_WARNING() << errMsg; vecMessages.insert(vecMessages.end(), errMsg.toStdString()); } } else { throw new terrama2::core::NoDataException(); } } } catch(const terrama2::core::NoDataException&) { QString errMsg = QObject::tr("No data available for data series '%1'.").arg(dataSeries->name.c_str()); TERRAMA2_LOG_WARNING() << errMsg; vecMessages.insert(vecMessages.end(), errMsg.toStdString()); } catch(const terrama2::Exception& e) { std::string errMsg = boost::get_error_info<terrama2::ErrorDescription>(e)->toStdString(); vecMessages.insert(vecMessages.end(), errMsg); TERRAMA2_LOG_ERROR() << errMsg; } } } } return vecMessages; }
void terrama2::services::view::core::Service::removeCompleteView(const ViewPtr& viewPtr, DataSeriesId dataSeriesId, bool removeAll) noexcept { ViewId id = viewPtr->id; try { std::lock_guard<std::mutex> lock(mutex_); TERRAMA2_LOG_INFO() << tr("Trying to remove view %1.").arg(id); auto it = timers_.find(id); if(it != timers_.end()) { auto timer = timers_.at(id); timer->disconnect(); timers_.erase(id); } // remove from queue processQueue_.erase(std::remove_if(processQueue_.begin(), processQueue_.end(), [&id](const terrama2::core::ExecutionPackage& executionPackage) { return id == executionPackage.processId; }), processQueue_.end()); if (removeAll) { terrama2::core::DataProviderPtr inputDataProvider; // Locking datamanager { auto dataManager = dataManager_.lock(); auto lock = dataManager->getLock(); terrama2::core::DataSeriesPtr inputDataSeries = dataManager->findDataSeries(dataSeriesId); inputDataProvider = dataManager->findDataProvider(inputDataSeries->dataProviderId); lock.unlock(); } try { // Retrieving Maps server handler MapsServerPtr mapsServer = MapsServerFactory::getInstance().make(mapsServerUri_, "GEOSERVER"); // removing from geoserver mapsServer->cleanup(viewPtr, inputDataProvider, logger_); } catch(const terrama2::services::view::core::NotFoundGeoserverException& e) { // Nothing TERRAMA2_LOG_DEBUG() << tr("There is no workspace in GeoServer"); } catch(const terrama2::services::view::core::ViewGeoserverException&) { // TODO: Improve validation in order to notify WebApp TERRAMA2_LOG_WARNING() << tr("Could not perform clean up in GeoServer and database. Please remove it manually"); } } waitQueue_.erase(id); TERRAMA2_LOG_INFO() << tr("View %1 removed successfully.").arg(id); } catch(const boost::exception& e) { TERRAMA2_LOG_ERROR() << boost::get_error_info<terrama2::ErrorDescription>(e); TERRAMA2_LOG_INFO() << tr("Could not remove view: %1.").arg(id); } catch(const std::exception& e) { TERRAMA2_LOG_ERROR() << e.what(); TERRAMA2_LOG_INFO() << tr("Could not remove view: %1.").arg(id); } catch(...) { TERRAMA2_LOG_ERROR() << tr("Unknown error"); TERRAMA2_LOG_INFO() << tr("Could not remove view: %1.").arg(id); } }
void terrama2::core::DataRetrieverFTP::retrieveDataCallback(const std::string& mask, const Filter& filter, const std::string& timezone, std::shared_ptr<terrama2::core::FileRemover> remover, const std::string& temporaryFolderUri, const std::string& foldersMask, std::function<void(const std::string &, const std::string &, const std::string&)> processFile) const { try { // find valid directories std::vector< std::string > baseUriList; baseUriList.push_back(dataProvider_->uri); auto tz = timezone.empty() ? "UTC+00" : timezone; if(!foldersMask.empty()) { auto uriList = getFoldersList(baseUriList, foldersMask, tz, filter); if(uriList.empty()) { QString errMsg = QObject::tr("No files found!"); TERRAMA2_LOG_WARNING() << errMsg; throw terrama2::core::NoDataException() << ErrorDescription(errMsg); } baseUriList = uriList; } // Build URI to get PATH fragment te::core::URI dataProviderURI(dataProvider_->uri); // Set temporary directory. When empty, creates a new pointing to dataProvider Path. // In this way, we will have something like "temporaryDir/dataProviderPath" // It is important due the folder may contains temporal mask std::string temporaryDataDir = temporaryFolderUri; if (temporaryFolderUri.empty()) temporaryDataDir = getTemporaryFolder(remover, temporaryFolderUri) + dataProviderURI.path(); // flag if there is any files for the dataset bool hasData = false; // Get a file listing from server for(const auto& uri : baseUriList) { std::vector<std::string> vectorFiles = curlwrapper_->listFiles(normalizeURI(uri)); std::vector<std::string> vectorNames; // filter file names that should be downloaded. for(const std::string& fileName: vectorFiles) { // FIXME: use timestamp std::shared_ptr< te::dt::TimeInstantTZ > timestamp; if(terrama2::core::isValidDataSetName(mask,filter, tz, fileName,timestamp)) vectorNames.push_back(fileName); } if(vectorNames.empty()) { continue; } hasData = true; te::core::URI u(uri); std::string uriPath = QString::fromStdString(u.path()).replace(dataProviderURI.path().c_str(), "/").toStdString(); // Performs the download of files in the vectorNames for(const auto& file: vectorNames) { // Create directory struct QString saveDir(QString::fromStdString(temporaryDataDir+ "/" + uriPath)); QString savePath = QUrl(saveDir).toLocalFile(); QDir dir(savePath); if(!dir.exists()) dir.mkpath(savePath); std::string uriOrigin = uri + "/" + file; std::string filePath = savePath.toStdString() + "/" + file; remover->addTemporaryFolder(temporaryDataDir); remover->addTemporaryFile(filePath); try { curlwrapper_->downloadFile(normalizeURI(uriOrigin).uri(), filePath); TERRAMA2_LOG_WARNING() << QObject::tr("Finished downloading file: %1").arg(QString::fromStdString(file)); processFile(temporaryDataDir, file, uriPath); } catch(const te::Exception& e) { QString errMsg = QObject::tr("Error during download of file %1.\n").arg(QString::fromStdString(file)); auto errStr = boost::get_error_info<te::ErrorDescription>(e); if(errStr) errMsg.append(QString::fromStdString(*errStr)); errMsg.append(e.what()); TERRAMA2_LOG_ERROR() << errMsg; throw DataRetrieverException() << ErrorDescription(errMsg); } } } if(!hasData) { QString errMsg = QObject::tr("No data in the remote server."); TERRAMA2_LOG_WARNING() << errMsg; throw NoDataException() << ErrorDescription(errMsg); } } catch(const NoDataException&) { throw; } catch(const DataRetrieverException&) { throw; } catch(const te::Exception& e) { QString errMsg = QObject::tr("Error during download.\n"); errMsg.append(boost::get_error_info<terrama2::ErrorDescription>(e)); errMsg.append(e.what()); TERRAMA2_LOG_ERROR() << errMsg; throw DataRetrieverException() << ErrorDescription(errMsg); } catch(const std::exception& e) { QString errMsg = QObject::tr("Error during download.\n"); errMsg.append(e.what()); TERRAMA2_LOG_ERROR() << errMsg; throw DataRetrieverException() << ErrorDescription(errMsg); } catch(...) { throw DataRetrieverException() << ErrorDescription(QObject::tr("Unknown Error.")); } }
std::string terrama2::core::DataRetrieverFTP::retrieveData(const std::string& mask, const Filter& filter, const std::string& timezone, std::shared_ptr<terrama2::core::FileRemover> remover, const std::string& temporaryFolderUri, const std::string& foldersMask) const { std::string downloadBaseFolderUri = getTemporaryFolder(remover, temporaryFolderUri); try { // find valid directories std::vector< std::string > baseUriList; baseUriList.push_back(dataProvider_->uri); if(!foldersMask.empty()) { auto uriList = getFoldersList(baseUriList, foldersMask); if(uriList.empty()) { QString errMsg = QObject::tr("No files found!"); TERRAMA2_LOG_WARNING() << errMsg; throw terrama2::core::NoDataException() << ErrorDescription(errMsg); } baseUriList = uriList; } // Get a file listing from server for(const auto& uri : baseUriList) { std::vector<std::string> vectorFiles = curlwrapper_->listFiles(te::core::URI(uri)); std::vector<std::string> vectorNames; // filter file names that should be downloaded. for(const std::string& fileName: vectorFiles) { // FIXME: use timestamp std::shared_ptr< te::dt::TimeInstantTZ > timestamp; if(terrama2::core::isValidDataSetName(mask,filter, timezone.empty() ? "UTC+00" : timezone, fileName,timestamp)) vectorNames.push_back(fileName); } if(vectorNames.empty()) { continue; } // Create directory struct QString saveDir(QString::fromStdString(uri)); saveDir.replace(QString::fromStdString(dataProvider_->uri), QString::fromStdString(downloadBaseFolderUri)); QString savePath = QUrl(saveDir).toString(QUrl::RemoveScheme); QDir dir(savePath); if(!dir.exists()) dir.mkpath(savePath); // Performs the download of files in the vectorNames for(const auto& file: vectorNames) { std::string uriOrigin = uri + "/" + file; std::string filePath = savePath.toStdString() + "/" + file; try { curlwrapper_->downloadFile(uriOrigin, filePath); } catch(const te::Exception& e) { QString errMsg = QObject::tr("Error during download of file %1.\n").arg(QString::fromStdString(file)); auto errStr = boost::get_error_info<te::ErrorDescription>(e); if(errStr) errMsg.append(QString::fromStdString(*errStr)); errMsg.append(e.what()); TERRAMA2_LOG_ERROR() << errMsg; throw DataRetrieverException() << ErrorDescription(errMsg); } remover->addTemporaryFile(filePath); } } } catch(const NoDataException&) { throw; } catch(const DataRetrieverException&) { throw; } catch(const te::Exception& e) { QString errMsg = QObject::tr("Error during download.\n"); errMsg.append(boost::get_error_info<terrama2::ErrorDescription>(e)); errMsg.append(e.what()); TERRAMA2_LOG_ERROR() << errMsg; throw DataRetrieverException() << ErrorDescription(errMsg); } catch(const std::exception& e) { QString errMsg = QObject::tr("Error during download.\n"); errMsg.append(e.what()); TERRAMA2_LOG_ERROR() << errMsg; throw DataRetrieverException() << ErrorDescription(errMsg); } catch(...) { throw DataRetrieverException() << ErrorDescription(QObject::tr("Unknown Error.")); } // returns the absolute path of the folder that contains the files that have been made the download. return downloadBaseFolderUri; }