std::vector<std::string> terrama2::core::CurlPtr::getListFiles(std::string url,
size_t(*write_vector)(void *ptr, size_t size, size_t nmemb, void *data),
std::string block)
{
std::vector<std::string> vectorFiles;
curl_easy_setopt(curl_, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl_, CURLOPT_DIRLISTONLY, 1);
curl_easy_setopt(curl_, CURLOPT_WRITEFUNCTION, write_vector);
curl_easy_setopt(curl_, CURLOPT_WRITEDATA, (void *)&block);
CURLcode status = curl_easy_perform(curl_);
if (status == CURLE_OK)
{
boost::split(vectorFiles, block, boost::is_any_of("\n"));
if(!vectorFiles.empty() && vectorFiles.back().empty())
vectorFiles.pop_back();
}
else
{
QString errMsg = QObject::tr("Could not list files in the FTP server. \n\n");
errMsg.append(curl_easy_strerror(status));
TERRAMA2_LOG_ERROR() << errMsg;
throw DataRetrieverException() << ErrorDescription(errMsg);
}
return vectorFiles;
}
terrama2::core::DataRetrieverFTP::DataRetrieverFTP(DataProviderPtr dataprovider, CurlPtr&& curlwrapper)
: DataRetriever(dataprovider),
curlwrapper_(std::move(curlwrapper))
{
temporaryFolder_ = "/tmp/terrama2-download/";
scheme_ = "file://";
// Create the directory where you will download the files.
QDir dir(temporaryFolder_.c_str());
if (!dir.exists())
dir.mkpath(temporaryFolder_.c_str());
CURLcode status;
curlwrapper_.init();
// Verifies that the FTP address is valid
try
{
std::string url = dataprovider->uri.c_str();
status = curlwrapper_.verifyURL(url);
if (status != CURLE_OK)
{
QString errMsg = QObject::tr("FTP address is invalid. \n\n");
errMsg.append(curl_easy_strerror(status));
TERRAMA2_LOG_ERROR() << errMsg;
throw DataRetrieverException() << ErrorDescription(errMsg);
}
}
catch(const std::exception& e)
{
QString errMsg = QObject::tr("FTP address is invalid! \n\n Details: \n");
errMsg.append(e.what());
TERRAMA2_LOG_ERROR() << errMsg;
throw DataRetrieverException() << ErrorDescription(errMsg);
}
catch(...)
{
throw DataRetrieverException() << ErrorDescription(QObject::tr("Unknown Error, FTP address is invalid!"));
}
}
terrama2::core::DataRetriever::DataRetriever(DataProviderPtr dataProvider)
: dataProvider_(dataProvider)
{
if(!dataProvider_.get())
{
QString errMsg = QObject::tr("Mandatory parameters not provided.");
TERRAMA2_LOG_ERROR() << errMsg;
throw DataRetrieverException() << ErrorDescription(errMsg);
}
}
terrama2::core::DataRetrieverFTP::DataRetrieverFTP(DataProviderPtr dataprovider, std::unique_ptr<CurlWrapperFtp>&& curlwrapper)
: DataRetriever(dataprovider),
curlwrapper_(std::move(curlwrapper))
{
//Set FTP mode
try
{
auto activeMode = dataProvider_->options.at("active_mode");
curlwrapper_->setActiveMode(activeMode == "true", "-");
}
catch(const std::out_of_range&)
{
//if not set, set to false
curlwrapper_->setActiveMode(false);
}
// Verifies that the FTP address is valid
try
{
curlwrapper_->verifyURL(dataprovider->uri, dataProvider_->timeout);
}
catch(const te::Exception& e)
{
QString errMsg = QObject::tr("FTP address is invalid! \n\n Details: \n");
auto errStr = boost::get_error_info<te::ErrorDescription>(e);
if(errStr)
errMsg.append(QString::fromStdString(*errStr));
errMsg.append(e.what());
TERRAMA2_LOG_ERROR() << errMsg;
throw DataRetrieverException() << ErrorDescription(errMsg);
}
catch(...)
{
throw DataRetrieverException() << ErrorDescription(QObject::tr("Unknown Error!"));
}
}
std::string terrama2::core::DataRetrieverFTP::retrieveData(const std::string& mask, const terrama2::core::Filter& filter)
{
std::string uriOrigin;
std::string uriInput;
std::vector<std::string> vectorFiles;
std::string block;
curlwrapper_.init();
try
{
// Get a file listing from server
if(curlwrapper_.fcurl())
{
uriInput = dataProvider_->uri+"/";
vectorFiles = curlwrapper_.getListFiles(uriInput, &terrama2::core::DataRetrieverFTP::write_vector, block);
// filter file names that should be downloaded.
for (std::string fileName: vectorFiles)
{
// FIXME: use timestamp
std::string timezone = "UTC+00";//FIXME: get timezone from dataset
std::shared_ptr< te::dt::TimeInstantTZ > timestamp;
if (terrama2::core::isValidDataSetName(mask,filter, timezone, fileName,timestamp))
vectorNames_.push_back(fileName);
}
if (!vectorNames_.empty())
{
for (std::string file: vectorNames_)
{
CURLcode res;
curlwrapper_.init();
// Performs the download of files in the vectorNames
if(curlwrapper_.fcurl())
{
uriOrigin = dataProvider_->uri +"/"+file;
std::string filePath = temporaryFolder_+"/"+file;
res = curlwrapper_.getDownloadFiles(uriOrigin, &terrama2::core::DataRetrieverFTP::write_response, filePath);
if (res != CURLE_OK)
{
QString errMsg = QObject::tr("Could not perform the download! \n\n");
errMsg.append(curl_easy_strerror(res));
TERRAMA2_LOG_ERROR() << errMsg;
throw DataRetrieverException() << ErrorDescription(errMsg);
}
}
}
}
else
{
QString errMsg = QObject::tr("Could not found files! \n\n");
TERRAMA2_LOG_ERROR() << errMsg;
throw DataRetrieverException() << ErrorDescription(errMsg);
}
}
}
catch(const std::exception& e)
{
QString errMsg = QObject::tr("Could not perform the download files! \n\n Details: \n");
errMsg.append(e.what());
TERRAMA2_LOG_ERROR() << errMsg;
throw DataRetrieverException() << ErrorDescription(errMsg);
}
catch(...)
{
throw DataRetrieverException() << ErrorDescription(QObject::tr("Unknown Error, Could not perform the download files!"));
}
// returns the absolute path of the folder that contains the files that have been made the download.
return scheme_+temporaryFolder_;
}
void terrama2::core::DataRetrieverFTP::retrieveDataCallback(const std::string& mask,
const Filter& filter,
const std::string& timezone,
std::shared_ptr<terrama2::core::FileRemover> remover,
const std::string& temporaryFolderUri,
const std::string& foldersMask,
std::function<void(const std::string &, const std::string &, const std::string&)> processFile) const
{
try
{
// find valid directories
std::vector< std::string > baseUriList;
baseUriList.push_back(dataProvider_->uri);
auto tz = timezone.empty() ? "UTC+00" : timezone;
if(!foldersMask.empty())
{
auto uriList = getFoldersList(baseUriList, foldersMask, tz, filter);
if(uriList.empty())
{
QString errMsg = QObject::tr("No files found!");
TERRAMA2_LOG_WARNING() << errMsg;
throw terrama2::core::NoDataException() << ErrorDescription(errMsg);
}
baseUriList = uriList;
}
// Build URI to get PATH fragment
te::core::URI dataProviderURI(dataProvider_->uri);
// Set temporary directory. When empty, creates a new pointing to dataProvider Path.
// In this way, we will have something like "temporaryDir/dataProviderPath"
// It is important due the folder may contains temporal mask
std::string temporaryDataDir = temporaryFolderUri;
if (temporaryFolderUri.empty())
temporaryDataDir = getTemporaryFolder(remover, temporaryFolderUri) + dataProviderURI.path();
// flag if there is any files for the dataset
bool hasData = false;
// Get a file listing from server
for(const auto& uri : baseUriList)
{
std::vector<std::string> vectorFiles = curlwrapper_->listFiles(normalizeURI(uri));
std::vector<std::string> vectorNames;
// filter file names that should be downloaded.
for(const std::string& fileName: vectorFiles)
{
// FIXME: use timestamp
std::shared_ptr< te::dt::TimeInstantTZ > timestamp;
if(terrama2::core::isValidDataSetName(mask,filter, tz, fileName,timestamp))
vectorNames.push_back(fileName);
}
if(vectorNames.empty())
{
continue;
}
hasData = true;
te::core::URI u(uri);
std::string uriPath = QString::fromStdString(u.path()).replace(dataProviderURI.path().c_str(), "/").toStdString();
// Performs the download of files in the vectorNames
for(const auto& file: vectorNames)
{
// Create directory struct
QString saveDir(QString::fromStdString(temporaryDataDir+ "/" + uriPath));
QString savePath = QUrl(saveDir).toLocalFile();
QDir dir(savePath);
if(!dir.exists())
dir.mkpath(savePath);
std::string uriOrigin = uri + "/" + file;
std::string filePath = savePath.toStdString() + "/" + file;
remover->addTemporaryFolder(temporaryDataDir);
remover->addTemporaryFile(filePath);
try
{
curlwrapper_->downloadFile(normalizeURI(uriOrigin).uri(), filePath);
TERRAMA2_LOG_WARNING() << QObject::tr("Finished downloading file: %1").arg(QString::fromStdString(file));
processFile(temporaryDataDir, file, uriPath);
}
catch(const te::Exception& e)
{
QString errMsg = QObject::tr("Error during download of file %1.\n").arg(QString::fromStdString(file));
auto errStr = boost::get_error_info<te::ErrorDescription>(e);
if(errStr)
errMsg.append(QString::fromStdString(*errStr));
errMsg.append(e.what());
TERRAMA2_LOG_ERROR() << errMsg;
throw DataRetrieverException() << ErrorDescription(errMsg);
}
}
}
if(!hasData)
{
QString errMsg = QObject::tr("No data in the remote server.");
TERRAMA2_LOG_WARNING() << errMsg;
throw NoDataException() << ErrorDescription(errMsg);
}
}
catch(const NoDataException&)
{
throw;
}
catch(const DataRetrieverException&)
{
throw;
}
catch(const te::Exception& e)
{
QString errMsg = QObject::tr("Error during download.\n");
errMsg.append(boost::get_error_info<terrama2::ErrorDescription>(e));
errMsg.append(e.what());
TERRAMA2_LOG_ERROR() << errMsg;
throw DataRetrieverException() << ErrorDescription(errMsg);
}
catch(const std::exception& e)
{
QString errMsg = QObject::tr("Error during download.\n");
errMsg.append(e.what());
TERRAMA2_LOG_ERROR() << errMsg;
throw DataRetrieverException() << ErrorDescription(errMsg);
}
catch(...)
{
throw DataRetrieverException() << ErrorDescription(QObject::tr("Unknown Error."));
}
}
std::string terrama2::core::DataRetrieverFTP::retrieveData(const std::string& mask,
const Filter& filter,
const std::string& timezone,
std::shared_ptr<terrama2::core::FileRemover> remover,
const std::string& temporaryFolderUri,
const std::string& foldersMask) const
{
std::string downloadBaseFolderUri = getTemporaryFolder(remover, temporaryFolderUri);
try
{
// find valid directories
std::vector< std::string > baseUriList;
baseUriList.push_back(dataProvider_->uri);
if(!foldersMask.empty())
{
auto uriList = getFoldersList(baseUriList, foldersMask);
if(uriList.empty())
{
QString errMsg = QObject::tr("No files found!");
TERRAMA2_LOG_WARNING() << errMsg;
throw terrama2::core::NoDataException() << ErrorDescription(errMsg);
}
baseUriList = uriList;
}
// Get a file listing from server
for(const auto& uri : baseUriList)
{
std::vector<std::string> vectorFiles = curlwrapper_->listFiles(te::core::URI(uri));
std::vector<std::string> vectorNames;
// filter file names that should be downloaded.
for(const std::string& fileName: vectorFiles)
{
// FIXME: use timestamp
std::shared_ptr< te::dt::TimeInstantTZ > timestamp;
if(terrama2::core::isValidDataSetName(mask,filter, timezone.empty() ? "UTC+00" : timezone, fileName,timestamp))
vectorNames.push_back(fileName);
}
if(vectorNames.empty())
{
continue;
}
// Create directory struct
QString saveDir(QString::fromStdString(uri));
saveDir.replace(QString::fromStdString(dataProvider_->uri), QString::fromStdString(downloadBaseFolderUri));
QString savePath = QUrl(saveDir).toString(QUrl::RemoveScheme);
QDir dir(savePath);
if(!dir.exists())
dir.mkpath(savePath);
// Performs the download of files in the vectorNames
for(const auto& file: vectorNames)
{
std::string uriOrigin = uri + "/" + file;
std::string filePath = savePath.toStdString() + "/" + file;
try
{
curlwrapper_->downloadFile(uriOrigin, filePath);
}
catch(const te::Exception& e)
{
QString errMsg = QObject::tr("Error during download of file %1.\n").arg(QString::fromStdString(file));
auto errStr = boost::get_error_info<te::ErrorDescription>(e);
if(errStr)
errMsg.append(QString::fromStdString(*errStr));
errMsg.append(e.what());
TERRAMA2_LOG_ERROR() << errMsg;
throw DataRetrieverException() << ErrorDescription(errMsg);
}
remover->addTemporaryFile(filePath);
}
}
}
catch(const NoDataException&)
{
throw;
}
catch(const DataRetrieverException&)
{
throw;
}
catch(const te::Exception& e)
{
QString errMsg = QObject::tr("Error during download.\n");
errMsg.append(boost::get_error_info<terrama2::ErrorDescription>(e));
errMsg.append(e.what());
TERRAMA2_LOG_ERROR() << errMsg;
throw DataRetrieverException() << ErrorDescription(errMsg);
}
catch(const std::exception& e)
{
QString errMsg = QObject::tr("Error during download.\n");
errMsg.append(e.what());
TERRAMA2_LOG_ERROR() << errMsg;
throw DataRetrieverException() << ErrorDescription(errMsg);
}
catch(...)
{
throw DataRetrieverException() << ErrorDescription(QObject::tr("Unknown Error."));
}
// returns the absolute path of the folder that contains the files that have been made the download.
return downloadBaseFolderUri;
}
