bool FileManager::getAllFolders(const std::string &rootFolder,
std::list<std::string> &result,
const bool remDotDirs)
{
std::stack<std::string> folderStack;
std::list<std::string> auxList;
std::string actualFolder;
result.clear();
folderStack.push(rootFolder);
/* we start the BFS */
while(!folderStack.empty()) {
actualFolder = folderStack.top();
folderStack.pop();
// add it to the result list
result.push_back(actualFolder);
if(!getFoldersList(actualFolder, auxList, false)){
return false;
}
// now put all the folders in the queue
for(std::list<std::string>::iterator it = auxList.begin();
it != auxList.end(); ++it){
folderStack.push(*it);
}
}
if (remDotDirs){
removeDotDirs(rootFolder, result);
}
return true;
}
void terrama2::core::DataRetrieverFTP::retrieveDataCallback(const std::string& mask,
const Filter& filter,
const std::string& timezone,
std::shared_ptr<terrama2::core::FileRemover> remover,
const std::string& temporaryFolderUri,
const std::string& foldersMask,
std::function<void(const std::string &, const std::string &, const std::string&)> processFile) const
{
try
{
// find valid directories
std::vector< std::string > baseUriList;
baseUriList.push_back(dataProvider_->uri);
auto tz = timezone.empty() ? "UTC+00" : timezone;
if(!foldersMask.empty())
{
auto uriList = getFoldersList(baseUriList, foldersMask, tz, filter);
if(uriList.empty())
{
QString errMsg = QObject::tr("No files found!");
TERRAMA2_LOG_WARNING() << errMsg;
throw terrama2::core::NoDataException() << ErrorDescription(errMsg);
}
baseUriList = uriList;
}
// Build URI to get PATH fragment
te::core::URI dataProviderURI(dataProvider_->uri);
// Set temporary directory. When empty, creates a new pointing to dataProvider Path.
// In this way, we will have something like "temporaryDir/dataProviderPath"
// It is important due the folder may contains temporal mask
std::string temporaryDataDir = temporaryFolderUri;
if (temporaryFolderUri.empty())
temporaryDataDir = getTemporaryFolder(remover, temporaryFolderUri) + dataProviderURI.path();
// flag if there is any files for the dataset
bool hasData = false;
// Get a file listing from server
for(const auto& uri : baseUriList)
{
std::vector<std::string> vectorFiles = curlwrapper_->listFiles(normalizeURI(uri));
std::vector<std::string> vectorNames;
// filter file names that should be downloaded.
for(const std::string& fileName: vectorFiles)
{
// FIXME: use timestamp
std::shared_ptr< te::dt::TimeInstantTZ > timestamp;
if(terrama2::core::isValidDataSetName(mask,filter, tz, fileName,timestamp))
vectorNames.push_back(fileName);
}
if(vectorNames.empty())
{
continue;
}
hasData = true;
te::core::URI u(uri);
std::string uriPath = QString::fromStdString(u.path()).replace(dataProviderURI.path().c_str(), "/").toStdString();
// Performs the download of files in the vectorNames
for(const auto& file: vectorNames)
{
// Create directory struct
QString saveDir(QString::fromStdString(temporaryDataDir+ "/" + uriPath));
QString savePath = QUrl(saveDir).toLocalFile();
QDir dir(savePath);
if(!dir.exists())
dir.mkpath(savePath);
std::string uriOrigin = uri + "/" + file;
std::string filePath = savePath.toStdString() + "/" + file;
remover->addTemporaryFolder(temporaryDataDir);
remover->addTemporaryFile(filePath);
try
{
curlwrapper_->downloadFile(normalizeURI(uriOrigin).uri(), filePath);
TERRAMA2_LOG_WARNING() << QObject::tr("Finished downloading file: %1").arg(QString::fromStdString(file));
processFile(temporaryDataDir, file, uriPath);
}
catch(const te::Exception& e)
{
QString errMsg = QObject::tr("Error during download of file %1.\n").arg(QString::fromStdString(file));
auto errStr = boost::get_error_info<te::ErrorDescription>(e);
if(errStr)
errMsg.append(QString::fromStdString(*errStr));
errMsg.append(e.what());
TERRAMA2_LOG_ERROR() << errMsg;
throw DataRetrieverException() << ErrorDescription(errMsg);
}
}
}
if(!hasData)
{
QString errMsg = QObject::tr("No data in the remote server.");
TERRAMA2_LOG_WARNING() << errMsg;
throw NoDataException() << ErrorDescription(errMsg);
}
}
catch(const NoDataException&)
{
throw;
}
catch(const DataRetrieverException&)
{
throw;
}
catch(const te::Exception& e)
{
QString errMsg = QObject::tr("Error during download.\n");
errMsg.append(boost::get_error_info<terrama2::ErrorDescription>(e));
errMsg.append(e.what());
TERRAMA2_LOG_ERROR() << errMsg;
throw DataRetrieverException() << ErrorDescription(errMsg);
}
catch(const std::exception& e)
{
QString errMsg = QObject::tr("Error during download.\n");
errMsg.append(e.what());
TERRAMA2_LOG_ERROR() << errMsg;
throw DataRetrieverException() << ErrorDescription(errMsg);
}
catch(...)
{
throw DataRetrieverException() << ErrorDescription(QObject::tr("Unknown Error."));
}
}
std::string terrama2::core::DataRetrieverFTP::retrieveData(const std::string& mask,
const Filter& filter,
const std::string& timezone,
std::shared_ptr<terrama2::core::FileRemover> remover,
const std::string& temporaryFolderUri,
const std::string& foldersMask) const
{
std::string downloadBaseFolderUri = getTemporaryFolder(remover, temporaryFolderUri);
try
{
// find valid directories
std::vector< std::string > baseUriList;
baseUriList.push_back(dataProvider_->uri);
if(!foldersMask.empty())
{
auto uriList = getFoldersList(baseUriList, foldersMask);
if(uriList.empty())
{
QString errMsg = QObject::tr("No files found!");
TERRAMA2_LOG_WARNING() << errMsg;
throw terrama2::core::NoDataException() << ErrorDescription(errMsg);
}
baseUriList = uriList;
}
// Get a file listing from server
for(const auto& uri : baseUriList)
{
std::vector<std::string> vectorFiles = curlwrapper_->listFiles(te::core::URI(uri));
std::vector<std::string> vectorNames;
// filter file names that should be downloaded.
for(const std::string& fileName: vectorFiles)
{
// FIXME: use timestamp
std::shared_ptr< te::dt::TimeInstantTZ > timestamp;
if(terrama2::core::isValidDataSetName(mask,filter, timezone.empty() ? "UTC+00" : timezone, fileName,timestamp))
vectorNames.push_back(fileName);
}
if(vectorNames.empty())
{
continue;
}
// Create directory struct
QString saveDir(QString::fromStdString(uri));
saveDir.replace(QString::fromStdString(dataProvider_->uri), QString::fromStdString(downloadBaseFolderUri));
QString savePath = QUrl(saveDir).toString(QUrl::RemoveScheme);
QDir dir(savePath);
if(!dir.exists())
dir.mkpath(savePath);
// Performs the download of files in the vectorNames
for(const auto& file: vectorNames)
{
std::string uriOrigin = uri + "/" + file;
std::string filePath = savePath.toStdString() + "/" + file;
try
{
curlwrapper_->downloadFile(uriOrigin, filePath);
}
catch(const te::Exception& e)
{
QString errMsg = QObject::tr("Error during download of file %1.\n").arg(QString::fromStdString(file));
auto errStr = boost::get_error_info<te::ErrorDescription>(e);
if(errStr)
errMsg.append(QString::fromStdString(*errStr));
errMsg.append(e.what());
TERRAMA2_LOG_ERROR() << errMsg;
throw DataRetrieverException() << ErrorDescription(errMsg);
}
remover->addTemporaryFile(filePath);
}
}
}
catch(const NoDataException&)
{
throw;
}
catch(const DataRetrieverException&)
{
throw;
}
catch(const te::Exception& e)
{
QString errMsg = QObject::tr("Error during download.\n");
errMsg.append(boost::get_error_info<terrama2::ErrorDescription>(e));
errMsg.append(e.what());
TERRAMA2_LOG_ERROR() << errMsg;
throw DataRetrieverException() << ErrorDescription(errMsg);
}
catch(const std::exception& e)
{
QString errMsg = QObject::tr("Error during download.\n");
errMsg.append(e.what());
TERRAMA2_LOG_ERROR() << errMsg;
throw DataRetrieverException() << ErrorDescription(errMsg);
}
catch(...)
{
throw DataRetrieverException() << ErrorDescription(QObject::tr("Unknown Error."));
}
// returns the absolute path of the folder that contains the files that have been made the download.
return downloadBaseFolderUri;
}
