bool FileManager::getAllFolders(const std::string &rootFolder, std::list<std::string> &result, const bool remDotDirs) { std::stack<std::string> folderStack; std::list<std::string> auxList; std::string actualFolder; result.clear(); folderStack.push(rootFolder); /* we start the BFS */ while(!folderStack.empty()) { actualFolder = folderStack.top(); folderStack.pop(); // add it to the result list result.push_back(actualFolder); if(!getFoldersList(actualFolder, auxList, false)){ return false; } // now put all the folders in the queue for(std::list<std::string>::iterator it = auxList.begin(); it != auxList.end(); ++it){ folderStack.push(*it); } } if (remDotDirs){ removeDotDirs(rootFolder, result); } return true; }
void terrama2::core::DataRetrieverFTP::retrieveDataCallback(const std::string& mask, const Filter& filter, const std::string& timezone, std::shared_ptr<terrama2::core::FileRemover> remover, const std::string& temporaryFolderUri, const std::string& foldersMask, std::function<void(const std::string &, const std::string &, const std::string&)> processFile) const { try { // find valid directories std::vector< std::string > baseUriList; baseUriList.push_back(dataProvider_->uri); auto tz = timezone.empty() ? "UTC+00" : timezone; if(!foldersMask.empty()) { auto uriList = getFoldersList(baseUriList, foldersMask, tz, filter); if(uriList.empty()) { QString errMsg = QObject::tr("No files found!"); TERRAMA2_LOG_WARNING() << errMsg; throw terrama2::core::NoDataException() << ErrorDescription(errMsg); } baseUriList = uriList; } // Build URI to get PATH fragment te::core::URI dataProviderURI(dataProvider_->uri); // Set temporary directory. When empty, creates a new pointing to dataProvider Path. // In this way, we will have something like "temporaryDir/dataProviderPath" // It is important due the folder may contains temporal mask std::string temporaryDataDir = temporaryFolderUri; if (temporaryFolderUri.empty()) temporaryDataDir = getTemporaryFolder(remover, temporaryFolderUri) + dataProviderURI.path(); // flag if there is any files for the dataset bool hasData = false; // Get a file listing from server for(const auto& uri : baseUriList) { std::vector<std::string> vectorFiles = curlwrapper_->listFiles(normalizeURI(uri)); std::vector<std::string> vectorNames; // filter file names that should be downloaded. for(const std::string& fileName: vectorFiles) { // FIXME: use timestamp std::shared_ptr< te::dt::TimeInstantTZ > timestamp; if(terrama2::core::isValidDataSetName(mask,filter, tz, fileName,timestamp)) vectorNames.push_back(fileName); } if(vectorNames.empty()) { continue; } hasData = true; te::core::URI u(uri); std::string uriPath = QString::fromStdString(u.path()).replace(dataProviderURI.path().c_str(), "/").toStdString(); // Performs the download of files in the vectorNames for(const auto& file: vectorNames) { // Create directory struct QString saveDir(QString::fromStdString(temporaryDataDir+ "/" + uriPath)); QString savePath = QUrl(saveDir).toLocalFile(); QDir dir(savePath); if(!dir.exists()) dir.mkpath(savePath); std::string uriOrigin = uri + "/" + file; std::string filePath = savePath.toStdString() + "/" + file; remover->addTemporaryFolder(temporaryDataDir); remover->addTemporaryFile(filePath); try { curlwrapper_->downloadFile(normalizeURI(uriOrigin).uri(), filePath); TERRAMA2_LOG_WARNING() << QObject::tr("Finished downloading file: %1").arg(QString::fromStdString(file)); processFile(temporaryDataDir, file, uriPath); } catch(const te::Exception& e) { QString errMsg = QObject::tr("Error during download of file %1.\n").arg(QString::fromStdString(file)); auto errStr = boost::get_error_info<te::ErrorDescription>(e); if(errStr) errMsg.append(QString::fromStdString(*errStr)); errMsg.append(e.what()); TERRAMA2_LOG_ERROR() << errMsg; throw DataRetrieverException() << ErrorDescription(errMsg); } } } if(!hasData) { QString errMsg = QObject::tr("No data in the remote server."); TERRAMA2_LOG_WARNING() << errMsg; throw NoDataException() << ErrorDescription(errMsg); } } catch(const NoDataException&) { throw; } catch(const DataRetrieverException&) { throw; } catch(const te::Exception& e) { QString errMsg = QObject::tr("Error during download.\n"); errMsg.append(boost::get_error_info<terrama2::ErrorDescription>(e)); errMsg.append(e.what()); TERRAMA2_LOG_ERROR() << errMsg; throw DataRetrieverException() << ErrorDescription(errMsg); } catch(const std::exception& e) { QString errMsg = QObject::tr("Error during download.\n"); errMsg.append(e.what()); TERRAMA2_LOG_ERROR() << errMsg; throw DataRetrieverException() << ErrorDescription(errMsg); } catch(...) { throw DataRetrieverException() << ErrorDescription(QObject::tr("Unknown Error.")); } }
std::string terrama2::core::DataRetrieverFTP::retrieveData(const std::string& mask, const Filter& filter, const std::string& timezone, std::shared_ptr<terrama2::core::FileRemover> remover, const std::string& temporaryFolderUri, const std::string& foldersMask) const { std::string downloadBaseFolderUri = getTemporaryFolder(remover, temporaryFolderUri); try { // find valid directories std::vector< std::string > baseUriList; baseUriList.push_back(dataProvider_->uri); if(!foldersMask.empty()) { auto uriList = getFoldersList(baseUriList, foldersMask); if(uriList.empty()) { QString errMsg = QObject::tr("No files found!"); TERRAMA2_LOG_WARNING() << errMsg; throw terrama2::core::NoDataException() << ErrorDescription(errMsg); } baseUriList = uriList; } // Get a file listing from server for(const auto& uri : baseUriList) { std::vector<std::string> vectorFiles = curlwrapper_->listFiles(te::core::URI(uri)); std::vector<std::string> vectorNames; // filter file names that should be downloaded. for(const std::string& fileName: vectorFiles) { // FIXME: use timestamp std::shared_ptr< te::dt::TimeInstantTZ > timestamp; if(terrama2::core::isValidDataSetName(mask,filter, timezone.empty() ? "UTC+00" : timezone, fileName,timestamp)) vectorNames.push_back(fileName); } if(vectorNames.empty()) { continue; } // Create directory struct QString saveDir(QString::fromStdString(uri)); saveDir.replace(QString::fromStdString(dataProvider_->uri), QString::fromStdString(downloadBaseFolderUri)); QString savePath = QUrl(saveDir).toString(QUrl::RemoveScheme); QDir dir(savePath); if(!dir.exists()) dir.mkpath(savePath); // Performs the download of files in the vectorNames for(const auto& file: vectorNames) { std::string uriOrigin = uri + "/" + file; std::string filePath = savePath.toStdString() + "/" + file; try { curlwrapper_->downloadFile(uriOrigin, filePath); } catch(const te::Exception& e) { QString errMsg = QObject::tr("Error during download of file %1.\n").arg(QString::fromStdString(file)); auto errStr = boost::get_error_info<te::ErrorDescription>(e); if(errStr) errMsg.append(QString::fromStdString(*errStr)); errMsg.append(e.what()); TERRAMA2_LOG_ERROR() << errMsg; throw DataRetrieverException() << ErrorDescription(errMsg); } remover->addTemporaryFile(filePath); } } } catch(const NoDataException&) { throw; } catch(const DataRetrieverException&) { throw; } catch(const te::Exception& e) { QString errMsg = QObject::tr("Error during download.\n"); errMsg.append(boost::get_error_info<terrama2::ErrorDescription>(e)); errMsg.append(e.what()); TERRAMA2_LOG_ERROR() << errMsg; throw DataRetrieverException() << ErrorDescription(errMsg); } catch(const std::exception& e) { QString errMsg = QObject::tr("Error during download.\n"); errMsg.append(e.what()); TERRAMA2_LOG_ERROR() << errMsg; throw DataRetrieverException() << ErrorDescription(errMsg); } catch(...) { throw DataRetrieverException() << ErrorDescription(QObject::tr("Unknown Error.")); } // returns the absolute path of the folder that contains the files that have been made the download. return downloadBaseFolderUri; }