void Downloader::downloadSync(const std::string &srcUrl, const std::string &storagePath, const std::string &customId/* = ""*/) { FileDescriptor fDesc; ProgressData pData; prepareDownload(srcUrl, storagePath, customId, false, &fDesc, &pData); if (fDesc.fp != NULL) { download(srcUrl, customId, fDesc, pData); } }
void Downloader::downloadAsync(const std::string &srcUrl, const std::string &storagePath, const std::string &customId/* = ""*/) { FileDescriptor fDesc; ProgressData pData; prepareDownload(srcUrl, storagePath, customId, false, &fDesc, &pData); if (fDesc.fp != NULL) { auto t = std::thread(&Downloader::download, this, srcUrl, customId, fDesc, pData); t.detach(); } }
void HttpDownload::slot_downloadFinished(QNetworkReply *reply) { /** * @brief The file name */ QString fileName; /** * @brief Flag for extraction */ bool needsExtraction = false; // Check in which download phase we are switch(downloadPhase) { case 0: fileName = "launcher/downloads/files_game.txt"; break; case 1: fileName = fileList.at(filesLeft); break; } // Open the file to write to QFile file(installationPath + fileName); if(file.open(QIODevice::WriteOnly)) { // Open a stream to write into the file QDataStream stream(&file); // Get the size of the file int size = reply->size(); // Add size to status int for displaying totalSizeDownloadedCurrent = qint64(0); totalSizeDownloaded += qint64(size); // Get the data of the file QByteArray temp = reply->readAll(); // Write the file stream.writeRawData(temp, size); // Set exe permissions file.setPermissions(QFile::ExeUser | QFile::ExeGroup | QFile::ExeOwner | QFile::WriteUser | QFile::WriteGroup | QFile::WriteOwner | QFile::ReadGroup | QFile::ReadOwner | QFile::ReadUser); // Close the file file.close(); // Check if it is an archive if(fileName.endsWith("tar.xz")) { // Create instance of archiver archive->setFile(installationPath + fileName, installationPath); // Connect as start as thread connect(archive, SIGNAL(finished()),SLOT(slot_extractionDone())); archive->start(); // Set flag and status needsExtraction = true; status = "extracting"; } } // If phase 0 take future steps if(!needsExtraction) { switch(downloadPhase) { case 0: prepareDownload(); downloadPhase = 1; break; case 1: getNextFile(); break; } } }
void Downloader::groupBatchDownload(const DownloadUnits &units) { CURLM* multi_handle = curl_multi_init(); int still_running = 0; for (auto it = units.cbegin(); it != units.cend(); ++it) { DownloadUnit unit = it->second; std::string srcUrl = unit.srcUrl; std::string storagePath = unit.storagePath; std::string customId = unit.customId; FileDescriptor *fDesc = new FileDescriptor(); ProgressData *data = new ProgressData(); prepareDownload(srcUrl, storagePath, customId, unit.resumeDownload, fDesc, data); if (fDesc->fp != NULL) { CURL* curl = curl_easy_init(); curl_easy_setopt(curl, CURLOPT_URL, srcUrl.c_str()); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fileWriteFunc); curl_easy_setopt(curl, CURLOPT_WRITEDATA, fDesc->fp); curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false); curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, batchDownloadProgressFunc); curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, data); curl_easy_setopt(curl, CURLOPT_FAILONERROR, true); if (_connectionTimeout) curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, _connectionTimeout); curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, LOW_SPEED_LIMIT); curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, LOW_SPEED_TIME); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true); curl_easy_setopt(curl, CURLOPT_MAXREDIRS, MAX_REDIRS); // Resuming download support if (_supportResuming && unit.resumeDownload) { // Check already downloaded size for current download unit long size = _fileUtils->getFileSize(storagePath + TEMP_EXT); if (size != -1) { curl_easy_setopt(curl, CURLOPT_RESUME_FROM_LARGE, size); } } fDesc->curl = curl; CURLMcode code = curl_multi_add_handle(multi_handle, curl); if (code != CURLM_OK) { // Avoid memory leak fclose(fDesc->fp); delete data; delete fDesc; std::string msg = StringUtils::format("Unable to add curl handler for %s: [curl error]%s", customId.c_str(), curl_multi_strerror(code)); this->notifyError(msg, code, customId); } else { // Add to list for tracking _progDatas.push_back(data); _files.push_back(fDesc); } } } // Query multi perform CURLMcode curlm_code = CURLM_CALL_MULTI_PERFORM; while(CURLM_CALL_MULTI_PERFORM == curlm_code) { curlm_code = curl_multi_perform(multi_handle, &still_running); } if (curlm_code != CURLM_OK) { std::string msg = StringUtils::format("Unable to continue the download process: [curl error]%s", curl_multi_strerror(curlm_code)); this->notifyError(msg, curlm_code); } else { bool failed = false; while (still_running > 0 && !failed) { // set a suitable timeout to play around with struct timeval select_tv; long curl_timeo = -1; select_tv.tv_sec = 1; select_tv.tv_usec = 0; curl_multi_timeout(multi_handle, &curl_timeo); if(curl_timeo >= 0) { select_tv.tv_sec = curl_timeo / 1000; if(select_tv.tv_sec > 1) select_tv.tv_sec = 1; else select_tv.tv_usec = (curl_timeo % 1000) * 1000; } int rc; fd_set fdread; fd_set fdwrite; fd_set fdexcep; int maxfd = -1; FD_ZERO(&fdread); FD_ZERO(&fdwrite); FD_ZERO(&fdexcep); // FIXME: when jenkins migrate to ubuntu, we should remove this hack code #if (CC_TARGET_PLATFORM == CC_PLATFORM_LINUX) curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd); rc = select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &select_tv); #else rc = curl_multi_wait(multi_handle, nullptr, 0, MAX_WAIT_MSECS, &maxfd); #endif switch(rc) { case -1: failed = true; break; case 0: default: curlm_code = CURLM_CALL_MULTI_PERFORM; while(CURLM_CALL_MULTI_PERFORM == curlm_code) { curlm_code = curl_multi_perform(multi_handle, &still_running); } if (curlm_code != CURLM_OK) { std::string msg = StringUtils::format("Unable to continue the download process: [curl error]%s", curl_multi_strerror(curlm_code)); this->notifyError(msg, curlm_code); } break; } } } // Clean up and close files curl_multi_cleanup(multi_handle); for (auto it = _files.begin(); it != _files.end(); ++it) { FILE *f = (*it)->fp; fclose(f); CURL *single = (CURL *)((*it)->curl); curl_multi_remove_handle(multi_handle, single); curl_easy_cleanup(single); } // Check unfinished files and notify errors, succeed files will be renamed from temporary file name to real name for (auto it = _progDatas.begin(); it != _progDatas.end(); ++it) { ProgressData *data = *it; if (data->downloaded < data->totalToDownload || data->totalToDownload == 0) { this->notifyError(ErrorCode::NETWORK, "Unable to download file", data->customId); } else { _fileUtils->renameFile(data->path, data->name + TEMP_EXT, data->name); } } clearBatchDownloadData(); }
void HttpDownload::slot_downloadFinished(QNetworkReply *reply) { /** * @brief The file name */ QString fileName; /** * @brief Flag for extraction */ bool needsExtraction = false; // Check in which download phase we are switch(downloadPhase) { case 0: fileName = "launcher/downloads/files_game.txt"; break; case 1: fileName = fileList.at(filesLeft); break; } Logging::addEntry(LOG_LEVEL_INF, "Savinge file: " + fileName, FUNCTION_NAME); // Open the file to write to QFile file(installationPath + fileName); if(file.open(QIODevice::WriteOnly)) { // Open a stream to write into the file QDataStream stream(&file); // Get the size of the file qint64 size = reply->size(); // Add size to status int for displaying totalSizeDownloadedCurrent = qint64(0); totalSizeDownloaded += qint64(size); // Get the data of the file QByteArray temp = reply->read(size); // Note: ReadAll will crash 32 bit builds! // Write the file stream.writeRawData(temp, size); // Set exe permissions file.setPermissions(QFile::ExeUser | QFile::ExeGroup | QFile::ExeOwner | QFile::WriteUser | QFile::WriteGroup | QFile::WriteOwner | QFile::ReadGroup | QFile::ReadOwner | QFile::ReadUser); // Close the file file.close(); // Check if it is an archive if(fileName.endsWith("xz")) { Logging::addEntry(LOG_LEVEL_INF, "Found archive, starting extracting", FUNCTION_NAME); // Get folder structure QStringList extractFolders = fileName.split("/"); QString extractFolder; for(int i = 0; i < extractFolders.size()-1; i++) { extractFolder += extractFolders.at(i) + "/"; } // Create sub folder to extract into QString targetFolder = extractFolders.last().split(".").at(0); QDir().mkdir(installationPath + extractFolder + targetFolder); // Create instance of archiver archive->setFile(extractFolders.last(), installationPath + extractFolder, targetFolder); // Connect as start as thread connect(archive, SIGNAL(extraction_done(bool)),SLOT(slot_extractionDone(bool))); archive->start(); // Set flag and status needsExtraction = true; status = "extracting"; } } else { Logging::addEntry(LOG_LEVEL_ERR, "File could not be opened", FUNCTION_NAME); } // If phase 0 take future steps if(!needsExtraction) { switch(downloadPhase) { case 0: prepareDownload(); downloadPhase = 1; break; case 1: getNextFile(); break; } } }