void AssetsManagerEx::updateAssets(const DownloadUnits& assets) { if (!_inited){ CCLOG("AssetsManagerEx : Manifests uninited.\n"); dispatchUpdateEvent(EventAssetsManagerEx::EventCode::ERROR_NO_LOCAL_MANIFEST); return; } if (_updateState != State::UPDATING && _localManifest->isLoaded() && _remoteManifest->isLoaded()) { int size = (int)(assets.size()); if (size > 0) { _updateState = State::UPDATING; _downloadUnits.clear(); _downloadUnits = assets; _totalWaitToDownload = _totalToDownload = (int)_downloadUnits.size(); this->batchDownload(); } else if (size == 0 && _totalWaitToDownload == 0) { updateSucceed(); } } }
void AssetsManagerEx::updateAssets(const DownloadUnits& assets) { if (!_inited){ CCLOG("AssetsManagerEx : Manifests uninited.\n"); dispatchUpdateEvent(EventAssetsManagerEx::EventCode::ERROR_NO_LOCAL_MANIFEST); return; } if (_updateState != State::UPDATING && _localManifest->isLoaded() && _remoteManifest->isLoaded()) { _updateState = State::UPDATING; _downloadUnits.clear(); _downloadedSize.clear(); _percent = _percentByFile = _sizeCollected = _totalSize = 0; _totalWaitToDownload = _totalToDownload = (int)assets.size(); _nextSavePoint = 0; _totalEnabled = false; if (_totalToDownload > 0) { _downloadUnits = assets; this->batchDownload(); } else if (_totalToDownload == 0) { onDownloadUnitsFinished(); } } }
void ModuleDownloader::batchDownloadSync(const DownloadUnits &units, const std::string &batchId/* = ""*/) { if (units.size() == 0) { return; } // Make sure ModuleDownloader won't be released std::weak_ptr<ModuleDownloader> ptr = shared_from_this(); // Test server download resuming support with the first unit _supportResuming = false; CURL *header = curl_easy_init(); // Make a resume request curl_easy_setopt(header, CURLOPT_RESUME_FROM_LARGE, 0); if (prepareHeader(header, units.begin()->second.srcUrl)) { long responseCode; curl_easy_getinfo(header, CURLINFO_RESPONSE_CODE, &responseCode); if (responseCode == HTTP_CODE_SUPPORT_RESUME) { _supportResuming = true; } } curl_easy_cleanup(header); int count = 0; DownloadUnits group; for (auto it = units.cbegin(); it != units.cend(); ++it, ++count) { if (count == FOPEN_MAX) { groupBatchDownload(group); group.clear(); count = 0; } const std::string &key = it->first; const DownloadUnit &unit = it->second; group.emplace(key, unit); } if (group.size() > 0) { groupBatchDownload(group); } Director::getInstance()->getScheduler()->performFunctionInCocosThread([ptr, batchId]{ if (!ptr.expired()) { std::shared_ptr<ModuleDownloader> downloader = ptr.lock(); auto callback = downloader->getSuccessCallback(); if (callback != nullptr) { callback("", "", batchId); } } }); _supportResuming = false; }
void Downloader::groupBatchDownload(const DownloadUnits &units) { CURLM* multi_handle = curl_multi_init(); int still_running = 0; for (auto it = units.cbegin(); it != units.cend(); ++it) { DownloadUnit unit = it->second; std::string srcUrl = unit.srcUrl; std::string storagePath = unit.storagePath; std::string customId = unit.customId; FileDescriptor *fDesc = new FileDescriptor(); ProgressData *data = new ProgressData(); prepareDownload(srcUrl, storagePath, customId, unit.resumeDownload, fDesc, data); if (fDesc->fp != NULL) { CURL* curl = curl_easy_init(); curl_easy_setopt(curl, CURLOPT_URL, srcUrl.c_str()); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fileWriteFunc); curl_easy_setopt(curl, CURLOPT_WRITEDATA, fDesc->fp); curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false); curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, batchDownloadProgressFunc); curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, data); curl_easy_setopt(curl, CURLOPT_FAILONERROR, true); if (_connectionTimeout) curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, _connectionTimeout); curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, LOW_SPEED_LIMIT); curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, LOW_SPEED_TIME); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true); curl_easy_setopt(curl, CURLOPT_MAXREDIRS, MAX_REDIRS); // Resuming download support if (_supportResuming && unit.resumeDownload) { // Check already downloaded size for current download unit long size = _fileUtils->getFileSize(storagePath + TEMP_EXT); if (size != -1) { curl_easy_setopt(curl, CURLOPT_RESUME_FROM_LARGE, size); } } fDesc->curl = curl; CURLMcode code = curl_multi_add_handle(multi_handle, curl); if (code != CURLM_OK) { // Avoid memory leak fclose(fDesc->fp); delete data; delete fDesc; std::string msg = StringUtils::format("Unable to add curl handler for %s: [curl error]%s", customId.c_str(), curl_multi_strerror(code)); this->notifyError(msg, code, customId); } else { // Add to list for tracking _progDatas.push_back(data); _files.push_back(fDesc); } } } // Query multi perform CURLMcode curlm_code = CURLM_CALL_MULTI_PERFORM; while(CURLM_CALL_MULTI_PERFORM == curlm_code) { curlm_code = curl_multi_perform(multi_handle, &still_running); } if (curlm_code != CURLM_OK) { std::string msg = StringUtils::format("Unable to continue the download process: [curl error]%s", curl_multi_strerror(curlm_code)); this->notifyError(msg, curlm_code); } else { bool failed = false; while (still_running > 0 && !failed) { // set a suitable timeout to play around with struct timeval select_tv; long curl_timeo = -1; select_tv.tv_sec = 1; select_tv.tv_usec = 0; curl_multi_timeout(multi_handle, &curl_timeo); if(curl_timeo >= 0) { select_tv.tv_sec = curl_timeo / 1000; if(select_tv.tv_sec > 1) select_tv.tv_sec = 1; else select_tv.tv_usec = (curl_timeo % 1000) * 1000; } int rc; fd_set fdread; fd_set fdwrite; fd_set fdexcep; int maxfd = -1; FD_ZERO(&fdread); FD_ZERO(&fdwrite); FD_ZERO(&fdexcep); // FIXME: when jenkins migrate to ubuntu, we should remove this hack code #if (CC_TARGET_PLATFORM == CC_PLATFORM_LINUX) curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd); rc = select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &select_tv); #else rc = curl_multi_wait(multi_handle, nullptr, 0, MAX_WAIT_MSECS, &maxfd); #endif switch(rc) { case -1: failed = true; break; case 0: default: curlm_code = CURLM_CALL_MULTI_PERFORM; while(CURLM_CALL_MULTI_PERFORM == curlm_code) { curlm_code = curl_multi_perform(multi_handle, &still_running); } if (curlm_code != CURLM_OK) { std::string msg = StringUtils::format("Unable to continue the download process: [curl error]%s", curl_multi_strerror(curlm_code)); this->notifyError(msg, curlm_code); } break; } } } // Clean up and close files curl_multi_cleanup(multi_handle); for (auto it = _files.begin(); it != _files.end(); ++it) { FILE *f = (*it)->fp; fclose(f); CURL *single = (CURL *)((*it)->curl); curl_multi_remove_handle(multi_handle, single); curl_easy_cleanup(single); } // Check unfinished files and notify errors, succeed files will be renamed from temporary file name to real name for (auto it = _progDatas.begin(); it != _progDatas.end(); ++it) { ProgressData *data = *it; if (data->downloaded < data->totalToDownload || data->totalToDownload == 0) { this->notifyError(ErrorCode::NETWORK, "Unable to download file", data->customId); } else { _fileUtils->renameFile(data->path, data->name + TEMP_EXT, data->name); } } clearBatchDownloadData(); }