void Downloader::downloadSync(const std::string &srcUrl, const std::string &storagePath, const std::string &customId/* = ""*/)
{
FileDescriptor fDesc;
ProgressData pData;
prepareDownload(srcUrl, storagePath, customId, false, &fDesc, &pData);
if (fDesc.fp != NULL)
{
download(srcUrl, customId, fDesc, pData);
}
}
void Downloader::downloadAsync(const std::string &srcUrl, const std::string &storagePath, const std::string &customId/* = ""*/)
{
FileDescriptor fDesc;
ProgressData pData;
prepareDownload(srcUrl, storagePath, customId, false, &fDesc, &pData);
if (fDesc.fp != NULL)
{
auto t = std::thread(&Downloader::download, this, srcUrl, customId, fDesc, pData);
t.detach();
}
}
void HttpDownload::slot_downloadFinished(QNetworkReply *reply)
{
/**
* @brief The file name
*/
QString fileName;
/**
* @brief Flag for extraction
*/
bool needsExtraction = false;
// Check in which download phase we are
switch(downloadPhase)
{
case 0:
fileName = "launcher/downloads/files_game.txt";
break;
case 1:
fileName = fileList.at(filesLeft);
break;
}
// Open the file to write to
QFile file(installationPath + fileName);
if(file.open(QIODevice::WriteOnly))
{
// Open a stream to write into the file
QDataStream stream(&file);
// Get the size of the file
int size = reply->size();
// Add size to status int for displaying
totalSizeDownloadedCurrent = qint64(0);
totalSizeDownloaded += qint64(size);
// Get the data of the file
QByteArray temp = reply->readAll();
// Write the file
stream.writeRawData(temp, size);
// Set exe permissions
file.setPermissions(QFile::ExeUser | QFile::ExeGroup | QFile::ExeOwner | QFile::WriteUser | QFile::WriteGroup | QFile::WriteOwner | QFile::ReadGroup | QFile::ReadOwner | QFile::ReadUser);
// Close the file
file.close();
// Check if it is an archive
if(fileName.endsWith("tar.xz"))
{
// Create instance of archiver
archive->setFile(installationPath + fileName, installationPath);
// Connect as start as thread
connect(archive, SIGNAL(finished()),SLOT(slot_extractionDone()));
archive->start();
// Set flag and status
needsExtraction = true;
status = "extracting";
}
}
// If phase 0 take future steps
if(!needsExtraction)
{
switch(downloadPhase)
{
case 0:
prepareDownload();
downloadPhase = 1;
break;
case 1:
getNextFile();
break;
}
}
}
void Downloader::groupBatchDownload(const DownloadUnits &units)
{
CURLM* multi_handle = curl_multi_init();
int still_running = 0;
for (auto it = units.cbegin(); it != units.cend(); ++it)
{
DownloadUnit unit = it->second;
std::string srcUrl = unit.srcUrl;
std::string storagePath = unit.storagePath;
std::string customId = unit.customId;
FileDescriptor *fDesc = new FileDescriptor();
ProgressData *data = new ProgressData();
prepareDownload(srcUrl, storagePath, customId, unit.resumeDownload, fDesc, data);
if (fDesc->fp != NULL)
{
CURL* curl = curl_easy_init();
curl_easy_setopt(curl, CURLOPT_URL, srcUrl.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fileWriteFunc);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fDesc->fp);
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false);
curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, batchDownloadProgressFunc);
curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, data);
curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
if (_connectionTimeout) curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, _connectionTimeout);
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, LOW_SPEED_LIMIT);
curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, LOW_SPEED_TIME);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true);
curl_easy_setopt(curl, CURLOPT_MAXREDIRS, MAX_REDIRS);
// Resuming download support
if (_supportResuming && unit.resumeDownload)
{
// Check already downloaded size for current download unit
long size = _fileUtils->getFileSize(storagePath + TEMP_EXT);
if (size != -1)
{
curl_easy_setopt(curl, CURLOPT_RESUME_FROM_LARGE, size);
}
}
fDesc->curl = curl;
CURLMcode code = curl_multi_add_handle(multi_handle, curl);
if (code != CURLM_OK)
{
// Avoid memory leak
fclose(fDesc->fp);
delete data;
delete fDesc;
std::string msg = StringUtils::format("Unable to add curl handler for %s: [curl error]%s", customId.c_str(), curl_multi_strerror(code));
this->notifyError(msg, code, customId);
}
else
{
// Add to list for tracking
_progDatas.push_back(data);
_files.push_back(fDesc);
}
}
}
// Query multi perform
CURLMcode curlm_code = CURLM_CALL_MULTI_PERFORM;
while(CURLM_CALL_MULTI_PERFORM == curlm_code) {
curlm_code = curl_multi_perform(multi_handle, &still_running);
}
if (curlm_code != CURLM_OK) {
std::string msg = StringUtils::format("Unable to continue the download process: [curl error]%s", curl_multi_strerror(curlm_code));
this->notifyError(msg, curlm_code);
}
else
{
bool failed = false;
while (still_running > 0 && !failed)
{
// set a suitable timeout to play around with
struct timeval select_tv;
long curl_timeo = -1;
select_tv.tv_sec = 1;
select_tv.tv_usec = 0;
curl_multi_timeout(multi_handle, &curl_timeo);
if(curl_timeo >= 0) {
select_tv.tv_sec = curl_timeo / 1000;
if(select_tv.tv_sec > 1)
select_tv.tv_sec = 1;
else
select_tv.tv_usec = (curl_timeo % 1000) * 1000;
}
int rc;
fd_set fdread;
fd_set fdwrite;
fd_set fdexcep;
int maxfd = -1;
FD_ZERO(&fdread);
FD_ZERO(&fdwrite);
FD_ZERO(&fdexcep);
// FIXME: when jenkins migrate to ubuntu, we should remove this hack code
#if (CC_TARGET_PLATFORM == CC_PLATFORM_LINUX)
curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);
rc = select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &select_tv);
#else
rc = curl_multi_wait(multi_handle, nullptr, 0, MAX_WAIT_MSECS, &maxfd);
#endif
switch(rc)
{
case -1:
failed = true;
break;
case 0:
default:
curlm_code = CURLM_CALL_MULTI_PERFORM;
while(CURLM_CALL_MULTI_PERFORM == curlm_code) {
curlm_code = curl_multi_perform(multi_handle, &still_running);
}
if (curlm_code != CURLM_OK) {
std::string msg = StringUtils::format("Unable to continue the download process: [curl error]%s", curl_multi_strerror(curlm_code));
this->notifyError(msg, curlm_code);
}
break;
}
}
}
// Clean up and close files
curl_multi_cleanup(multi_handle);
for (auto it = _files.begin(); it != _files.end(); ++it)
{
FILE *f = (*it)->fp;
fclose(f);
CURL *single = (CURL *)((*it)->curl);
curl_multi_remove_handle(multi_handle, single);
curl_easy_cleanup(single);
}
// Check unfinished files and notify errors, succeed files will be renamed from temporary file name to real name
for (auto it = _progDatas.begin(); it != _progDatas.end(); ++it) {
ProgressData *data = *it;
if (data->downloaded < data->totalToDownload || data->totalToDownload == 0)
{
this->notifyError(ErrorCode::NETWORK, "Unable to download file", data->customId);
}
else
{
_fileUtils->renameFile(data->path, data->name + TEMP_EXT, data->name);
}
}
clearBatchDownloadData();
}
void HttpDownload::slot_downloadFinished(QNetworkReply *reply)
{
/**
* @brief The file name
*/
QString fileName;
/**
* @brief Flag for extraction
*/
bool needsExtraction = false;
// Check in which download phase we are
switch(downloadPhase)
{
case 0:
fileName = "launcher/downloads/files_game.txt";
break;
case 1:
fileName = fileList.at(filesLeft);
break;
}
Logging::addEntry(LOG_LEVEL_INF, "Savinge file: " + fileName, FUNCTION_NAME);
// Open the file to write to
QFile file(installationPath + fileName);
if(file.open(QIODevice::WriteOnly))
{
// Open a stream to write into the file
QDataStream stream(&file);
// Get the size of the file
qint64 size = reply->size();
// Add size to status int for displaying
totalSizeDownloadedCurrent = qint64(0);
totalSizeDownloaded += qint64(size);
// Get the data of the file
QByteArray temp = reply->read(size); // Note: ReadAll will crash 32 bit builds!
// Write the file
stream.writeRawData(temp, size);
// Set exe permissions
file.setPermissions(QFile::ExeUser | QFile::ExeGroup | QFile::ExeOwner | QFile::WriteUser | QFile::WriteGroup | QFile::WriteOwner | QFile::ReadGroup | QFile::ReadOwner | QFile::ReadUser);
// Close the file
file.close();
// Check if it is an archive
if(fileName.endsWith("xz"))
{
Logging::addEntry(LOG_LEVEL_INF, "Found archive, starting extracting", FUNCTION_NAME);
// Get folder structure
QStringList extractFolders = fileName.split("/");
QString extractFolder;
for(int i = 0; i < extractFolders.size()-1; i++)
{
extractFolder += extractFolders.at(i) + "/";
}
// Create sub folder to extract into
QString targetFolder = extractFolders.last().split(".").at(0);
QDir().mkdir(installationPath + extractFolder + targetFolder);
// Create instance of archiver
archive->setFile(extractFolders.last(), installationPath + extractFolder, targetFolder);
// Connect as start as thread
connect(archive, SIGNAL(extraction_done(bool)),SLOT(slot_extractionDone(bool)));
archive->start();
// Set flag and status
needsExtraction = true;
status = "extracting";
}
}
else
{
Logging::addEntry(LOG_LEVEL_ERR, "File could not be opened", FUNCTION_NAME);
}
// If phase 0 take future steps
if(!needsExtraction)
{
switch(downloadPhase)
{
case 0:
prepareDownload();
downloadPhase = 1;
break;
case 1:
getNextFile();
break;
}
}
}