void PropagateUploadFileNG::slotDeleteJobFinished() { auto job = qobject_cast<DeleteJob *>(sender()); ASSERT(job); _jobs.remove(_jobs.indexOf(job)); QNetworkReply::NetworkError err = job->reply()->error(); if (err != QNetworkReply::NoError && err != QNetworkReply::ContentNotFoundError) { const int httpStatus = job->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); SyncFileItem::Status status = classifyError(err, httpStatus); if (status == SyncFileItem::FatalError) { abortWithError(status, job->errorString()); return; } else { qCWarning(lcPropagateUpload) << "DeleteJob errored out" << job->errorString() << job->reply()->url(); _removeJobError = true; // Let the other jobs finish } } if (_jobs.isEmpty()) { propagator()->_activeJobList.removeOne(this); if (_removeJobError) { // There was an error removing some files, just start over startNewUpload(); } else { startNextChunk(); } } }
void PropagateUploadFileNG::slotMkColFinished(QNetworkReply::NetworkError) { propagator()->_activeJobList.removeOne(this); auto job = qobject_cast<MkColJob *>(sender()); slotJobDestroyed(job); // remove it from the _jobs list QNetworkReply::NetworkError err = job->reply()->error(); _item->_httpErrorCode = job->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); if (err != QNetworkReply::NoError || _item->_httpErrorCode != 201) { SyncFileItem::Status status = classifyError(err, _item->_httpErrorCode, &propagator()->_anotherSyncNeeded); abortWithError(status, job->errorStringParsingBody()); return; } startNextChunk(); }
void PropagateUploadFileNG::slotPropfindFinished() { auto job = qobject_cast<LsColJob *>(sender()); slotJobDestroyed(job); // remove it from the _jobs list propagator()->_activeJobList.removeOne(this); _currentChunk = 0; _sent = 0; while (_serverChunks.contains(_currentChunk)) { _sent += _serverChunks[_currentChunk].size; _serverChunks.remove(_currentChunk); ++_currentChunk; } if (_sent > _item->_size) { // Normally this can't happen because the size is xor'ed with the transfer id, and it is // therefore impossible that there is more data on the server than on the file. qCCritical(lcPropagateUpload) << "Inconsistency while resuming " << _item->_file << ": the size on the server (" << _sent << ") is bigger than the size of the file (" << _item->_size << ")"; startNewUpload(); return; } qCInfo(lcPropagateUpload) << "Resuming " << _item->_file << " from chunk " << _currentChunk << "; sent =" << _sent; if (!_serverChunks.isEmpty()) { qCInfo(lcPropagateUpload) << "To Delete" << _serverChunks.keys(); propagator()->_activeJobList.append(this); _removeJobError = false; // Make sure that if there is a "hole" and then a few more chunks, on the server // we should remove the later chunks. Otherwise when we do dynamic chunk sizing, we may end up // with corruptions if there are too many chunks, or if we abort and there are still stale chunks. for (auto it = _serverChunks.begin(); it != _serverChunks.end(); ++it) { auto job = new DeleteJob(propagator()->account(), Utility::concatUrlPath(chunkUrl(), it->originalName), this); QObject::connect(job, &DeleteJob::finishedSignal, this, &PropagateUploadFileNG::slotDeleteJobFinished); _jobs.append(job); job->start(); } _serverChunks.clear(); return; } startNextChunk(); }
void PropagateUploadFileQNAM::slotPutFinished() { PUTFileJob *job = qobject_cast<PUTFileJob *>(sender()); Q_ASSERT(job); slotJobDestroyed(job); // remove it from the _jobs list qDebug() << Q_FUNC_INFO << job->reply()->request().url() << "FINISHED WITH STATUS" << job->reply()->error() << (job->reply()->error() == QNetworkReply::NoError ? QLatin1String("") : job->reply()->errorString()) << job->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute) << job->reply()->attribute(QNetworkRequest::HttpReasonPhraseAttribute); _propagator->_activeJobList.removeOne(this); if (_finished) { // We have sent the finished signal already. We don't need to handle any remaining jobs return; } QNetworkReply::NetworkError err = job->reply()->error(); #if QT_VERSION < QT_VERSION_CHECK(5, 4, 2) if (err == QNetworkReply::OperationCanceledError && job->reply()->property(owncloudShouldSoftCancelPropertyName).isValid()) { // Abort the job and try again later. // This works around a bug in QNAM wich might reuse a non-empty buffer for the next request. qDebug() << "Forcing job abort on HTTP connection reset with Qt < 5.4.2."; _propagator->_anotherSyncNeeded = true; abortWithError(SyncFileItem::SoftError, tr("Forcing job abort on HTTP connection reset with Qt < 5.4.2.")); return; } #endif if (err != QNetworkReply::NoError) { _item->_httpErrorCode = job->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); if(checkForProblemsWithShared(_item->_httpErrorCode, tr("The file was edited locally but is part of a read only share. " "It is restored and your edit is in the conflict file."))) { return; } QByteArray replyContent = job->reply()->readAll(); qDebug() << replyContent; // display the XML error in the debug QString errorString = errorMessage(job->errorString(), replyContent); if (job->reply()->hasRawHeader("OC-ErrorString")) { errorString = job->reply()->rawHeader("OC-ErrorString"); } if (_item->_httpErrorCode == 412) { // Precondition Failed: Maybe the bad etag is in the database, we need to clear the // parent folder etag so we won't read from DB next sync. _propagator->_journal->avoidReadFromDbOnNextSync(_item->_file); _propagator->_anotherSyncNeeded = true; } SyncFileItem::Status status = classifyError(err, _item->_httpErrorCode, &_propagator->_anotherSyncNeeded); abortWithError(status, errorString); return; } _item->_httpErrorCode = job->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); // The server needs some time to process the request and provide us with a poll URL if (_item->_httpErrorCode == 202) { _finished = true; QString path = QString::fromUtf8(job->reply()->rawHeader("OC-Finish-Poll")); if (path.isEmpty()) { done(SyncFileItem::NormalError, tr("Poll URL missing")); return; } startPollJob(path); return; } // Check the file again post upload. // Two cases must be considered separately: If the upload is finished, // the file is on the server and has a changed ETag. In that case, // the etag has to be properly updated in the client journal, and because // of that we can bail out here with an error. But we can reschedule a // sync ASAP. // But if the upload is ongoing, because not all chunks were uploaded // yet, the upload can be stopped and an error can be displayed, because // the server hasn't registered the new file yet. QByteArray etag = getEtagFromReply(job->reply()); bool finished = etag.length() > 0; // Check if the file still exists const QString fullFilePath(_propagator->getFilePath(_item->_file)); if( !FileSystem::fileExists(fullFilePath) ) { if (!finished) { abortWithError(SyncFileItem::SoftError, tr("The local file was removed during sync.")); return; } else { _propagator->_anotherSyncNeeded = true; } } // Check whether the file changed since discovery. if (! FileSystem::verifyFileUnchanged(fullFilePath, _item->_size, _item->_modtime)) { _propagator->_anotherSyncNeeded = true; if( !finished ) { abortWithError(SyncFileItem::SoftError, tr("Local file changed during sync.")); // FIXME: the legacy code was retrying for a few seconds. // and also checking that after the last chunk, and removed the file in case of INSTRUCTION_NEW return; } } if (!finished) { // Proceed to next chunk. if (_currentChunk >= _chunkCount) { if (!_jobs.empty()) { // just wait for the other job to finish. return; } _finished = true; done(SyncFileItem::NormalError, tr("The server did not acknowledge the last chunk. (No e-tag was present)")); return; } // Deletes an existing blacklist entry on successful chunk upload if (_item->_hasBlacklistEntry) { _propagator->_journal->wipeErrorBlacklistEntry(_item->_file); _item->_hasBlacklistEntry = false; } SyncJournalDb::UploadInfo pi; pi._valid = true; auto currentChunk = job->_chunk; foreach (auto *job, _jobs) { // Take the minimum finished one if (auto putJob = qobject_cast<PUTFileJob*>(job)) { currentChunk = qMin(currentChunk, putJob->_chunk - 1); } } pi._chunk = (currentChunk + _startChunk + 1) % _chunkCount ; // next chunk to start with pi._transferid = _transferId; pi._modtime = Utility::qDateTimeFromTime_t(_item->_modtime); _propagator->_journal->setUploadInfo(_item->_file, pi); _propagator->_journal->commit("Upload info"); startNextChunk(); return; } // the following code only happens after all chunks were uploaded. _finished = true; // the file id should only be empty for new files up- or downloaded QByteArray fid = job->reply()->rawHeader("OC-FileID"); if( !fid.isEmpty() ) { if( !_item->_fileId.isEmpty() && _item->_fileId != fid ) { qDebug() << "WARN: File ID changed!" << _item->_fileId << fid; } _item->_fileId = fid; } _item->_etag = etag; _item->_responseTimeStamp = job->responseTimestamp(); if (job->reply()->rawHeader("X-OC-MTime") != "accepted") { // X-OC-MTime is supported since owncloud 5.0. But not when chunking. // Normally Owncloud 6 always puts X-OC-MTime qWarning() << "Server does not support X-OC-MTime" << job->reply()->rawHeader("X-OC-MTime"); // Well, the mtime was not set done(SyncFileItem::SoftError, "Server does not support X-OC-MTime"); } // performance logging _item->_requestDuration = _stopWatch.stop(); qDebug() << "*==* duration UPLOAD" << _item->_size << _stopWatch.durationOfLap(QLatin1String("ContentChecksum")) << _stopWatch.durationOfLap(QLatin1String("TransmissionChecksum")) << _item->_requestDuration; finalize(*_item); }
void PropagateUploadFileQNAM::startNextChunk() { if (_propagator->_abortRequested.fetchAndAddRelaxed(0)) return; if (! _jobs.isEmpty() && _currentChunk + _startChunk >= _chunkCount - 1) { // Don't do parallel upload of chunk if this might be the last chunk because the server cannot handle that // https://github.com/owncloud/core/issues/11106 // We return now and when the _jobs are finished we will proceed with the last chunk // NOTE: Some other parts of the code such as slotUploadProgress also assume that the last chunk // is sent last. return; } quint64 fileSize = _item->_size; QMap<QByteArray, QByteArray> headers; headers["OC-Total-Length"] = QByteArray::number(fileSize); headers["OC-Async"] = "1"; headers["OC-Chunk-Size"]= QByteArray::number(quint64(chunkSize())); headers["Content-Type"] = "application/octet-stream"; headers["X-OC-Mtime"] = QByteArray::number(qint64(_item->_modtime)); if(_item->_file.contains(".sys.admin#recall#")) { // This is a file recall triggered by the admin. Note: the // recall list file created by the admin and downloaded by the // client (.sys.admin#recall#) also falls into this category // (albeit users are not supposed to mess up with it) // We use a special tag header so that the server may decide to store this file away in some admin stage area // And not directly in the user's area (which would trigger redownloads etc). headers["OC-Tag"] = ".sys.admin#recall#"; } if (!_item->_etag.isEmpty() && _item->_etag != "empty_etag" && _item->_instruction != CSYNC_INSTRUCTION_NEW // On new files never send a If-Match && _item->_instruction != CSYNC_INSTRUCTION_TYPE_CHANGE && !_deleteExisting ) { // We add quotes because the owncloud server always adds quotes around the etag, and // csync_owncloud.c's owncloud_file_id always strips the quotes. headers["If-Match"] = '"' + _item->_etag + '"'; } QString path = _item->_file; UploadDevice *device = new UploadDevice(&_propagator->_bandwidthManager); qint64 chunkStart = 0; qint64 currentChunkSize = fileSize; bool isFinalChunk = false; if (_chunkCount > 1) { int sendingChunk = (_currentChunk + _startChunk) % _chunkCount; // XOR with chunk size to make sure everything goes well if chunk size changes between runs uint transid = _transferId ^ chunkSize(); qDebug() << "Upload chunk" << sendingChunk << "of" << _chunkCount << "transferid(remote)=" << transid; path += QString("-chunking-%1-%2-%3").arg(transid).arg(_chunkCount).arg(sendingChunk); headers["OC-Chunked"] = "1"; chunkStart = chunkSize() * quint64(sendingChunk); currentChunkSize = chunkSize(); if (sendingChunk == _chunkCount - 1) { // last chunk currentChunkSize = (fileSize % chunkSize()); if( currentChunkSize == 0 ) { // if the last chunk pretends to be 0, its actually the full chunk size. currentChunkSize = chunkSize(); } isFinalChunk = true; } } else { // if there's only one chunk, it's the final one isFinalChunk = true; } if (isFinalChunk && !_transmissionChecksumType.isEmpty()) { headers[checkSumHeaderC] = makeChecksumHeader( _transmissionChecksumType, _transmissionChecksum); } const QString fileName = _propagator->getFilePath(_item->_file); if (! device->prepareAndOpen(fileName, chunkStart, currentChunkSize)) { qDebug() << "ERR: Could not prepare upload device: " << device->errorString(); // If the file is currently locked, we want to retry the sync // when it becomes available again. if (FileSystem::isFileLocked(fileName)) { emit _propagator->seenLockedFile(fileName); } // Soft error because this is likely caused by the user modifying his files while syncing abortWithError( SyncFileItem::SoftError, device->errorString() ); delete device; return; } // job takes ownership of device via a QScopedPointer. Job deletes itself when finishing PUTFileJob* job = new PUTFileJob(_propagator->account(), _propagator->_remoteFolder + path, device, headers, _currentChunk); _jobs.append(job); connect(job, SIGNAL(finishedSignal()), this, SLOT(slotPutFinished())); connect(job, SIGNAL(uploadProgress(qint64,qint64)), this, SLOT(slotUploadProgress(qint64,qint64))); connect(job, SIGNAL(uploadProgress(qint64,qint64)), device, SLOT(slotJobUploadProgress(qint64,qint64))); connect(job, SIGNAL(destroyed(QObject*)), this, SLOT(slotJobDestroyed(QObject*))); job->start(); _propagator->_activeJobList.append(this); _currentChunk++; bool parallelChunkUpload = true; QByteArray env = qgetenv("OWNCLOUD_PARALLEL_CHUNK"); if (!env.isEmpty()) { parallelChunkUpload = env != "false" && env != "0"; } else { int versionNum = _propagator->account()->serverVersionInt(); if (versionNum < 0x080003) { // Disable parallel chunk upload severs older than 8.0.3 to avoid too many // internal sever errors (#2743, #2938) parallelChunkUpload = false; } } if (_currentChunk + _startChunk >= _chunkCount - 1) { // Don't do parallel upload of chunk if this might be the last chunk because the server cannot handle that // https://github.com/owncloud/core/issues/11106 parallelChunkUpload = false; } if (parallelChunkUpload && (_propagator->_activeJobList.count() < _propagator->maximumActiveJob()) && _currentChunk < _chunkCount ) { startNextChunk(); } if (!parallelChunkUpload || _chunkCount - _currentChunk <= 0) { emit ready(); } }
void PropagateUploadFileQNAM::startNextChunk() { if (_propagator->_abortRequested.fetchAndAddRelaxed(0)) return; if (! _jobs.isEmpty() && _currentChunk + _startChunk >= _chunkCount - 1) { // Don't do parallel upload of chunk if this might be the last chunk because the server cannot handle that // https://github.com/owncloud/core/issues/11106 // We return now and when the _jobs will be finished we will proceed the last chunk return; } quint64 fileSize = _item._size; QMap<QByteArray, QByteArray> headers; headers["OC-Total-Length"] = QByteArray::number(fileSize); headers["OC-Async"] = "1"; headers["Content-Type"] = "application/octet-stream"; headers["X-OC-Mtime"] = QByteArray::number(qint64(_item._modtime)); if (!_item._etag.isEmpty() && _item._etag != "empty_etag" && _item._instruction != CSYNC_INSTRUCTION_NEW // On new files never send a If-Match ) { // We add quotes because the owncloud server always add quotes around the etag, and // csync_owncloud.c's owncloud_file_id always strip the quotes. headers["If-Match"] = '"' + _item._etag + '"'; } QString path = _item._file; UploadDevice *device = 0; if (_chunkCount > 1) { int sendingChunk = (_currentChunk + _startChunk) % _chunkCount; // XOR with chunk size to make sure everything goes well if chunk size change between runs uint transid = _transferId ^ chunkSize(); path += QString("-chunking-%1-%2-%3").arg(transid).arg(_chunkCount).arg(sendingChunk); headers["OC-Chunked"] = "1"; int currentChunkSize = chunkSize(); if (sendingChunk == _chunkCount - 1) { // last chunk currentChunkSize = (fileSize % chunkSize()); if( currentChunkSize == 0 ) { // if the last chunk pretents to be 0, its actually the full chunk size. currentChunkSize = chunkSize(); } } device = new UploadDevice(_file, chunkSize() * quint64(sendingChunk), currentChunkSize, &_propagator->_bandwidthManager); } else { device = new UploadDevice(_file, 0, fileSize, &_propagator->_bandwidthManager); } bool isOpen = true; if (!device->isOpen()) { isOpen = device->open(QIODevice::ReadOnly); } if( isOpen ) { PUTFileJob* job = new PUTFileJob(AccountManager::instance()->account(), _propagator->_remoteFolder + path, device, headers, _currentChunk); _jobs.append(job); job->setTimeout(_propagator->httpTimeout() * 1000); connect(job, SIGNAL(finishedSignal()), this, SLOT(slotPutFinished())); connect(job, SIGNAL(uploadProgress(qint64,qint64)), this, SLOT(slotUploadProgress(qint64,qint64))); connect(job, SIGNAL(uploadProgress(qint64,qint64)), device, SLOT(slotJobUploadProgress(qint64,qint64))); connect(job, SIGNAL(destroyed(QObject*)), this, SLOT(slotJobDestroyed(QObject*))); job->start(); _propagator->_activeJobs++; _currentChunk++; QByteArray env = qgetenv("OWNCLOUD_PARALLEL_CHUNK"); bool parallelChunkUpload = env=="true" || env =="1";; if (_currentChunk + _startChunk >= _chunkCount - 1) { // Don't do parallel upload of chunk if this might be the last chunk because the server cannot handle that // https://github.com/owncloud/core/issues/11106 parallelChunkUpload = false; } if (parallelChunkUpload && (_propagator->_activeJobs < _propagator->maximumActiveJob()) && _currentChunk < _chunkCount ) { startNextChunk(); } if (!parallelChunkUpload || _chunkCount - _currentChunk <= 0) { emitReady(); } } else {
void PropagateUploadFileQNAM::slotPutFinished() { PUTFileJob *job = qobject_cast<PUTFileJob *>(sender()); Q_ASSERT(job); slotJobDestroyed(job); // remove it from the _jobs list qDebug() << Q_FUNC_INFO << job->reply()->request().url() << "FINISHED WITH STATUS" << job->reply()->error() << (job->reply()->error() == QNetworkReply::NoError ? QLatin1String("") : job->reply()->errorString()) << job->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute) << job->reply()->attribute(QNetworkRequest::HttpReasonPhraseAttribute); _propagator->_activeJobs--; if (_finished) { // We have send the finished signal already. We don't need to handle any remaining jobs return; } QNetworkReply::NetworkError err = job->reply()->error(); if (err != QNetworkReply::NoError) { _item._httpErrorCode = job->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); if(checkForProblemsWithShared(_item._httpErrorCode, tr("The file was edited locally but is part of a read only share. " "It is restored and your edit is in the conflict file."))) { return; } QString errorString = job->errorString(); QByteArray replyContent = job->reply()->readAll(); qDebug() << replyContent; // display the XML error in the debug QRegExp rx("<s:message>(.*)</s:message>"); // Issue #1366: display server exception if (rx.indexIn(QString::fromUtf8(replyContent)) != -1) { errorString += QLatin1String(" (") + rx.cap(1) + QLatin1Char(')'); } if (job->reply()->hasRawHeader("OC-ErrorString")) { errorString = job->reply()->rawHeader("OC-ErrorString"); } if (_item._httpErrorCode == 412) { // Precondition Failed: Maybe the bad etag is in the database, we need to clear the // parent folder etag so we won't read from DB next sync. _propagator->_journal->avoidReadFromDbOnNextSync(_item._file); _propagator->_anotherSyncNeeded = true; } abortWithError(classifyError(err, _item._httpErrorCode), errorString); return; } _item._httpErrorCode = job->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); // The server needs some time to process the request and provide with a poll URL if (_item._httpErrorCode == 202) { _finished = true; QString path = QString::fromUtf8(job->reply()->rawHeader("OC-Finish-Poll")); if (path.isEmpty()) { done(SyncFileItem::NormalError, tr("Poll URL missing")); return; } startPollJob(path); return; } // Check the file again post upload. // Two cases must be considered separately: If the upload is finished, // the file is on the server and has a changed ETag. In that case, // the etag has to be properly updated in the client journal, and because // of that we can bail out here with an error. But we can reschedule a // sync ASAP. // But if the upload is ongoing, because not all chunks were uploaded // yet, the upload can be stopped and an error can be displayed, because // the server hasn't registered the new file yet. QByteArray etag = getEtagFromReply(job->reply()); bool finished = etag.length() > 0; // Check if the file still exists const QString fullFilePath(_propagator->getFilePath(_item._file)); if( !FileSystem::fileExists(fullFilePath) ) { if (!finished) { abortWithError(SyncFileItem::SoftError, tr("The local file was removed during sync.")); return; } else { _propagator->_anotherSyncNeeded = true; } } // compare expected and real modification time of the file and size const time_t new_mtime = FileSystem::getModTime(fullFilePath); const quint64 new_size = static_cast<quint64>(FileSystem::getSize(fullFilePath)); QFileInfo fi(_propagator->getFilePath(_item._file)); if (new_mtime != _item._modtime || new_size != _item._size) { qDebug() << "The local file has changed during upload:" << "mtime: " << _item._modtime << "<->" << new_mtime << ", size: " << _item._size << "<->" << new_size << ", QFileInfo: " << Utility::qDateTimeToTime_t(fi.lastModified()) << fi.lastModified(); _propagator->_anotherSyncNeeded = true; if( !finished ) { abortWithError(SyncFileItem::SoftError, tr("Local file changed during sync.")); // FIXME: the legacy code was retrying for a few seconds. // and also checking that after the last chunk, and removed the file in case of INSTRUCTION_NEW return; } } if (!finished) { // Proceed to next chunk. if (_currentChunk >= _chunkCount) { if (!_jobs.empty()) { // just wait for the other job to finish. return; } _finished = true; done(SyncFileItem::NormalError, tr("The server did not acknowledge the last chunk. (No e-tag were present)")); return; } SyncJournalDb::UploadInfo pi; pi._valid = true; auto currentChunk = job->_chunk; foreach (auto *job, _jobs) { // Take the minimum finished one currentChunk = qMin(currentChunk, job->_chunk - 1); } pi._chunk = (currentChunk + _startChunk + 1) % _chunkCount ; // next chunk to start with pi._transferid = _transferId; pi._modtime = Utility::qDateTimeFromTime_t(_item._modtime); _propagator->_journal->setUploadInfo(_item._file, pi); _propagator->_journal->commit("Upload info"); startNextChunk(); return; }
void PropagateUploadFileQNAM::startNextChunk() { if (_propagator->_abortRequested.fetchAndAddRelaxed(0)) return; if (! _jobs.isEmpty() && _currentChunk + _startChunk >= _chunkCount - 1) { // Don't do parallel upload of chunk if this might be the last chunk because the server cannot handle that // https://github.com/owncloud/core/issues/11106 // We return now and when the _jobs will be finished we will proceed the last chunk // NOTE: Some other part of the code such as slotUploadProgress assume also that the last chunk // is sent last. return; } quint64 fileSize = _item._size; QMap<QByteArray, QByteArray> headers; headers["OC-Total-Length"] = QByteArray::number(fileSize); headers["OC-Async"] = "1"; headers["OC-Chunk-Size"]= QByteArray::number(quint64(chunkSize())); headers["Content-Type"] = "application/octet-stream"; headers["X-OC-Mtime"] = QByteArray::number(qint64(_item._modtime)); if (!_item._etag.isEmpty() && _item._etag != "empty_etag" && _item._instruction != CSYNC_INSTRUCTION_NEW // On new files never send a If-Match ) { // We add quotes because the owncloud server always add quotes around the etag, and // csync_owncloud.c's owncloud_file_id always strip the quotes. headers["If-Match"] = '"' + _item._etag + '"'; } QString path = _item._file; UploadDevice *device = new UploadDevice(&_propagator->_bandwidthManager); qint64 chunkStart = 0; qint64 currentChunkSize = fileSize; if (_chunkCount > 1) { int sendingChunk = (_currentChunk + _startChunk) % _chunkCount; // XOR with chunk size to make sure everything goes well if chunk size change between runs uint transid = _transferId ^ chunkSize(); path += QString("-chunking-%1-%2-%3").arg(transid).arg(_chunkCount).arg(sendingChunk); headers["OC-Chunked"] = "1"; chunkStart = chunkSize() * quint64(sendingChunk); currentChunkSize = chunkSize(); if (sendingChunk == _chunkCount - 1) { // last chunk currentChunkSize = (fileSize % chunkSize()); if( currentChunkSize == 0 ) { // if the last chunk pretents to be 0, its actually the full chunk size. currentChunkSize = chunkSize(); } } } if (! device->prepareAndOpen(_propagator->getFilePath(_item._file), chunkStart, currentChunkSize)) { qDebug() << "ERR: Could not prepare upload device: " << device->errorString(); // Soft error because this is likely caused by the user modifying his files while syncing abortWithError( SyncFileItem::SoftError, device->errorString() ); delete device; return; } // job takes ownership of device via a QScopedPointer. Job deletes itself when finishing PUTFileJob* job = new PUTFileJob(_propagator->account(), _propagator->_remoteFolder + path, device, headers, _currentChunk); _jobs.append(job); connect(job, SIGNAL(finishedSignal()), this, SLOT(slotPutFinished())); connect(job, SIGNAL(uploadProgress(qint64,qint64)), this, SLOT(slotUploadProgress(qint64,qint64))); connect(job, SIGNAL(uploadProgress(qint64,qint64)), device, SLOT(slotJobUploadProgress(qint64,qint64))); connect(job, SIGNAL(destroyed(QObject*)), this, SLOT(slotJobDestroyed(QObject*))); job->start(); _propagator->_activeJobs++; _currentChunk++; bool parallelChunkUpload = true; QByteArray env = qgetenv("OWNCLOUD_PARALLEL_CHUNK"); if (!env.isEmpty()) { parallelChunkUpload = env != "false" && env != "0"; } else { auto version = _propagator->account()->serverVersion(); auto components = version.split('.'); int versionNum = (components.value(0).toInt() << 16) + (components.value(1).toInt() << 8) + components.value(2).toInt(); if (versionNum < 0x080003) { // Disable parallel chunk upload severs older than 8.0.3 to avoid too many // internal sever errors (#2743, #2938) parallelChunkUpload = false; } } if (_currentChunk + _startChunk >= _chunkCount - 1) { // Don't do parallel upload of chunk if this might be the last chunk because the server cannot handle that // https://github.com/owncloud/core/issues/11106 parallelChunkUpload = false; } if (parallelChunkUpload && (_propagator->_activeJobs < _propagator->maximumActiveJob()) && _currentChunk < _chunkCount ) { startNextChunk(); } if (!parallelChunkUpload || _chunkCount - _currentChunk <= 0) { emit ready(); } }
void PropagateUploadFileNG::slotPutFinished() { PUTFileJob *job = qobject_cast<PUTFileJob *>(sender()); ASSERT(job); slotJobDestroyed(job); // remove it from the _jobs list propagator()->_activeJobList.removeOne(this); if (_finished) { // We have sent the finished signal already. We don't need to handle any remaining jobs return; } QNetworkReply::NetworkError err = job->reply()->error(); if (err != QNetworkReply::NoError) { _item->_httpErrorCode = job->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); commonErrorHandling(job); return; } ENFORCE(_sent <= _item->_size, "can't send more than size"); // Adjust the chunk size for the time taken. // // Dynamic chunk sizing is enabled if the server configured a // target duration for each chunk upload. double targetDuration = propagator()->syncOptions()._targetChunkUploadDuration; if (targetDuration > 0) { double uploadTime = job->msSinceStart() + 1; // add one to avoid div-by-zero auto predictedGoodSize = static_cast<quint64>( _currentChunkSize / uploadTime * targetDuration); // The whole targeting is heuristic. The predictedGoodSize will fluctuate // quite a bit because of external factors (like available bandwidth) // and internal factors (like number of parallel uploads). // // We use an exponential moving average here as a cheap way of smoothing // the chunk sizes a bit. quint64 targetSize = (propagator()->_chunkSize + predictedGoodSize) / 2; // Adjust the dynamic chunk size _chunkSize used for sizing of the item's chunks to be send propagator()->_chunkSize = qBound( propagator()->syncOptions()._minChunkSize, targetSize, propagator()->syncOptions()._maxChunkSize); qCInfo(lcPropagateUpload) << "Chunked upload of" << _currentChunkSize << "bytes took" << uploadTime << "ms, desired is" << targetDuration << "ms, expected good chunk size is" << predictedGoodSize << "bytes and nudged next chunk size to " << propagator()->_chunkSize << "bytes"; } bool finished = _sent == _item->_size; // Check if the file still exists const QString fullFilePath(propagator()->getFilePath(_item->_file)); if (!FileSystem::fileExists(fullFilePath)) { if (!finished) { abortWithError(SyncFileItem::SoftError, tr("The local file was removed during sync.")); return; } else { propagator()->_anotherSyncNeeded = true; } } // Check whether the file changed since discovery. if (!FileSystem::verifyFileUnchanged(fullFilePath, _item->_size, _item->_modtime)) { propagator()->_anotherSyncNeeded = true; if (!finished) { abortWithError(SyncFileItem::SoftError, tr("Local file changed during sync.")); return; } } if (!finished) { // Deletes an existing blacklist entry on successful chunk upload if (_item->_hasBlacklistEntry) { propagator()->_journal->wipeErrorBlacklistEntry(_item->_file); _item->_hasBlacklistEntry = false; } // Reset the error count on successful chunk upload auto uploadInfo = propagator()->_journal->getUploadInfo(_item->_file); uploadInfo._errorCount = 0; propagator()->_journal->setUploadInfo(_item->_file, uploadInfo); propagator()->_journal->commit("Upload info"); } startNextChunk(); }