void NetworkManager::putRequest(const QString &url, const QByteArray &data)
{
request.setUrl(QUrl(url));
reply = networkManager->put(request, data);
connect(reply, SIGNAL(finished()), this, SLOT(slotUploadFinished()));
connect(reply, SIGNAL(uploadProgress(qint64, qint64)), this, SLOT(slotUploadProgress(qint64, qint64)));
connectErrorHandlers();
}
void PropagateUploadFileQNAM::startNextChunk()
{
if (_propagator->_abortRequested.fetchAndAddRelaxed(0))
return;
if (! _jobs.isEmpty() && _currentChunk + _startChunk >= _chunkCount - 1) {
// Don't do parallel upload of chunk if this might be the last chunk because the server cannot handle that
// https://github.com/owncloud/core/issues/11106
// We return now and when the _jobs are finished we will proceed with the last chunk
// NOTE: Some other parts of the code such as slotUploadProgress also assume that the last chunk
// is sent last.
return;
}
quint64 fileSize = _item->_size;
QMap<QByteArray, QByteArray> headers;
headers["OC-Total-Length"] = QByteArray::number(fileSize);
headers["OC-Async"] = "1";
headers["OC-Chunk-Size"]= QByteArray::number(quint64(chunkSize()));
headers["Content-Type"] = "application/octet-stream";
headers["X-OC-Mtime"] = QByteArray::number(qint64(_item->_modtime));
if(_item->_file.contains(".sys.admin#recall#")) {
// This is a file recall triggered by the admin. Note: the
// recall list file created by the admin and downloaded by the
// client (.sys.admin#recall#) also falls into this category
// (albeit users are not supposed to mess up with it)
// We use a special tag header so that the server may decide to store this file away in some admin stage area
// And not directly in the user's area (which would trigger redownloads etc).
headers["OC-Tag"] = ".sys.admin#recall#";
}
if (!_item->_etag.isEmpty() && _item->_etag != "empty_etag"
&& _item->_instruction != CSYNC_INSTRUCTION_NEW // On new files never send a If-Match
&& _item->_instruction != CSYNC_INSTRUCTION_TYPE_CHANGE
&& !_deleteExisting
) {
// We add quotes because the owncloud server always adds quotes around the etag, and
// csync_owncloud.c's owncloud_file_id always strips the quotes.
headers["If-Match"] = '"' + _item->_etag + '"';
}
QString path = _item->_file;
UploadDevice *device = new UploadDevice(&_propagator->_bandwidthManager);
qint64 chunkStart = 0;
qint64 currentChunkSize = fileSize;
bool isFinalChunk = false;
if (_chunkCount > 1) {
int sendingChunk = (_currentChunk + _startChunk) % _chunkCount;
// XOR with chunk size to make sure everything goes well if chunk size changes between runs
uint transid = _transferId ^ chunkSize();
qDebug() << "Upload chunk" << sendingChunk << "of" << _chunkCount << "transferid(remote)=" << transid;
path += QString("-chunking-%1-%2-%3").arg(transid).arg(_chunkCount).arg(sendingChunk);
headers["OC-Chunked"] = "1";
chunkStart = chunkSize() * quint64(sendingChunk);
currentChunkSize = chunkSize();
if (sendingChunk == _chunkCount - 1) { // last chunk
currentChunkSize = (fileSize % chunkSize());
if( currentChunkSize == 0 ) { // if the last chunk pretends to be 0, its actually the full chunk size.
currentChunkSize = chunkSize();
}
isFinalChunk = true;
}
} else {
// if there's only one chunk, it's the final one
isFinalChunk = true;
}
if (isFinalChunk && !_transmissionChecksumType.isEmpty()) {
headers[checkSumHeaderC] = makeChecksumHeader(
_transmissionChecksumType, _transmissionChecksum);
}
const QString fileName = _propagator->getFilePath(_item->_file);
if (! device->prepareAndOpen(fileName, chunkStart, currentChunkSize)) {
qDebug() << "ERR: Could not prepare upload device: " << device->errorString();
// If the file is currently locked, we want to retry the sync
// when it becomes available again.
if (FileSystem::isFileLocked(fileName)) {
emit _propagator->seenLockedFile(fileName);
}
// Soft error because this is likely caused by the user modifying his files while syncing
abortWithError( SyncFileItem::SoftError, device->errorString() );
delete device;
return;
}
// job takes ownership of device via a QScopedPointer. Job deletes itself when finishing
PUTFileJob* job = new PUTFileJob(_propagator->account(), _propagator->_remoteFolder + path, device, headers, _currentChunk);
_jobs.append(job);
connect(job, SIGNAL(finishedSignal()), this, SLOT(slotPutFinished()));
connect(job, SIGNAL(uploadProgress(qint64,qint64)), this, SLOT(slotUploadProgress(qint64,qint64)));
connect(job, SIGNAL(uploadProgress(qint64,qint64)), device, SLOT(slotJobUploadProgress(qint64,qint64)));
connect(job, SIGNAL(destroyed(QObject*)), this, SLOT(slotJobDestroyed(QObject*)));
job->start();
_propagator->_activeJobList.append(this);
_currentChunk++;
bool parallelChunkUpload = true;
QByteArray env = qgetenv("OWNCLOUD_PARALLEL_CHUNK");
if (!env.isEmpty()) {
parallelChunkUpload = env != "false" && env != "0";
} else {
int versionNum = _propagator->account()->serverVersionInt();
if (versionNum < 0x080003) {
// Disable parallel chunk upload severs older than 8.0.3 to avoid too many
// internal sever errors (#2743, #2938)
parallelChunkUpload = false;
}
}
if (_currentChunk + _startChunk >= _chunkCount - 1) {
// Don't do parallel upload of chunk if this might be the last chunk because the server cannot handle that
// https://github.com/owncloud/core/issues/11106
parallelChunkUpload = false;
}
if (parallelChunkUpload && (_propagator->_activeJobList.count() < _propagator->maximumActiveJob())
&& _currentChunk < _chunkCount ) {
startNextChunk();
}
if (!parallelChunkUpload || _chunkCount - _currentChunk <= 0) {
emit ready();
}
}
void PropagateUploadFileQNAM::startNextChunk()
{
if (_propagator->_abortRequested.fetchAndAddRelaxed(0))
return;
if (! _jobs.isEmpty() && _currentChunk + _startChunk >= _chunkCount - 1) {
// Don't do parallel upload of chunk if this might be the last chunk because the server cannot handle that
// https://github.com/owncloud/core/issues/11106
// We return now and when the _jobs will be finished we will proceed the last chunk
return;
}
quint64 fileSize = _item._size;
QMap<QByteArray, QByteArray> headers;
headers["OC-Total-Length"] = QByteArray::number(fileSize);
headers["OC-Async"] = "1";
headers["Content-Type"] = "application/octet-stream";
headers["X-OC-Mtime"] = QByteArray::number(qint64(_item._modtime));
if (!_item._etag.isEmpty() && _item._etag != "empty_etag" &&
_item._instruction != CSYNC_INSTRUCTION_NEW // On new files never send a If-Match
) {
// We add quotes because the owncloud server always add quotes around the etag, and
// csync_owncloud.c's owncloud_file_id always strip the quotes.
headers["If-Match"] = '"' + _item._etag + '"';
}
QString path = _item._file;
UploadDevice *device = 0;
if (_chunkCount > 1) {
int sendingChunk = (_currentChunk + _startChunk) % _chunkCount;
// XOR with chunk size to make sure everything goes well if chunk size change between runs
uint transid = _transferId ^ chunkSize();
path += QString("-chunking-%1-%2-%3").arg(transid).arg(_chunkCount).arg(sendingChunk);
headers["OC-Chunked"] = "1";
int currentChunkSize = chunkSize();
if (sendingChunk == _chunkCount - 1) { // last chunk
currentChunkSize = (fileSize % chunkSize());
if( currentChunkSize == 0 ) { // if the last chunk pretents to be 0, its actually the full chunk size.
currentChunkSize = chunkSize();
}
}
device = new UploadDevice(_file, chunkSize() * quint64(sendingChunk), currentChunkSize, &_propagator->_bandwidthManager);
} else {
device = new UploadDevice(_file, 0, fileSize, &_propagator->_bandwidthManager);
}
bool isOpen = true;
if (!device->isOpen()) {
isOpen = device->open(QIODevice::ReadOnly);
}
if( isOpen ) {
PUTFileJob* job = new PUTFileJob(AccountManager::instance()->account(), _propagator->_remoteFolder + path, device, headers, _currentChunk);
_jobs.append(job);
job->setTimeout(_propagator->httpTimeout() * 1000);
connect(job, SIGNAL(finishedSignal()), this, SLOT(slotPutFinished()));
connect(job, SIGNAL(uploadProgress(qint64,qint64)), this, SLOT(slotUploadProgress(qint64,qint64)));
connect(job, SIGNAL(uploadProgress(qint64,qint64)), device, SLOT(slotJobUploadProgress(qint64,qint64)));
connect(job, SIGNAL(destroyed(QObject*)), this, SLOT(slotJobDestroyed(QObject*)));
job->start();
_propagator->_activeJobs++;
_currentChunk++;
QByteArray env = qgetenv("OWNCLOUD_PARALLEL_CHUNK");
bool parallelChunkUpload = env=="true" || env =="1";;
if (_currentChunk + _startChunk >= _chunkCount - 1) {
// Don't do parallel upload of chunk if this might be the last chunk because the server cannot handle that
// https://github.com/owncloud/core/issues/11106
parallelChunkUpload = false;
}
if (parallelChunkUpload && (_propagator->_activeJobs < _propagator->maximumActiveJob())
&& _currentChunk < _chunkCount ) {
startNextChunk();
}
if (!parallelChunkUpload || _chunkCount - _currentChunk <= 0) {
emitReady();
}
} else {
void PropagateUploadFileQNAM::startNextChunk()
{
if (_propagator->_abortRequested.fetchAndAddRelaxed(0))
return;
if (! _jobs.isEmpty() && _currentChunk + _startChunk >= _chunkCount - 1) {
// Don't do parallel upload of chunk if this might be the last chunk because the server cannot handle that
// https://github.com/owncloud/core/issues/11106
// We return now and when the _jobs will be finished we will proceed the last chunk
// NOTE: Some other part of the code such as slotUploadProgress assume also that the last chunk
// is sent last.
return;
}
quint64 fileSize = _item._size;
QMap<QByteArray, QByteArray> headers;
headers["OC-Total-Length"] = QByteArray::number(fileSize);
headers["OC-Async"] = "1";
headers["OC-Chunk-Size"]= QByteArray::number(quint64(chunkSize()));
headers["Content-Type"] = "application/octet-stream";
headers["X-OC-Mtime"] = QByteArray::number(qint64(_item._modtime));
if (!_item._etag.isEmpty() && _item._etag != "empty_etag" &&
_item._instruction != CSYNC_INSTRUCTION_NEW // On new files never send a If-Match
) {
// We add quotes because the owncloud server always add quotes around the etag, and
// csync_owncloud.c's owncloud_file_id always strip the quotes.
headers["If-Match"] = '"' + _item._etag + '"';
}
QString path = _item._file;
UploadDevice *device = new UploadDevice(&_propagator->_bandwidthManager);
qint64 chunkStart = 0;
qint64 currentChunkSize = fileSize;
if (_chunkCount > 1) {
int sendingChunk = (_currentChunk + _startChunk) % _chunkCount;
// XOR with chunk size to make sure everything goes well if chunk size change between runs
uint transid = _transferId ^ chunkSize();
path += QString("-chunking-%1-%2-%3").arg(transid).arg(_chunkCount).arg(sendingChunk);
headers["OC-Chunked"] = "1";
chunkStart = chunkSize() * quint64(sendingChunk);
currentChunkSize = chunkSize();
if (sendingChunk == _chunkCount - 1) { // last chunk
currentChunkSize = (fileSize % chunkSize());
if( currentChunkSize == 0 ) { // if the last chunk pretents to be 0, its actually the full chunk size.
currentChunkSize = chunkSize();
}
}
}
if (! device->prepareAndOpen(_propagator->getFilePath(_item._file), chunkStart, currentChunkSize)) {
qDebug() << "ERR: Could not prepare upload device: " << device->errorString();
// Soft error because this is likely caused by the user modifying his files while syncing
abortWithError( SyncFileItem::SoftError, device->errorString() );
delete device;
return;
}
// job takes ownership of device via a QScopedPointer. Job deletes itself when finishing
PUTFileJob* job = new PUTFileJob(_propagator->account(), _propagator->_remoteFolder + path, device, headers, _currentChunk);
_jobs.append(job);
connect(job, SIGNAL(finishedSignal()), this, SLOT(slotPutFinished()));
connect(job, SIGNAL(uploadProgress(qint64,qint64)), this, SLOT(slotUploadProgress(qint64,qint64)));
connect(job, SIGNAL(uploadProgress(qint64,qint64)), device, SLOT(slotJobUploadProgress(qint64,qint64)));
connect(job, SIGNAL(destroyed(QObject*)), this, SLOT(slotJobDestroyed(QObject*)));
job->start();
_propagator->_activeJobs++;
_currentChunk++;
bool parallelChunkUpload = true;
QByteArray env = qgetenv("OWNCLOUD_PARALLEL_CHUNK");
if (!env.isEmpty()) {
parallelChunkUpload = env != "false" && env != "0";
} else {
auto version = _propagator->account()->serverVersion();
auto components = version.split('.');
int versionNum = (components.value(0).toInt() << 16)
+ (components.value(1).toInt() << 8)
+ components.value(2).toInt();
if (versionNum < 0x080003) {
// Disable parallel chunk upload severs older than 8.0.3 to avoid too many
// internal sever errors (#2743, #2938)
parallelChunkUpload = false;
}
}
if (_currentChunk + _startChunk >= _chunkCount - 1) {
// Don't do parallel upload of chunk if this might be the last chunk because the server cannot handle that
// https://github.com/owncloud/core/issues/11106
parallelChunkUpload = false;
}
if (parallelChunkUpload && (_propagator->_activeJobs < _propagator->maximumActiveJob())
&& _currentChunk < _chunkCount ) {
startNextChunk();
}
if (!parallelChunkUpload || _chunkCount - _currentChunk <= 0) {
emit ready();
}
}
