void CSyncThread::handleSyncError(CSYNC *ctx, const char *state) {
CSYNC_STATUS err = csync_get_status( ctx );
const char *errMsg = csync_get_status_string( ctx );
QString errStr = csyncErrorToString(err);
if( errMsg ) {
if( !errStr.endsWith(" ")) {
errStr.append(" ");
}
errStr += QString::fromUtf8(errMsg);
}
// if there is csyncs url modifier in the error message, replace it.
if( errStr.contains("ownclouds://") ) errStr.replace("ownclouds://", "https://");
if( errStr.contains("owncloud://") ) errStr.replace("owncloud://", "http://");
qDebug() << " #### ERROR during "<< state << ": " << errStr;
if( CSYNC_STATUS_IS_EQUAL( err, CSYNC_STATUS_ABORTED) ) {
qDebug() << "Update phase was aborted by user!";
} else if( CSYNC_STATUS_IS_EQUAL( err, CSYNC_STATUS_SERVICE_UNAVAILABLE ) ||
CSYNC_STATUS_IS_EQUAL( err, CSYNC_STATUS_CONNECT_ERROR )) {
emit csyncUnavailable();
} else {
emit csyncError(errStr);
}
csync_commit(_csync_ctx);
emit finished();
_syncMutex.unlock();
_thread.quit();
}
~CSyncRunScopeHelper() {
csync_commit(_ctx);
qDebug() << "CSync run took " << _t.elapsed() << " Milliseconds";
emit(_parent->finished());
_parent->_syncMutex.unlock();
}
void CSyncThread::startNextTransfer()
{
while (_iterator < _syncedItems.size()) {
const SyncFileItem &item = _syncedItems.at(_iterator);
++_iterator;
while (!_directoriesToUpdate.isEmpty() && !item._file.startsWith(_directoriesToUpdate.last()._path)) {
// We are leaving a directory. Everything we needed to download from that directory is done.
// Update the directory etag in the database to the new one.
_journal->setFileRecord(_directoriesToUpdate.pop());
}
if (!_lastDeleted.isEmpty() &&
item._file.startsWith(_lastDeleted) ) {
if( item._instruction != CSYNC_INSTRUCTION_REMOVE ) {
qDebug() << "WRN: Child of a deleted directory has different instruction than delete."
<< item._file << _lastDeleted << item._instruction;
} else {
// If the item's name starts with the name of the previously deleted directory, we
// can assume this file was already destroyed by the previous call.
_journal->deleteFileRecord(item._file);
continue;
}
}
if (item._instruction == CSYNC_INSTRUCTION_SYNC || item._instruction == CSYNC_INSTRUCTION_NEW
|| item._instruction == CSYNC_INSTRUCTION_CONFLICT) {
slotProgress((item._dir != SyncFileItem::Up) ? Progress::StartDownload : Progress::StartUpload,
item._file, 0, item._size);
}
_propagator->propagate(item);
return; //propagate is async.
}
// We are finished !!
while (!_directoriesToUpdate.isEmpty()) {
// Save the etag of directories to the database.
_journal->setFileRecord(_directoriesToUpdate.pop());
}
// emit the treewalk results.
emit treeWalkResult(_syncedItems);
csync_commit(_csync_ctx);
qDebug() << "CSync run took " << _syncTime.elapsed() << " Milliseconds";
slotProgress(Progress::EndSync,QString(), 0 , 0);
emit finished();
_propagator.reset(0);
_syncMutex.unlock();
thread()->quit();
}
void SyncEngine::finalize()
{
csync_commit(_csync_ctx);
qDebug() << "CSync run took " << _stopWatch.addLapTime(QLatin1String("Sync Finished"));
_stopWatch.stop();
_propagator.reset(0);
_thread.quit();
_thread.wait();
_syncRunning = false;
emit finished();
}
void SyncEngine::finalize()
{
_thread.quit();
_thread.wait();
csync_commit(_csync_ctx);
qDebug() << "CSync run took " << _stopWatch.addLapTime(QLatin1String("Sync Finished"));
_stopWatch.stop();
_syncRunning = false;
emit finished();
// Delete the propagator only after emitting the signal.
_propagator.clear();
}
void CSyncThread::startNextTransfer()
{
while (_iterator < _syncedItems.size()) {
const SyncFileItem &item = _syncedItems.at(_iterator);
++_iterator;
while (!_directoriesToUpdate.isEmpty() && !item._file.startsWith(_directoriesToUpdate.last()._path)) {
_journal->setFileRecord(_directoriesToUpdate.pop());
}
if (!_lastDeleted.isEmpty() && item._file.startsWith(_lastDeleted)
&& item._instruction == CSYNC_INSTRUCTION_REMOVE) {
// If the item's name starts with the name of the previously deleted directory, we
// can assume this file was already destroyed by the previous recursive call.
_journal->deleteFileRecord(item._file);
continue;
}
if (item._instruction == CSYNC_INSTRUCTION_SYNC || item._instruction == CSYNC_INSTRUCTION_NEW
|| item._instruction == CSYNC_INSTRUCTION_CONFLICT) {
slotProgress((item._dir != SyncFileItem::Up) ? Progress::StartDownload : Progress::StartUpload,
item._file, 0, item._size);
}
_propagator->propagate(item);
return; //propagate is async.
}
while (!_directoriesToUpdate.isEmpty()) {
_journal->setFileRecord(_directoriesToUpdate.pop());
}
// Everything is finished.
_progressDataBase.save(_localPath);
// emit the treewalk results.
emit treeWalkResult(_syncedItems);
csync_commit(_csync_ctx);
qDebug() << "CSync run took " << _syncTime.elapsed() << " Milliseconds";
slotProgress(Progress::EndSync,QString(), 0 , 0);
emit finished();
_propagator.reset(0);
_syncMutex.unlock();
thread()->quit();
}
void CSyncThread::slotFinished()
{
// emit the treewalk results.
if( ! _journal->postSyncCleanup( _seenFiles ) ) {
qDebug() << "Cleaning of synced ";
}
_journal->commit("All Finished.", false);
emit treeWalkResult(_syncedItems);
csync_commit(_csync_ctx);
qDebug() << "CSync run took " << _syncTime.elapsed() << " Milliseconds";
slotProgress(Progress::EndSync,SyncFileItem(), 0 , 0);
emit finished();
_propagator.reset(0);
_syncMutex.unlock();
_thread.quit();
}
void CSyncThread::handleSyncError(CSYNC *ctx, const char *state) {
CSYNC_STATUS err = CSYNC_STATUS(csync_get_status( ctx ));
const char *errMsg = csync_get_status_string( ctx );
QString errStr = csyncErrorToString(err);
if( errMsg ) {
errStr += QString::fromUtf8(errMsg);
}
qDebug() << " #### ERROR during "<< state << ": " << errStr;
if( CSYNC_STATUS_IS_EQUAL( err, CSYNC_STATUS_SERVICE_UNAVAILABLE ) ||
CSYNC_STATUS_IS_EQUAL( err, CSYNC_STATUS_CONNECT_ERROR )) {
emit csyncUnavailable();
} else {
emit csyncError(errStr);
}
csync_commit(_csync_ctx);
emit finished();
_syncMutex.unlock();
thread()->quit();
}
void SyncEngine::finalize()
{
_thread.quit();
_thread.wait();
#ifdef USE_NEON
// De-init the neon HTTP(S) connections
owncloud_commit(_csync_ctx);
#endif
csync_commit(_csync_ctx);
qDebug() << "CSync run took " << _stopWatch.addLapTime(QLatin1String("Sync Finished"));
_stopWatch.stop();
_syncRunning = false;
emit finished();
// Delete the propagator only after emitting the signal.
_propagator.clear();
}
void SyncEngine::slotDiscoveryJobFinished(int discoveryResult)
{
// To clean the progress info
emit folderDiscovered(false, QString());
if (discoveryResult < 0 ) {
handleSyncError(_csync_ctx, "csync_update");
return;
}
qDebug() << "<<#### Discovery end #################################################### " << _stopWatch.addLapTime(QLatin1String("Discovery Finished"));
// Sanity check
if (!_journal->isConnected()) {
qDebug() << "Bailing out, DB failure";
emit csyncError(tr("Cannot open the sync journal"));
finalize(false);
return;
} else {
// Commits a possibly existing (should not though) transaction and starts a new one for the propagate phase
_journal->commitIfNeededAndStartNewTransaction("Post discovery");
}
if( csync_reconcile(_csync_ctx) < 0 ) {
handleSyncError(_csync_ctx, "csync_reconcile");
return;
}
qDebug() << "<<#### Reconcile end #################################################### " << _stopWatch.addLapTime(QLatin1String("Reconcile Finished"));
_hasNoneFiles = false;
_hasRemoveFile = false;
bool walkOk = true;
_seenFiles.clear();
_temporarilyUnavailablePaths.clear();
if( csync_walk_local_tree(_csync_ctx, &treewalkLocal, 0) < 0 ) {
qDebug() << "Error in local treewalk.";
walkOk = false;
}
if( walkOk && csync_walk_remote_tree(_csync_ctx, &treewalkRemote, 0) < 0 ) {
qDebug() << "Error in remote treewalk.";
}
if (_csync_ctx->remote.root_perms) {
_remotePerms[QLatin1String("")] = _csync_ctx->remote.root_perms;
qDebug() << "Permissions of the root folder: " << _remotePerms[QLatin1String("")];
}
// Re-init the csync context to free memory
csync_commit(_csync_ctx);
// The map was used for merging trees, convert it to a list:
_syncedItems = _syncItemMap.values().toVector();
_syncItemMap.clear(); // free memory
// Adjust the paths for the renames.
for (SyncFileItemVector::iterator it = _syncedItems.begin();
it != _syncedItems.end(); ++it) {
(*it)->_file = adjustRenamedPath((*it)->_file);
}
// Sort items per destination
std::sort(_syncedItems.begin(), _syncedItems.end());
// make sure everything is allowed
checkForPermission();
// To announce the beginning of the sync
emit aboutToPropagate(_syncedItems);
// it's important to do this before ProgressInfo::start(), to announce start of new sync
emit transmissionProgress(*_progressInfo);
_progressInfo->start();
if (!_hasNoneFiles && _hasRemoveFile) {
qDebug() << Q_FUNC_INFO << "All the files are going to be changed, asking the user";
bool cancel = false;
emit aboutToRemoveAllFiles(_syncedItems.first()->_direction, &cancel);
if (cancel) {
qDebug() << Q_FUNC_INFO << "Abort sync";
finalize(false);
return;
}
}
// post update phase script: allow to tweak stuff by a custom script in debug mode.
if( !qgetenv("OWNCLOUD_POST_UPDATE_SCRIPT").isEmpty() ) {
#ifndef NDEBUG
QString script = qgetenv("OWNCLOUD_POST_UPDATE_SCRIPT");
qDebug() << "OOO => Post Update Script: " << script;
QProcess::execute(script.toUtf8());
#else
qWarning() << "**** Attention: POST_UPDATE_SCRIPT installed, but not executed because compiled with NDEBUG";
#endif
}
// do a database commit
_journal->commit("post treewalk");
_propagator = QSharedPointer<OwncloudPropagator>(
new OwncloudPropagator (_account, _localPath, _remoteUrl, _remotePath, _journal));
connect(_propagator.data(), SIGNAL(itemCompleted(const SyncFileItem &, const PropagatorJob &)),
this, SLOT(slotItemCompleted(const SyncFileItem &, const PropagatorJob &)));
connect(_propagator.data(), SIGNAL(progress(const SyncFileItem &,quint64)),
this, SLOT(slotProgress(const SyncFileItem &,quint64)));
connect(_propagator.data(), SIGNAL(finished()), this, SLOT(slotFinished()), Qt::QueuedConnection);
// apply the network limits to the propagator
setNetworkLimits(_uploadLimit, _downloadLimit);
deleteStaleDownloadInfos();
deleteStaleUploadInfos();
deleteStaleErrorBlacklistEntries();
_journal->commit("post stale entry removal");
// Emit the started signal only after the propagator has been set up.
if (_needsUpdate)
emit(started());
_propagator->start(_syncedItems);
qDebug() << "<<#### Post-Reconcile end #################################################### " << _stopWatch.addLapTime(QLatin1String("Post-Reconcile Finished"));
}
void CSyncThread::startSync()
{
if (!_syncMutex.tryLock()) {
qDebug() << Q_FUNC_INFO << "WARNING: Another sync seems to be running. Not starting a new one.";
return;
}
if( ! _csync_ctx ) {
qDebug() << "XXXXXXXXXXXXXXXX FAIL: do not have csync_ctx!";
}
qDebug() << Q_FUNC_INFO << "Sync started";
qDebug() << "starting to sync " << qApp->thread() << QThread::currentThread();
_syncedItems.clear();
_needsUpdate = false;
csync_resume(_csync_ctx);
if (!_journal->exists()) {
qDebug() << "=====sync looks new (no DB exists), activating recursive PROPFIND if csync supports it";
bool no_recursive_propfind = false;
csync_set_module_property(_csync_ctx, "no_recursive_propfind", &no_recursive_propfind);
} else {
// retrieve the file count from the db and close it afterwards because
// csync_update also opens the database.
int fileRecordCount = 0;
fileRecordCount = _journal->getFileRecordCount();
_journal->close();
if( fileRecordCount == -1 ) {
qDebug() << "No way to create a sync journal!";
emit csyncError(tr("Unable to initialize a sync journal."));
csync_commit(_csync_ctx);
emit finished();
_syncMutex.unlock();
_thread.quit();
return;
// database creation error!
} else if ( fileRecordCount < 50 ) {
qDebug() << "=====sync DB has only" << fileRecordCount << "items, enable recursive PROPFIND if csync supports it";
bool no_recursive_propfind = false;
csync_set_module_property(_csync_ctx, "no_recursive_propfind", &no_recursive_propfind);
} else {
qDebug() << "=====sync with existing DB";
}
}
csync_set_module_property(_csync_ctx, "csync_context", _csync_ctx);
csync_set_userdata(_csync_ctx, this);
// TODO: This should be a part of this method, but we don't have
// any way to get "session_key" module property from csync. Had we
// have it, then we could keep this code and remove it from
// AbstractCredentials implementations.
if (Account *account = AccountManager::instance()->account()) {
account->credentials()->syncContextPreStart(_csync_ctx);
} else {
qDebug() << Q_FUNC_INFO << "No default Account object, huh?";
}
// if (_lastAuthCookies.length() > 0) {
// // Stuff cookies inside csync, then we can avoid the intermediate HTTP 401 reply
// // when https://github.com/owncloud/core/pull/4042 is merged.
// QString cookiesAsString;
// foreach(QNetworkCookie c, _lastAuthCookies) {
// cookiesAsString += c.name();
// cookiesAsString += '=';
// cookiesAsString += c.value();
// cookiesAsString += "; ";
// }
// csync_set_module_property(_csync_ctx, "session_key", cookiesAsString.to
// }
// csync_set_auth_callback( _csync_ctx, getauth );
csync_set_log_callback( csyncLogCatcher );
//csync_set_log_level( 11 ); don't set the loglevel here, it shall be done by folder.cpp or owncloudcmd.cpp
_syncTime.start();
// Only used for the updater progress as we use the new propagator right now which does its own thing
csync_set_progress_callback(_csync_ctx, csyncthread_updater_progress_callback);
qDebug() << "#### Update start #################################################### >>";
UpdateJob *job = new UpdateJob(_csync_ctx);
job->moveToThread(&_thread);
connect(job, SIGNAL(finished(int)), this, SLOT(slotUpdateFinished(int)));
QMetaObject::invokeMethod(job, "start", Qt::QueuedConnection);
}
void CSyncThread::startSync()
{
if (!_syncMutex.tryLock()) {
qDebug() << Q_FUNC_INFO << "WARNING: Another sync seems to be running. Not starting a new one.";
return;
}
if( ! _csync_ctx ) {
qDebug() << "XXXXXXXXXXXXXXXX FAIL: do not have csync_ctx!";
}
qDebug() << Q_FUNC_INFO << "Sync started";
qDebug() << "starting to sync " << qApp->thread() << QThread::currentThread();
_syncedItems.clear();
_needsUpdate = false;
_abortRequestedMutex.lock();
if (!_abortRequested.fetchAndAddRelease(0)) {
csync_resume(_csync_ctx);
}
_abortRequestedMutex.unlock();
// maybe move this somewhere else where it can influence a running sync?
MirallConfigFile cfg;
if (!_journal->exists()) {
qDebug() << "=====sync looks new (no DB exists), activating recursive PROPFIND if csync supports it";
bool no_recursive_propfind = false;
csync_set_module_property(_csync_ctx, "no_recursive_propfind", &no_recursive_propfind);
} else {
// retrieve the file count from the db and close it afterwards because
// csync_update also opens the database.
int fileRecordCount = 0;
fileRecordCount = _journal->getFileRecordCount();
_journal->close();
if( fileRecordCount == -1 ) {
qDebug() << "No way to create a sync journal!";
emit csyncError(tr("Unable to initialize a sync journal."));
csync_commit(_csync_ctx);
emit finished();
_syncMutex.unlock();
thread()->quit();
return;
// database creation error!
} else if ( fileRecordCount < 50 ) {
qDebug() << "=====sync DB has only" << fileRecordCount << "items, enable recursive PROPFIND if csync supports it";
bool no_recursive_propfind = false;
csync_set_module_property(_csync_ctx, "no_recursive_propfind", &no_recursive_propfind);
} else {
qDebug() << "=====sync with existing DB";
}
}
csync_set_module_property(_csync_ctx, "csync_context", _csync_ctx);
csync_set_userdata(_csync_ctx, this);
// TODO: This should be a part of this method, but we don't have
// any way to get "session_key" module property from csync. Had we
// have it, then we could keep this code and remove it from
// AbstractCredentials implementations.
if (Account *account = AccountManager::instance()->account()) {
account->credentials()->syncContextPreStart(_csync_ctx);
} else {
qDebug() << Q_FUNC_INFO << "No default Account object, huh?";
}
// if (_lastAuthCookies.length() > 0) {
// // Stuff cookies inside csync, then we can avoid the intermediate HTTP 401 reply
// // when https://github.com/owncloud/core/pull/4042 is merged.
// QString cookiesAsString;
// foreach(QNetworkCookie c, _lastAuthCookies) {
// cookiesAsString += c.name();
// cookiesAsString += '=';
// cookiesAsString += c.value();
// cookiesAsString += "; ";
// }
// csync_set_module_property(_csync_ctx, "session_key", cookiesAsString.to
// }
// csync_set_auth_callback( _csync_ctx, getauth );
csync_set_log_callback( csyncLogCatcher );
csync_set_log_level( 11 );
_syncTime.start();
QElapsedTimer updateTime;
updateTime.start();
qDebug() << "#### Update start #################################################### >>";
if( csync_update(_csync_ctx) < 0 ) {
handleSyncError(_csync_ctx, "csync_update");
return;
}
qDebug() << "<<#### Update end #################################################### " << updateTime.elapsed();
if( csync_reconcile(_csync_ctx) < 0 ) {
handleSyncError(_csync_ctx, "csync_reconcile");
return;
}
slotProgress(Progress::StartSync, SyncFileItem(), 0, 0);
_progressInfo = Progress::Info();
_hasFiles = false;
bool walkOk = true;
_seenFiles.clear();
if( csync_walk_local_tree(_csync_ctx, &treewalkLocal, 0) < 0 ) {
qDebug() << "Error in local treewalk.";
walkOk = false;
}
if( walkOk && csync_walk_remote_tree(_csync_ctx, &treewalkRemote, 0) < 0 ) {
qDebug() << "Error in remote treewalk.";
}
// Adjust the paths for the renames.
for (SyncFileItemVector::iterator it = _syncedItems.begin();
it != _syncedItems.end(); ++it) {
it->_file = adjustRenamedPath(it->_file);
}
if (!_hasFiles && !_syncedItems.isEmpty()) {
qDebug() << Q_FUNC_INFO << "All the files are going to be removed, asking the user";
bool cancel = false;
emit aboutToRemoveAllFiles(_syncedItems.first()._dir, &cancel);
if (cancel) {
qDebug() << Q_FUNC_INFO << "Abort sync";
return;
}
}
if (_needsUpdate)
emit(started());
ne_session_s *session = 0;
// that call to set property actually is a get which will return the session
// FIXME add a csync_get_module_property to csync
csync_set_module_property(_csync_ctx, "get_dav_session", &session);
Q_ASSERT(session);
_propagator.reset(new OwncloudPropagator (session, _localPath, _remotePath,
_journal, &_abortRequested));
connect(_propagator.data(), SIGNAL(completed(SyncFileItem)),
this, SLOT(transferCompleted(SyncFileItem)), Qt::QueuedConnection);
connect(_propagator.data(), SIGNAL(progress(Progress::Kind,SyncFileItem,quint64,quint64)),
this, SLOT(slotProgress(Progress::Kind,SyncFileItem,quint64,quint64)));
connect(_propagator.data(), SIGNAL(progressChanged(qint64)), this, SLOT(slotProgressChanged(qint64)));
connect(_propagator.data(), SIGNAL(finished()), this, SLOT(slotFinished()));
int downloadLimit = 0;
if (cfg.useDownloadLimit()) {
downloadLimit = cfg.downloadLimit() * 1000;
}
_propagator->_downloadLimit = downloadLimit;
int uploadLimit = -75; // 75%
int useUpLimit = cfg.useUploadLimit();
if ( useUpLimit >= 1) {
uploadLimit = cfg.uploadLimit() * 1000;
} else if (useUpLimit == 0) {
uploadLimit = 0;
}
_propagator->_uploadLimit = uploadLimit;
_propagator->start(_syncedItems);
}
void CSyncThread::transferCompleted(const SyncFileItem &item)
{
qDebug() << Q_FUNC_INFO << item._file << item._status << item._errorString;
/* Update the _syncedItems vector */
// Search for the item in the starting from _iterator because it should be a bit before it.
// This works because SyncFileItem::operator== only compare the file name;
int idx = _syncedItems.lastIndexOf(item, _iterator);
if (idx >= 0) {
_syncedItems[idx]._instruction = item._instruction;
_syncedItems[idx]._errorString = item._errorString;
_syncedItems[idx]._status = item._status;
}
/* Remember deleted directory */
if (item._isDirectory && item._instruction == CSYNC_INSTRUCTION_DELETED) {
_lastDeleted = item._file;
} else {
_lastDeleted.clear();
}
/* Update the database */
if (item._instruction == CSYNC_INSTRUCTION_DELETED) {
_journal->deleteFileRecord(item._originalFile);
if (!item._renameTarget.isEmpty()) {
SyncJournalFileRecord record(item, _localPath + item._renameTarget);
record._path = item._renameTarget;
_journal->setFileRecord(record);
}
} else if(item._instruction == CSYNC_INSTRUCTION_ERROR) {
// Don't update parents directories
_directoriesToUpdate.clear();
} else if (item._isDirectory) {
// directory must not be saved to the db before we finished processing them.
SyncJournalFileRecord record(item, _localPath + item._file);
_directoriesToUpdate.push(record);
} else if(item._instruction == CSYNC_INSTRUCTION_UPDATED) {
SyncJournalFileRecord record(item, _localPath + item._file);
_journal->setFileRecord(record);
slotProgress((item._dir != SyncFileItem::Up) ? Progress::EndDownload : Progress::EndUpload,
item._file, item._size, item._size);
_progressInfo.current_file_no++;
_progressInfo.overall_current_bytes += item._size;
}
/* Start the transfer of the next file or abort if there is an error */
if (item._status != SyncFileItem::FatalError) {
startNextTransfer();
} else {
emit treeWalkResult(_syncedItems);
emit csyncError(item._errorString);
emit finished();
csync_commit(_csync_ctx);
_syncMutex.unlock();
thread()->quit();
}
}
