int SyncEngine::treewalkFile( TREE_WALK_FILE *file, bool remote )
{
if( ! file ) return -1;
SyncFileItem item;
item._file = QString::fromUtf8( file->path );
item._originalFile = item._file;
item._instruction = file->instruction;
item._direction = SyncFileItem::None;
item._fileId = file->file_id;
if (file->directDownloadUrl) {
item._directDownloadUrl = QString::fromUtf8( file->directDownloadUrl );
}
if (file->directDownloadCookies) {
item._directDownloadCookies = QString::fromUtf8( file->directDownloadCookies );
}
if (file->remotePerm) {
item._remotePerm = QByteArray(file->remotePerm);
}
// record the seen files to be able to clean the journal later
_seenFiles[item._file] = QString();
switch(file->error_status) {
case CSYNC_STATUS_OK:
break;
case CSYNC_STATUS_INDIVIDUAL_IS_SYMLINK:
item._errorString = tr("Symbolic links are not supported in syncing.");
break;
case CSYNC_STATUS_INDIVIDUAL_IGNORE_LIST:
item._errorString = tr("File is listed on the ignore list.");
break;
case CSYNC_STATUS_INDIVIDUAL_IS_INVALID_CHARS:
item._errorString = tr("File contains invalid characters that can not be synced cross platform.");
break;
case CYSNC_STATUS_FILE_LOCKED_OR_OPEN:
item._errorString = QLatin1String("File locked"); // don't translate, internal use!
break;
default:
Q_ASSERT("Non handled error-status");
/* No error string */
}
item._isDirectory = file->type == CSYNC_FTW_TYPE_DIR;
item._modtime = file->modtime;
item._etag = file->etag;
item._size = file->size;
item._inode = file->inode;
item._should_update_etag = file->should_update_etag;
switch( file->type ) {
case CSYNC_FTW_TYPE_DIR:
item._type = SyncFileItem::Directory;
break;
case CSYNC_FTW_TYPE_FILE:
item._type = SyncFileItem::File;
break;
case CSYNC_FTW_TYPE_SLINK:
item._type = SyncFileItem::SoftLink;
break;
default:
item._type = SyncFileItem::UnknownType;
}
SyncFileItem::Direction dir;
int re = 0;
switch(file->instruction) {
case CSYNC_INSTRUCTION_NONE:
if (file->should_update_etag && !item._isDirectory) {
// Update the database now already (new fileid or etag or remotePerm)
_journal->setFileRecord(SyncJournalFileRecord(item, _localPath + item._file));
item._should_update_etag = false;
}
if (item._isDirectory && remote) {
// Because we want still to update etags of directories
dir = SyncFileItem::None;
} else {
// No need to do anything.
_hasFiles = true;
emit syncItemDiscovered(item);
return re;
}
break;
case CSYNC_INSTRUCTION_RENAME:
dir = !remote ? SyncFileItem::Down : SyncFileItem::Up;
item._renameTarget = QString::fromUtf8( file->rename_path );
if (item._isDirectory)
_renamedFolders.insert(item._file, item._renameTarget);
break;
case CSYNC_INSTRUCTION_REMOVE:
dir = !remote ? SyncFileItem::Down : SyncFileItem::Up;
break;
case CSYNC_INSTRUCTION_CONFLICT:
case CSYNC_INSTRUCTION_IGNORE:
case CSYNC_INSTRUCTION_ERROR:
dir = SyncFileItem::None;
break;
case CSYNC_INSTRUCTION_EVAL:
case CSYNC_INSTRUCTION_NEW:
case CSYNC_INSTRUCTION_SYNC:
case CSYNC_INSTRUCTION_STAT_ERROR:
default:
dir = remote ? SyncFileItem::Down : SyncFileItem::Up;
break;
}
item._direction = dir;
// check for blacklisting of this item.
// if the item is on blacklist, the instruction was set to IGNORE
checkBlacklisting( &item );
if (file->instruction != CSYNC_INSTRUCTION_IGNORE
&& file->instruction != CSYNC_INSTRUCTION_REMOVE
&& file->instruction != CSYNC_INSTRUCTION_ERROR) {
_hasFiles = true;
}
if (!item._isDirectory) {
_progressInfo._totalFileCount++;
if (Progress::isSizeDependent(file->instruction)) {
_progressInfo._totalSize += file->size;
}
}
_needsUpdate = true;
item.log._etag = file->etag;
item.log._fileId = file->file_id;
item.log._instruction = file->instruction;
item.log._modtime = file->modtime;
item.log._size = file->size;
item.log._other_etag = file->other.etag;
item.log._other_fileId = file->other.file_id;
item.log._other_instruction = file->other.instruction;
item.log._other_modtime = file->other.modtime;
item.log._other_size = file->other.size;
_syncedItems.append(item);
emit syncItemDiscovered(item);
return re;
}
int SyncEngine::treewalkFile( TREE_WALK_FILE *file, bool remote )
{
if( ! file ) return -1;
QTextCodec::ConverterState utf8State;
static QTextCodec *codec = QTextCodec::codecForName("UTF-8");
Q_ASSERT(codec);
QString fileUtf8 = codec->toUnicode(file->path, qstrlen(file->path), &utf8State);
QString renameTarget;
QString key = fileUtf8;
auto instruction = file->instruction;
if (utf8State.invalidChars > 0 || utf8State.remainingChars > 0) {
qWarning() << "File ignored because of invalid utf-8 sequence: " << file->path;
instruction = CSYNC_INSTRUCTION_IGNORE;
} else {
renameTarget = codec->toUnicode(file->rename_path, qstrlen(file->rename_path), &utf8State);
if (utf8State.invalidChars > 0 || utf8State.remainingChars > 0) {
qWarning() << "File ignored because of invalid utf-8 sequence in the rename_path: " << file->path << file->rename_path;
instruction = CSYNC_INSTRUCTION_IGNORE;
}
if (instruction == CSYNC_INSTRUCTION_RENAME) {
key = renameTarget;
}
}
// Gets a default-constructed SyncFileItemPtr or the one from the first walk (=local walk)
SyncFileItemPtr item = _syncItemMap.value(key);
if (!item)
item = SyncFileItemPtr(new SyncFileItem);
if (item->_file.isEmpty() || instruction == CSYNC_INSTRUCTION_RENAME) {
item->_file = fileUtf8;
}
item->_originalFile = item->_file;
if (item->_instruction == CSYNC_INSTRUCTION_NONE
|| (item->_instruction == CSYNC_INSTRUCTION_IGNORE && instruction != CSYNC_INSTRUCTION_NONE)) {
item->_instruction = instruction;
item->_modtime = file->modtime;
} else {
if (instruction != CSYNC_INSTRUCTION_NONE) {
qDebug() << "ERROR: Instruction" << item->_instruction << "vs" << instruction << "for" << fileUtf8;
Q_ASSERT(!"Instructions are both unequal NONE");
return -1;
}
}
if (file->file_id && file->file_id[0]) {
item->_fileId = file->file_id;
}
if (file->directDownloadUrl) {
item->_directDownloadUrl = QString::fromUtf8( file->directDownloadUrl );
}
if (file->directDownloadCookies) {
item->_directDownloadCookies = QString::fromUtf8( file->directDownloadCookies );
}
if (file->remotePerm && file->remotePerm[0]) {
item->_remotePerm = QByteArray(file->remotePerm);
}
item->_should_update_metadata = item->_should_update_metadata || file->should_update_metadata;
/* The flag "serverHasIgnoredFiles" is true if item in question is a directory
* that has children which are ignored in sync, either because the files are
* matched by an ignore pattern, or because they are hidden.
*
* Only the information about the server side ignored files is stored to the
* database and thus written to the item here. For the local repository its
* generated by the walk through the real file tree by discovery phase.
*
* It needs to go to the sync journal becasue the stat information about remote
* files are often read from database rather than being pulled from remote.
*/
if( remote ) {
item->_serverHasIgnoredFiles = (file->has_ignored_files > 0);
}
// record the seen files to be able to clean the journal later
_seenFiles.insert(item->_file);
if (!renameTarget.isEmpty()) {
// Yes, this records both the rename renameTarget and the original so we keep both in case of a rename
_seenFiles.insert(renameTarget);
}
if (remote && file->remotePerm && file->remotePerm[0]) {
_remotePerms[item->_file] = file->remotePerm;
}
switch(file->error_status) {
case CSYNC_STATUS_OK:
break;
case CSYNC_STATUS_INDIVIDUAL_IS_SYMLINK:
item->_errorString = tr("Symbolic links are not supported in syncing.");
break;
case CSYNC_STATUS_INDIVIDUAL_IGNORE_LIST:
item->_errorString = tr("File is listed on the ignore list.");
break;
case CSYNC_STATUS_INDIVIDUAL_IS_INVALID_CHARS:
item->_errorString = tr("Filename contains invalid characters that can not be synced cross platform.");
break;
case CSYNC_STATUS_INDIVIDUAL_EXCLUDE_LONG_FILENAME:
item->_errorString = tr("Filename is too long.");
break;
case CSYNC_STATUS_INDIVIDUAL_EXCLUDE_HIDDEN:
item->_errorString = tr("File is ignored because it's hidden.");
break;
case CYSNC_STATUS_FILE_LOCKED_OR_OPEN:
item->_errorString = QLatin1String("File locked"); // don't translate, internal use!
break;
case CSYNC_STATUS_INDIVIDUAL_STAT_FAILED:
item->_errorString = tr("Stat failed.");
break;
case CSYNC_STATUS_SERVICE_UNAVAILABLE:
item->_errorString = QLatin1String("Server temporarily unavailable.");
break;
case CSYNC_STATUS_STORAGE_UNAVAILABLE:
item->_errorString = QLatin1String("Directory temporarily not available on server.");
item->_status = SyncFileItem::SoftError;
_temporarilyUnavailablePaths.insert(item->_file);
break;
case CSYNC_STATUS_PERMISSION_DENIED:
item->_errorString = QLatin1String("Directory not accessible on client, permission denied.");
item->_status = SyncFileItem::SoftError;
break;
default:
Q_ASSERT("Non handled error-status");
/* No error string */
}
if (item->_instruction == CSYNC_INSTRUCTION_IGNORE && (utf8State.invalidChars > 0 || utf8State.remainingChars > 0)) {
item->_status = SyncFileItem::NormalError;
//item->_instruction = CSYNC_INSTRUCTION_ERROR;
item->_errorString = tr("Filename encoding is not valid");
}
item->_isDirectory = file->type == CSYNC_FTW_TYPE_DIR;
if (file->etag && file->etag[0]) {
item->_etag = file->etag;
}
item->_size = file->size;
if (!remote) {
item->_inode = file->inode;
}
switch( file->type ) {
case CSYNC_FTW_TYPE_DIR:
item->_type = SyncFileItem::Directory;
break;
case CSYNC_FTW_TYPE_FILE:
item->_type = SyncFileItem::File;
break;
case CSYNC_FTW_TYPE_SLINK:
item->_type = SyncFileItem::SoftLink;
break;
default:
item->_type = SyncFileItem::UnknownType;
}
SyncFileItem::Direction dir = SyncFileItem::None;
int re = 0;
switch(file->instruction) {
case CSYNC_INSTRUCTION_NONE:
if (remote && item->_should_update_metadata && !item->_isDirectory && item->_instruction == CSYNC_INSTRUCTION_NONE) {
// Update the database now already: New fileid or Etag or RemotePerm
// Or for files that were detected as "resolved conflict".
// They should have been a conflict because they both were new, or both
// had their local mtime or remote etag modified, but the size and mtime
// is the same on the server. This typically happens when the database is removed.
// Nothing will be done for those files, but we still need to update the database.
// Even if the mtime is different on the server, we always want to keep the mtime from
// the file system in the DB, this is to avoid spurious upload on the next sync
item->_modtime = file->other.modtime;
_journal->updateFileRecordMetadata(SyncJournalFileRecord(*item, _localPath + item->_file));
item->_should_update_metadata = false;
}
if (item->_isDirectory && file->should_update_metadata) {
// Because we want to still update etags of directories
dir = SyncFileItem::None;
} else {
// No need to do anything.
if (file->other.instruction == CSYNC_INSTRUCTION_NONE
// Directories with ignored files does not count as 'None'
&& (file->type != CSYNC_FTW_TYPE_DIR || !file->has_ignored_files)) {
_hasNoneFiles = true;
}
return re;
}
break;
case CSYNC_INSTRUCTION_RENAME:
dir = !remote ? SyncFileItem::Down : SyncFileItem::Up;
item->_renameTarget = renameTarget;
if (item->_isDirectory)
_renamedFolders.insert(item->_file, item->_renameTarget);
break;
case CSYNC_INSTRUCTION_REMOVE:
_hasRemoveFile = true;
dir = !remote ? SyncFileItem::Down : SyncFileItem::Up;
break;
case CSYNC_INSTRUCTION_CONFLICT:
case CSYNC_INSTRUCTION_IGNORE:
case CSYNC_INSTRUCTION_ERROR:
dir = SyncFileItem::None;
break;
case CSYNC_INSTRUCTION_EVAL:
case CSYNC_INSTRUCTION_NEW:
case CSYNC_INSTRUCTION_SYNC:
case CSYNC_INSTRUCTION_STAT_ERROR:
default:
dir = remote ? SyncFileItem::Down : SyncFileItem::Up;
if (!remote && file->instruction == CSYNC_INSTRUCTION_SYNC) {
// An upload of an existing file means that the file was left unchanged on the server
// This counts as a NONE for detecting if all the files on the server were changed
_hasNoneFiles = true;
}
break;
}
item->_direction = dir;
if (instruction != CSYNC_INSTRUCTION_NONE) {
// check for blacklisting of this item.
// if the item is on blacklist, the instruction was set to ERROR
checkErrorBlacklisting( *item );
}
_progressInfo->adjustTotalsForFile(*item);
_needsUpdate = true;
item->log._etag = file->etag;
item->log._fileId = file->file_id;
item->log._instruction = file->instruction;
item->log._modtime = file->modtime;
item->log._size = file->size;
item->log._other_etag = file->other.etag;
item->log._other_fileId = file->other.file_id;
item->log._other_instruction = file->other.instruction;
item->log._other_modtime = file->other.modtime;
item->log._other_size = file->other.size;
_syncItemMap.insert(key, item);
emit syncItemDiscovered(*item);
return re;
}
int SyncEngine::treewalkFile( TREE_WALK_FILE *file, bool remote )
{
if( ! file ) return -1;
QString fileUtf8 = QString::fromUtf8( file->path );
// Gets a default-contructed SyncFileItem or the one from the first walk (=local walk)
SyncFileItem item = _syncItemMap.value(fileUtf8);
item._file = fileUtf8;
item._originalFile = item._file;
if (item._instruction == CSYNC_INSTRUCTION_NONE
|| (item._instruction == CSYNC_INSTRUCTION_IGNORE && file->instruction != CSYNC_INSTRUCTION_NONE)) {
item._instruction = file->instruction;
item._modtime = file->modtime;
} else {
if (file->instruction != CSYNC_INSTRUCTION_NONE) {
Q_ASSERT(!"Instructions are both unequal NONE");
}
}
if (file->file_id && strlen(file->file_id) > 0) {
item._fileId = file->file_id;
}
if (file->directDownloadUrl) {
item._directDownloadUrl = QString::fromUtf8( file->directDownloadUrl );
}
if (file->directDownloadCookies) {
item._directDownloadCookies = QString::fromUtf8( file->directDownloadCookies );
}
if (file->remotePerm && file->remotePerm[0]) {
item._remotePerm = QByteArray(file->remotePerm);
}
item._should_update_etag = item._should_update_etag || file->should_update_etag;
// record the seen files to be able to clean the journal later
_seenFiles.insert(item._file);
if (remote && file->remotePerm && file->remotePerm[0]) {
_remotePerms[item._file] = file->remotePerm;
}
switch(file->error_status) {
case CSYNC_STATUS_OK:
break;
case CSYNC_STATUS_INDIVIDUAL_IS_SYMLINK:
item._errorString = tr("Symbolic links are not supported in syncing.");
break;
case CSYNC_STATUS_INDIVIDUAL_IS_HARDLINK:
item._errorString = tr("Hard links are not supported in syncing.");
break;
case CSYNC_STATUS_INDIVIDUAL_IGNORE_LIST:
item._errorString = tr("File is listed on the ignore list.");
break;
case CSYNC_STATUS_INDIVIDUAL_IS_INVALID_CHARS:
item._errorString = tr("File contains invalid characters that can not be synced cross platform.");
break;
case CYSNC_STATUS_FILE_LOCKED_OR_OPEN:
item._errorString = QLatin1String("File locked"); // don't translate, internal use!
break;
default:
Q_ASSERT("Non handled error-status");
/* No error string */
}
item._isDirectory = file->type == CSYNC_FTW_TYPE_DIR;
// The etag is already set in the previous sync phases somewhere. Maybe we should remove it there
// and do it here so we have a consistent state about which tree stores information from which source.
item._etag = file->etag;
item._size = file->size;
if (!remote) {
item._inode = file->inode;
}
switch( file->type ) {
case CSYNC_FTW_TYPE_DIR:
item._type = SyncFileItem::Directory;
break;
case CSYNC_FTW_TYPE_FILE:
item._type = SyncFileItem::File;
break;
case CSYNC_FTW_TYPE_SLINK:
item._type = SyncFileItem::SoftLink;
break;
default:
item._type = SyncFileItem::UnknownType;
}
SyncFileItem::Direction dir;
int re = 0;
switch(file->instruction) {
case CSYNC_INSTRUCTION_NONE:
if (remote && item._should_update_etag && !item._isDirectory && item._instruction == CSYNC_INSTRUCTION_NONE) {
// Update the database now already (new fileid or etag or remotePerm)
// Those are files that were detected as "resolved conflict".
// They should have been a conflict because they both were new, or both
// had their local mtime or remote etag modified, but the size and mtime
// is the same on the server. This typically happen when the database is removed.
// Nothing will be done for those file, but we still need to update the database.
_journal->setFileRecord(SyncJournalFileRecord(item, _localPath + item._file));
item._should_update_etag = false;
}
if (item._isDirectory && (remote || file->should_update_etag)) {
// Because we want still to update etags of directories
dir = SyncFileItem::None;
} else {
// No need to do anything.
_hasNoneFiles = true;
emit syncItemDiscovered(item);
return re;
}
break;
case CSYNC_INSTRUCTION_RENAME:
dir = !remote ? SyncFileItem::Down : SyncFileItem::Up;
item._renameTarget = QString::fromUtf8( file->rename_path );
if (item._isDirectory)
_renamedFolders.insert(item._file, item._renameTarget);
break;
case CSYNC_INSTRUCTION_REMOVE:
_hasRemoveFile = true;
dir = !remote ? SyncFileItem::Down : SyncFileItem::Up;
break;
case CSYNC_INSTRUCTION_CONFLICT:
case CSYNC_INSTRUCTION_IGNORE:
case CSYNC_INSTRUCTION_ERROR:
dir = SyncFileItem::None;
break;
case CSYNC_INSTRUCTION_EVAL:
case CSYNC_INSTRUCTION_NEW:
case CSYNC_INSTRUCTION_SYNC:
case CSYNC_INSTRUCTION_STAT_ERROR:
default:
dir = remote ? SyncFileItem::Down : SyncFileItem::Up;
if (!remote && file->instruction == CSYNC_INSTRUCTION_SYNC) {
// An upload of an existing file means that the file was left unchanged on the server
// This count as a NONE for detecting if all the file on the server were changed
_hasNoneFiles = true;
}
break;
}
item._direction = dir;
// check for blacklisting of this item.
// if the item is on blacklist, the instruction was set to IGNORE
checkBlacklisting( &item );
if (!item._isDirectory) {
_progressInfo._totalFileCount++;
if (Progress::isSizeDependent(file->instruction)) {
_progressInfo._totalSize += file->size;
}
}
_needsUpdate = true;
item.log._etag = file->etag;
item.log._fileId = file->file_id;
item.log._instruction = file->instruction;
item.log._modtime = file->modtime;
item.log._size = file->size;
item.log._other_etag = file->other.etag;
item.log._other_fileId = file->other.file_id;
item.log._other_instruction = file->other.instruction;
item.log._other_modtime = file->other.modtime;
item.log._other_size = file->other.size;
_syncItemMap.insert(fileUtf8, item);
emit syncItemDiscovered(item);
return re;
}
