int SyncEngine::treewalkFile( TREE_WALK_FILE *file, bool remote ) { if( ! file ) return -1; SyncFileItem item; item._file = QString::fromUtf8( file->path ); item._originalFile = item._file; item._instruction = file->instruction; item._direction = SyncFileItem::None; item._fileId = file->file_id; if (file->directDownloadUrl) { item._directDownloadUrl = QString::fromUtf8( file->directDownloadUrl ); } if (file->directDownloadCookies) { item._directDownloadCookies = QString::fromUtf8( file->directDownloadCookies ); } if (file->remotePerm) { item._remotePerm = QByteArray(file->remotePerm); } // record the seen files to be able to clean the journal later _seenFiles[item._file] = QString(); switch(file->error_status) { case CSYNC_STATUS_OK: break; case CSYNC_STATUS_INDIVIDUAL_IS_SYMLINK: item._errorString = tr("Symbolic links are not supported in syncing."); break; case CSYNC_STATUS_INDIVIDUAL_IGNORE_LIST: item._errorString = tr("File is listed on the ignore list."); break; case CSYNC_STATUS_INDIVIDUAL_IS_INVALID_CHARS: item._errorString = tr("File contains invalid characters that can not be synced cross platform."); break; case CYSNC_STATUS_FILE_LOCKED_OR_OPEN: item._errorString = QLatin1String("File locked"); // don't translate, internal use! break; default: Q_ASSERT("Non handled error-status"); /* No error string */ } item._isDirectory = file->type == CSYNC_FTW_TYPE_DIR; item._modtime = file->modtime; item._etag = file->etag; item._size = file->size; item._inode = file->inode; item._should_update_etag = file->should_update_etag; switch( file->type ) { case CSYNC_FTW_TYPE_DIR: item._type = SyncFileItem::Directory; break; case CSYNC_FTW_TYPE_FILE: item._type = SyncFileItem::File; break; case CSYNC_FTW_TYPE_SLINK: item._type = SyncFileItem::SoftLink; break; default: item._type = SyncFileItem::UnknownType; } SyncFileItem::Direction dir; int re = 0; switch(file->instruction) { case CSYNC_INSTRUCTION_NONE: if (file->should_update_etag && !item._isDirectory) { // Update the database now already (new fileid or etag or remotePerm) _journal->setFileRecord(SyncJournalFileRecord(item, _localPath + item._file)); item._should_update_etag = false; } if (item._isDirectory && remote) { // Because we want still to update etags of directories dir = SyncFileItem::None; } else { // No need to do anything. _hasFiles = true; emit syncItemDiscovered(item); return re; } break; case CSYNC_INSTRUCTION_RENAME: dir = !remote ? SyncFileItem::Down : SyncFileItem::Up; item._renameTarget = QString::fromUtf8( file->rename_path ); if (item._isDirectory) _renamedFolders.insert(item._file, item._renameTarget); break; case CSYNC_INSTRUCTION_REMOVE: dir = !remote ? SyncFileItem::Down : SyncFileItem::Up; break; case CSYNC_INSTRUCTION_CONFLICT: case CSYNC_INSTRUCTION_IGNORE: case CSYNC_INSTRUCTION_ERROR: dir = SyncFileItem::None; break; case CSYNC_INSTRUCTION_EVAL: case CSYNC_INSTRUCTION_NEW: case CSYNC_INSTRUCTION_SYNC: case CSYNC_INSTRUCTION_STAT_ERROR: default: dir = remote ? SyncFileItem::Down : SyncFileItem::Up; break; } item._direction = dir; // check for blacklisting of this item. // if the item is on blacklist, the instruction was set to IGNORE checkBlacklisting( &item ); if (file->instruction != CSYNC_INSTRUCTION_IGNORE && file->instruction != CSYNC_INSTRUCTION_REMOVE && file->instruction != CSYNC_INSTRUCTION_ERROR) { _hasFiles = true; } if (!item._isDirectory) { _progressInfo._totalFileCount++; if (Progress::isSizeDependent(file->instruction)) { _progressInfo._totalSize += file->size; } } _needsUpdate = true; item.log._etag = file->etag; item.log._fileId = file->file_id; item.log._instruction = file->instruction; item.log._modtime = file->modtime; item.log._size = file->size; item.log._other_etag = file->other.etag; item.log._other_fileId = file->other.file_id; item.log._other_instruction = file->other.instruction; item.log._other_modtime = file->other.modtime; item.log._other_size = file->other.size; _syncedItems.append(item); emit syncItemDiscovered(item); return re; }
int SyncEngine::treewalkFile( TREE_WALK_FILE *file, bool remote ) { if( ! file ) return -1; QTextCodec::ConverterState utf8State; static QTextCodec *codec = QTextCodec::codecForName("UTF-8"); Q_ASSERT(codec); QString fileUtf8 = codec->toUnicode(file->path, qstrlen(file->path), &utf8State); QString renameTarget; QString key = fileUtf8; auto instruction = file->instruction; if (utf8State.invalidChars > 0 || utf8State.remainingChars > 0) { qWarning() << "File ignored because of invalid utf-8 sequence: " << file->path; instruction = CSYNC_INSTRUCTION_IGNORE; } else { renameTarget = codec->toUnicode(file->rename_path, qstrlen(file->rename_path), &utf8State); if (utf8State.invalidChars > 0 || utf8State.remainingChars > 0) { qWarning() << "File ignored because of invalid utf-8 sequence in the rename_path: " << file->path << file->rename_path; instruction = CSYNC_INSTRUCTION_IGNORE; } if (instruction == CSYNC_INSTRUCTION_RENAME) { key = renameTarget; } } // Gets a default-constructed SyncFileItemPtr or the one from the first walk (=local walk) SyncFileItemPtr item = _syncItemMap.value(key); if (!item) item = SyncFileItemPtr(new SyncFileItem); if (item->_file.isEmpty() || instruction == CSYNC_INSTRUCTION_RENAME) { item->_file = fileUtf8; } item->_originalFile = item->_file; if (item->_instruction == CSYNC_INSTRUCTION_NONE || (item->_instruction == CSYNC_INSTRUCTION_IGNORE && instruction != CSYNC_INSTRUCTION_NONE)) { item->_instruction = instruction; item->_modtime = file->modtime; } else { if (instruction != CSYNC_INSTRUCTION_NONE) { qDebug() << "ERROR: Instruction" << item->_instruction << "vs" << instruction << "for" << fileUtf8; Q_ASSERT(!"Instructions are both unequal NONE"); return -1; } } if (file->file_id && file->file_id[0]) { item->_fileId = file->file_id; } if (file->directDownloadUrl) { item->_directDownloadUrl = QString::fromUtf8( file->directDownloadUrl ); } if (file->directDownloadCookies) { item->_directDownloadCookies = QString::fromUtf8( file->directDownloadCookies ); } if (file->remotePerm && file->remotePerm[0]) { item->_remotePerm = QByteArray(file->remotePerm); } item->_should_update_metadata = item->_should_update_metadata || file->should_update_metadata; /* The flag "serverHasIgnoredFiles" is true if item in question is a directory * that has children which are ignored in sync, either because the files are * matched by an ignore pattern, or because they are hidden. * * Only the information about the server side ignored files is stored to the * database and thus written to the item here. For the local repository its * generated by the walk through the real file tree by discovery phase. * * It needs to go to the sync journal becasue the stat information about remote * files are often read from database rather than being pulled from remote. */ if( remote ) { item->_serverHasIgnoredFiles = (file->has_ignored_files > 0); } // record the seen files to be able to clean the journal later _seenFiles.insert(item->_file); if (!renameTarget.isEmpty()) { // Yes, this records both the rename renameTarget and the original so we keep both in case of a rename _seenFiles.insert(renameTarget); } if (remote && file->remotePerm && file->remotePerm[0]) { _remotePerms[item->_file] = file->remotePerm; } switch(file->error_status) { case CSYNC_STATUS_OK: break; case CSYNC_STATUS_INDIVIDUAL_IS_SYMLINK: item->_errorString = tr("Symbolic links are not supported in syncing."); break; case CSYNC_STATUS_INDIVIDUAL_IGNORE_LIST: item->_errorString = tr("File is listed on the ignore list."); break; case CSYNC_STATUS_INDIVIDUAL_IS_INVALID_CHARS: item->_errorString = tr("Filename contains invalid characters that can not be synced cross platform."); break; case CSYNC_STATUS_INDIVIDUAL_EXCLUDE_LONG_FILENAME: item->_errorString = tr("Filename is too long."); break; case CSYNC_STATUS_INDIVIDUAL_EXCLUDE_HIDDEN: item->_errorString = tr("File is ignored because it's hidden."); break; case CYSNC_STATUS_FILE_LOCKED_OR_OPEN: item->_errorString = QLatin1String("File locked"); // don't translate, internal use! break; case CSYNC_STATUS_INDIVIDUAL_STAT_FAILED: item->_errorString = tr("Stat failed."); break; case CSYNC_STATUS_SERVICE_UNAVAILABLE: item->_errorString = QLatin1String("Server temporarily unavailable."); break; case CSYNC_STATUS_STORAGE_UNAVAILABLE: item->_errorString = QLatin1String("Directory temporarily not available on server."); item->_status = SyncFileItem::SoftError; _temporarilyUnavailablePaths.insert(item->_file); break; case CSYNC_STATUS_PERMISSION_DENIED: item->_errorString = QLatin1String("Directory not accessible on client, permission denied."); item->_status = SyncFileItem::SoftError; break; default: Q_ASSERT("Non handled error-status"); /* No error string */ } if (item->_instruction == CSYNC_INSTRUCTION_IGNORE && (utf8State.invalidChars > 0 || utf8State.remainingChars > 0)) { item->_status = SyncFileItem::NormalError; //item->_instruction = CSYNC_INSTRUCTION_ERROR; item->_errorString = tr("Filename encoding is not valid"); } item->_isDirectory = file->type == CSYNC_FTW_TYPE_DIR; if (file->etag && file->etag[0]) { item->_etag = file->etag; } item->_size = file->size; if (!remote) { item->_inode = file->inode; } switch( file->type ) { case CSYNC_FTW_TYPE_DIR: item->_type = SyncFileItem::Directory; break; case CSYNC_FTW_TYPE_FILE: item->_type = SyncFileItem::File; break; case CSYNC_FTW_TYPE_SLINK: item->_type = SyncFileItem::SoftLink; break; default: item->_type = SyncFileItem::UnknownType; } SyncFileItem::Direction dir = SyncFileItem::None; int re = 0; switch(file->instruction) { case CSYNC_INSTRUCTION_NONE: if (remote && item->_should_update_metadata && !item->_isDirectory && item->_instruction == CSYNC_INSTRUCTION_NONE) { // Update the database now already: New fileid or Etag or RemotePerm // Or for files that were detected as "resolved conflict". // They should have been a conflict because they both were new, or both // had their local mtime or remote etag modified, but the size and mtime // is the same on the server. This typically happens when the database is removed. // Nothing will be done for those files, but we still need to update the database. // Even if the mtime is different on the server, we always want to keep the mtime from // the file system in the DB, this is to avoid spurious upload on the next sync item->_modtime = file->other.modtime; _journal->updateFileRecordMetadata(SyncJournalFileRecord(*item, _localPath + item->_file)); item->_should_update_metadata = false; } if (item->_isDirectory && file->should_update_metadata) { // Because we want to still update etags of directories dir = SyncFileItem::None; } else { // No need to do anything. if (file->other.instruction == CSYNC_INSTRUCTION_NONE // Directories with ignored files does not count as 'None' && (file->type != CSYNC_FTW_TYPE_DIR || !file->has_ignored_files)) { _hasNoneFiles = true; } return re; } break; case CSYNC_INSTRUCTION_RENAME: dir = !remote ? SyncFileItem::Down : SyncFileItem::Up; item->_renameTarget = renameTarget; if (item->_isDirectory) _renamedFolders.insert(item->_file, item->_renameTarget); break; case CSYNC_INSTRUCTION_REMOVE: _hasRemoveFile = true; dir = !remote ? SyncFileItem::Down : SyncFileItem::Up; break; case CSYNC_INSTRUCTION_CONFLICT: case CSYNC_INSTRUCTION_IGNORE: case CSYNC_INSTRUCTION_ERROR: dir = SyncFileItem::None; break; case CSYNC_INSTRUCTION_EVAL: case CSYNC_INSTRUCTION_NEW: case CSYNC_INSTRUCTION_SYNC: case CSYNC_INSTRUCTION_STAT_ERROR: default: dir = remote ? SyncFileItem::Down : SyncFileItem::Up; if (!remote && file->instruction == CSYNC_INSTRUCTION_SYNC) { // An upload of an existing file means that the file was left unchanged on the server // This counts as a NONE for detecting if all the files on the server were changed _hasNoneFiles = true; } break; } item->_direction = dir; if (instruction != CSYNC_INSTRUCTION_NONE) { // check for blacklisting of this item. // if the item is on blacklist, the instruction was set to ERROR checkErrorBlacklisting( *item ); } _progressInfo->adjustTotalsForFile(*item); _needsUpdate = true; item->log._etag = file->etag; item->log._fileId = file->file_id; item->log._instruction = file->instruction; item->log._modtime = file->modtime; item->log._size = file->size; item->log._other_etag = file->other.etag; item->log._other_fileId = file->other.file_id; item->log._other_instruction = file->other.instruction; item->log._other_modtime = file->other.modtime; item->log._other_size = file->other.size; _syncItemMap.insert(key, item); emit syncItemDiscovered(*item); return re; }
int SyncEngine::treewalkFile( TREE_WALK_FILE *file, bool remote ) { if( ! file ) return -1; QString fileUtf8 = QString::fromUtf8( file->path ); // Gets a default-contructed SyncFileItem or the one from the first walk (=local walk) SyncFileItem item = _syncItemMap.value(fileUtf8); item._file = fileUtf8; item._originalFile = item._file; if (item._instruction == CSYNC_INSTRUCTION_NONE || (item._instruction == CSYNC_INSTRUCTION_IGNORE && file->instruction != CSYNC_INSTRUCTION_NONE)) { item._instruction = file->instruction; item._modtime = file->modtime; } else { if (file->instruction != CSYNC_INSTRUCTION_NONE) { Q_ASSERT(!"Instructions are both unequal NONE"); } } if (file->file_id && strlen(file->file_id) > 0) { item._fileId = file->file_id; } if (file->directDownloadUrl) { item._directDownloadUrl = QString::fromUtf8( file->directDownloadUrl ); } if (file->directDownloadCookies) { item._directDownloadCookies = QString::fromUtf8( file->directDownloadCookies ); } if (file->remotePerm && file->remotePerm[0]) { item._remotePerm = QByteArray(file->remotePerm); } item._should_update_etag = item._should_update_etag || file->should_update_etag; // record the seen files to be able to clean the journal later _seenFiles.insert(item._file); if (remote && file->remotePerm && file->remotePerm[0]) { _remotePerms[item._file] = file->remotePerm; } switch(file->error_status) { case CSYNC_STATUS_OK: break; case CSYNC_STATUS_INDIVIDUAL_IS_SYMLINK: item._errorString = tr("Symbolic links are not supported in syncing."); break; case CSYNC_STATUS_INDIVIDUAL_IS_HARDLINK: item._errorString = tr("Hard links are not supported in syncing."); break; case CSYNC_STATUS_INDIVIDUAL_IGNORE_LIST: item._errorString = tr("File is listed on the ignore list."); break; case CSYNC_STATUS_INDIVIDUAL_IS_INVALID_CHARS: item._errorString = tr("File contains invalid characters that can not be synced cross platform."); break; case CYSNC_STATUS_FILE_LOCKED_OR_OPEN: item._errorString = QLatin1String("File locked"); // don't translate, internal use! break; default: Q_ASSERT("Non handled error-status"); /* No error string */ } item._isDirectory = file->type == CSYNC_FTW_TYPE_DIR; // The etag is already set in the previous sync phases somewhere. Maybe we should remove it there // and do it here so we have a consistent state about which tree stores information from which source. item._etag = file->etag; item._size = file->size; if (!remote) { item._inode = file->inode; } switch( file->type ) { case CSYNC_FTW_TYPE_DIR: item._type = SyncFileItem::Directory; break; case CSYNC_FTW_TYPE_FILE: item._type = SyncFileItem::File; break; case CSYNC_FTW_TYPE_SLINK: item._type = SyncFileItem::SoftLink; break; default: item._type = SyncFileItem::UnknownType; } SyncFileItem::Direction dir; int re = 0; switch(file->instruction) { case CSYNC_INSTRUCTION_NONE: if (remote && item._should_update_etag && !item._isDirectory && item._instruction == CSYNC_INSTRUCTION_NONE) { // Update the database now already (new fileid or etag or remotePerm) // Those are files that were detected as "resolved conflict". // They should have been a conflict because they both were new, or both // had their local mtime or remote etag modified, but the size and mtime // is the same on the server. This typically happen when the database is removed. // Nothing will be done for those file, but we still need to update the database. _journal->setFileRecord(SyncJournalFileRecord(item, _localPath + item._file)); item._should_update_etag = false; } if (item._isDirectory && (remote || file->should_update_etag)) { // Because we want still to update etags of directories dir = SyncFileItem::None; } else { // No need to do anything. _hasNoneFiles = true; emit syncItemDiscovered(item); return re; } break; case CSYNC_INSTRUCTION_RENAME: dir = !remote ? SyncFileItem::Down : SyncFileItem::Up; item._renameTarget = QString::fromUtf8( file->rename_path ); if (item._isDirectory) _renamedFolders.insert(item._file, item._renameTarget); break; case CSYNC_INSTRUCTION_REMOVE: _hasRemoveFile = true; dir = !remote ? SyncFileItem::Down : SyncFileItem::Up; break; case CSYNC_INSTRUCTION_CONFLICT: case CSYNC_INSTRUCTION_IGNORE: case CSYNC_INSTRUCTION_ERROR: dir = SyncFileItem::None; break; case CSYNC_INSTRUCTION_EVAL: case CSYNC_INSTRUCTION_NEW: case CSYNC_INSTRUCTION_SYNC: case CSYNC_INSTRUCTION_STAT_ERROR: default: dir = remote ? SyncFileItem::Down : SyncFileItem::Up; if (!remote && file->instruction == CSYNC_INSTRUCTION_SYNC) { // An upload of an existing file means that the file was left unchanged on the server // This count as a NONE for detecting if all the file on the server were changed _hasNoneFiles = true; } break; } item._direction = dir; // check for blacklisting of this item. // if the item is on blacklist, the instruction was set to IGNORE checkBlacklisting( &item ); if (!item._isDirectory) { _progressInfo._totalFileCount++; if (Progress::isSizeDependent(file->instruction)) { _progressInfo._totalSize += file->size; } } _needsUpdate = true; item.log._etag = file->etag; item.log._fileId = file->file_id; item.log._instruction = file->instruction; item.log._modtime = file->modtime; item.log._size = file->size; item.log._other_etag = file->other.etag; item.log._other_fileId = file->other.file_id; item.log._other_instruction = file->other.instruction; item.log._other_modtime = file->other.modtime; item.log._other_size = file->other.size; _syncItemMap.insert(fileUtf8, item); emit syncItemDiscovered(item); return re; }