static void check_csync_excluded_performance(void **state)
{
CSYNC *csync = *state;
const int N = 10000;
int totalRc = 0;
// Being able to use QElapsedTimer for measurement would be nice...
{
struct timeval before, after;
gettimeofday(&before, 0);
for (int i = 0; i < N; ++i) {
totalRc += csync_excluded(csync, "/this/is/quite/a/long/path/with/many/components", CSYNC_FTW_TYPE_DIR);
totalRc += csync_excluded(csync, "/1/2/3/4/5/6/7/8/9/10/11/12/13/14/15/16/17/18/19/20/21/22/23/24/25/26/27/29", CSYNC_FTW_TYPE_FILE);
}
assert_int_equal(totalRc, CSYNC_NOT_EXCLUDED); // mainly to avoid optimization
gettimeofday(&after, 0);
const double total = (after.tv_sec - before.tv_sec)
+ (after.tv_usec - before.tv_usec) / 1.0e6;
const double perCallMs = total / 2 / N * 1000;
printf("csync_excluded: %f ms per call\n", perCallMs);
}
{
struct timeval before, after;
gettimeofday(&before, 0);
for (int i = 0; i < N; ++i) {
totalRc += csync_excluded_traversal(csync->excludes, "/this/is/quite/a/long/path/with/many/components", CSYNC_FTW_TYPE_DIR);
totalRc += csync_excluded_traversal(csync->excludes, "/1/2/3/4/5/6/7/8/9/10/11/12/13/14/15/16/17/18/19/20/21/22/23/24/25/26/27/29", CSYNC_FTW_TYPE_FILE);
}
assert_int_equal(totalRc, CSYNC_NOT_EXCLUDED); // mainly to avoid optimization
gettimeofday(&after, 0);
const double total = (after.tv_sec - before.tv_sec)
+ (after.tv_usec - before.tv_usec) / 1.0e6;
const double perCallMs = total / 2 / N * 1000;
printf("csync_excluded_traversal: %f ms per call\n", perCallMs);
}
}
int csync_statedb_get_below_path( CSYNC *ctx, const char *path ) {
int rc;
sqlite3_stmt *stmt = NULL;
int64_t cnt = 0;
if( !path ) {
return -1;
}
if( !ctx || ctx->db_is_empty ) {
return -1;
}
/* Select the entries for anything that starts with (path+'/')
* In other words, anything that is between path+'/' and path+'0',
* (because '0' follows '/' in ascii)
*/
const char *below_path_query = "SELECT phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize, ignoredChildrenRemote FROM metadata WHERE path > (?||'/') AND path < (?||'0')";
SQLITE_BUSY_HANDLED(sqlite3_prepare_v2(ctx->statedb.db, below_path_query, -1, &stmt, NULL));
ctx->statedb.lastReturnValue = rc;
if( rc != SQLITE_OK ) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "WRN: Unable to create stmt for below path query.");
return -1;
}
if (stmt == NULL) {
return -1;
}
sqlite3_bind_text(stmt, 1, path, -1, SQLITE_STATIC);
sqlite3_bind_text(stmt, 2, path, -1, SQLITE_STATIC);
cnt = 0;
ctx->statedb.lastReturnValue = rc;
do {
csync_file_stat_t *st = NULL;
rc = _csync_file_stat_from_metadata_table( &st, stmt);
if( st ) {
/* Check for exclusion from the tree.
* Note that this is only a safety net in case the ignore list changes
* without a full remote discovery being triggered. */
CSYNC_EXCLUDE_TYPE excluded = csync_excluded_traversal(ctx->excludes, st->path, st->type);
if (excluded != CSYNC_NOT_EXCLUDED) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "%s excluded (%d)", st->path, excluded);
if (excluded == CSYNC_FILE_EXCLUDE_AND_REMOVE
|| excluded == CSYNC_FILE_SILENTLY_EXCLUDED) {
SAFE_FREE(st);
continue;
}
st->instruction = CSYNC_INSTRUCTION_IGNORE;
}
/* store into result list. */
if (c_rbtree_insert(ctx->remote.tree, (void *) st) < 0) {
SAFE_FREE(st);
ctx->status_code = CSYNC_STATUS_TREE_ERROR;
break;
}
cnt++;
}
} while( rc == SQLITE_ROW );
ctx->statedb.lastReturnValue = rc;
if( rc != SQLITE_DONE ) {
ctx->status_code = CSYNC_STATUS_TREE_ERROR;
} else {
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "%" PRId64 " entries read below path %s from db.", cnt, path);
}
sqlite3_finalize(stmt);
return 0;
}
static int _csync_detect_update(CSYNC *ctx, const char *file,
const csync_vio_file_stat_t *fs, const int type) {
uint64_t h = 0;
size_t len = 0;
size_t size = 0;
const char *path = NULL;
csync_file_stat_t *st = NULL;
csync_file_stat_t *tmp = NULL;
CSYNC_EXCLUDE_TYPE excluded;
if ((file == NULL) || (fs == NULL)) {
errno = EINVAL;
ctx->status_code = CSYNC_STATUS_PARAM_ERROR;
return -1;
}
path = file;
switch (ctx->current) {
case LOCAL_REPLICA:
if (strlen(path) <= strlen(ctx->local.uri)) {
ctx->status_code = CSYNC_STATUS_PARAM_ERROR;
return -1;
}
path += strlen(ctx->local.uri) + 1;
break;
case REMOTE_REPLICA:
if (strlen(path) <= strlen(ctx->remote.uri)) {
ctx->status_code = CSYNC_STATUS_PARAM_ERROR;
return -1;
}
path += strlen(ctx->remote.uri) + 1;
break;
default:
path = NULL;
ctx->status_code = CSYNC_STATUS_PARAM_ERROR;
return -1;
}
len = strlen(path);
if (type == CSYNC_FTW_TYPE_SKIP) {
excluded =CSYNC_FILE_EXCLUDE_STAT_FAILED;
} else {
/* Check if file is excluded */
excluded = csync_excluded_traversal(ctx->excludes, path, type);
}
if( excluded == CSYNC_NOT_EXCLUDED ) {
/* Even if it is not excluded by a pattern, maybe it is to be ignored
* because it's a hidden file that should not be synced.
* This code should probably be in csync_exclude, but it does not have the fs parameter.
* Keep it here for now */
if (ctx->ignore_hidden_files && (fs->flags & CSYNC_VIO_FILE_FLAGS_HIDDEN)) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "file excluded because it is a hidden file: %s", path);
excluded = CSYNC_FILE_EXCLUDE_HIDDEN;
}
} else {
/* File is ignored because it's matched by a user- or system exclude pattern. */
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "%s excluded (%d)", path, excluded);
if (excluded == CSYNC_FILE_EXCLUDE_AND_REMOVE) {
return 1;
}
if (excluded == CSYNC_FILE_SILENTLY_EXCLUDED) {
return 1;
}
}
if (ctx->current == REMOTE_REPLICA && ctx->callbacks.checkSelectiveSyncBlackListHook) {
if (ctx->callbacks.checkSelectiveSyncBlackListHook(ctx->callbacks.update_callback_userdata, path)) {
return 1;
}
}
h = _hash_of_file(ctx, file );
if( h == 0 ) {
return -1;
}
size = sizeof(csync_file_stat_t) + len + 1;
st = c_malloc(size);
/* Set instruction by default to none */
st->instruction = CSYNC_INSTRUCTION_NONE;
st->etag = NULL;
st->child_modified = 0;
st->has_ignored_files = 0;
if (type == CSYNC_FTW_TYPE_FILE ) {
if (fs->mtime == 0) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "file: %s - mtime is zero!", path);
}
}
if (excluded > CSYNC_NOT_EXCLUDED || type == CSYNC_FTW_TYPE_SLINK) {
st->instruction = CSYNC_INSTRUCTION_IGNORE;
if (ctx->current_fs) {
ctx->current_fs->has_ignored_files = true;
}
goto out;
}
/* Update detection: Check if a database entry exists.
* If not, the file is either new or has been renamed. To see if it is
* renamed, the db gets queried by the inode of the file as that one
* does not change on rename.
*/
if (csync_get_statedb_exists(ctx)) {
tmp = csync_statedb_get_stat_by_hash(ctx, h);
if(_last_db_return_error(ctx)) {
SAFE_FREE(st);
SAFE_FREE(tmp);
ctx->status_code = CSYNC_STATUS_UNSUCCESSFUL;
return -1;
}
if(tmp && tmp->phash == h ) { /* there is an entry in the database */
/* we have an update! */
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Database entry found, compare: %" PRId64 " <-> %" PRId64
", etag: %s <-> %s, inode: %" PRId64 " <-> %" PRId64
", size: %" PRId64 " <-> %" PRId64 ", perms: %s <-> %s, ignore: %d",
((int64_t) fs->mtime), ((int64_t) tmp->modtime),
fs->etag, tmp->etag, (uint64_t) fs->inode, (uint64_t) tmp->inode,
(uint64_t) fs->size, (uint64_t) tmp->size, fs->remotePerm, tmp->remotePerm, tmp->has_ignored_files );
if (ctx->current == REMOTE_REPLICA && !c_streq(fs->etag, tmp->etag)) {
st->instruction = CSYNC_INSTRUCTION_EVAL;
// Preserve the EVAL flag later on if the type has changed.
if (_csync_filetype_different(tmp, fs)) {
st->child_modified = 1;
}
goto out;
}
if (ctx->current == LOCAL_REPLICA &&
(!_csync_mtime_equal(fs->mtime, tmp->modtime)
// zero size in statedb can happen during migration
|| (tmp->size != 0 && fs->size != tmp->size))) {
// Checksum comparison at this stage is only enabled for .eml files,
// check #4754 #4755
bool isEmlFile = csync_fnmatch("*.eml", file, FNM_CASEFOLD) == 0;
if (isEmlFile && fs->size == tmp->size && tmp->checksumTypeId) {
if (ctx->callbacks.checksum_hook) {
st->checksum = ctx->callbacks.checksum_hook(
file, tmp->checksumTypeId,
ctx->callbacks.checksum_userdata);
}
bool checksumIdentical = false;
if (st->checksum) {
st->checksumTypeId = tmp->checksumTypeId;
checksumIdentical = strncmp(st->checksum, tmp->checksum, 1000) == 0;
}
if (checksumIdentical) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "NOTE: Checksums are identical, file did not actually change: %s", path);
st->instruction = CSYNC_INSTRUCTION_NONE;
st->should_update_metadata = true;
goto out;
}
}
// Preserve the EVAL flag later on if the type has changed.
if (_csync_filetype_different(tmp, fs)) {
st->child_modified = 1;
}
st->instruction = CSYNC_INSTRUCTION_EVAL;
goto out;
}
bool metadata_differ = (ctx->current == REMOTE_REPLICA && (!c_streq(fs->file_id, tmp->file_id)
|| !c_streq(fs->remotePerm, tmp->remotePerm)))
|| (ctx->current == LOCAL_REPLICA && fs->inode != tmp->inode);
if (type == CSYNC_FTW_TYPE_DIR && ctx->current == REMOTE_REPLICA
&& !metadata_differ && ctx->read_remote_from_db) {
/* If both etag and file id are equal for a directory, read all contents from
* the database.
* The metadata comparison ensure that we fetch all the file id or permission when
* upgrading owncloud
*/
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Reading from database: %s", path);
ctx->remote.read_from_db = true;
}
if (metadata_differ) {
/* file id or permissions has changed. Which means we need to update them in the DB. */
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Need to update metadata for: %s", path);
st->should_update_metadata = true;
}
/* If it was remembered in the db that the remote dir has ignored files, store
* that so that the reconciler can make advantage of.
*/
if( ctx->current == REMOTE_REPLICA ) {
st->has_ignored_files = tmp->has_ignored_files;
}
st->instruction = CSYNC_INSTRUCTION_NONE;
} else {
enum csync_vio_file_type_e tmp_vio_type = CSYNC_VIO_FILE_TYPE_UNKNOWN;
/* tmp might point to malloc mem, so free it here before reusing tmp */
SAFE_FREE(tmp);
/* check if it's a file and has been renamed */
if (ctx->current == LOCAL_REPLICA) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Checking for rename based on inode # %" PRId64 "", (uint64_t) fs->inode);
tmp = csync_statedb_get_stat_by_inode(ctx, fs->inode);
if(_last_db_return_error(ctx)) {
SAFE_FREE(st);
ctx->status_code = CSYNC_STATUS_UNSUCCESSFUL;
return -1;
}
/* translate the file type between the two stat types csync has. */
if( tmp && tmp->type == CSYNC_FTW_TYPE_FILE ) {
tmp_vio_type = CSYNC_VIO_FILE_TYPE_REGULAR;
} else if( tmp && tmp->type == CSYNC_FTW_TYPE_DIR) {
tmp_vio_type = CSYNC_VIO_FILE_TYPE_DIRECTORY;
} else if( tmp && tmp->type == CSYNC_FTW_TYPE_SLINK ) {
tmp_vio_type = CSYNC_VIO_FILE_TYPE_SYMBOLIC_LINK;
} else {
tmp_vio_type = CSYNC_VIO_FILE_TYPE_UNKNOWN;
}
// Default to NEW unless we're sure it's a rename.
st->instruction = CSYNC_INSTRUCTION_NEW;
bool isRename =
tmp && tmp->inode == fs->inode && tmp_vio_type == fs->type
&& (tmp->modtime == fs->mtime || fs->type == CSYNC_VIO_FILE_TYPE_DIRECTORY)
#ifdef NO_RENAME_EXTENSION
&& _csync_sameextension(tmp->path, path)
#endif
;
// Verify the checksum where possible
if (isRename && tmp->checksumTypeId && ctx->callbacks.checksum_hook
&& fs->type == CSYNC_VIO_FILE_TYPE_REGULAR) {
st->checksum = ctx->callbacks.checksum_hook(
file, tmp->checksumTypeId,
ctx->callbacks.checksum_userdata);
if (st->checksum) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "checking checksum of potential rename %s %s <-> %s", path, st->checksum, tmp->checksum);
st->checksumTypeId = tmp->checksumTypeId;
isRename = strncmp(st->checksum, tmp->checksum, 1000) == 0;
}
}
if (isRename) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "pot rename detected based on inode # %" PRId64 "", (uint64_t) fs->inode);
/* inode found so the file has been renamed */
st->instruction = CSYNC_INSTRUCTION_EVAL_RENAME;
if (fs->type == CSYNC_VIO_FILE_TYPE_DIRECTORY) {
csync_rename_record(ctx, tmp->path, path);
}
}
goto out;
} else {
/* Remote Replica Rename check */
tmp = csync_statedb_get_stat_by_file_id(ctx, fs->file_id);
if(_last_db_return_error(ctx)) {
SAFE_FREE(st);
ctx->status_code = CSYNC_STATUS_UNSUCCESSFUL;
return -1;
}
if(tmp ) { /* tmp existing at all */
if ( _csync_filetype_different(tmp, fs)) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "WARN: file types different is not!");
st->instruction = CSYNC_INSTRUCTION_NEW;
goto out;
}
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "remote rename detected based on fileid %s %s", tmp->path, file);
st->instruction = CSYNC_INSTRUCTION_EVAL_RENAME;
if (fs->type == CSYNC_VIO_FILE_TYPE_DIRECTORY) {
csync_rename_record(ctx, tmp->path, path);
} else {
if( !c_streq(tmp->etag, fs->etag) ) {
/* CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "ETags are different!"); */
/* File with different etag, don't do a rename, but download the file again */
st->instruction = CSYNC_INSTRUCTION_NEW;
}
}
goto out;
} else {
/* file not found in statedb */
st->instruction = CSYNC_INSTRUCTION_NEW;
if (fs->type == CSYNC_VIO_FILE_TYPE_DIRECTORY && ctx->current == REMOTE_REPLICA && ctx->callbacks.checkSelectiveSyncNewFolderHook) {
if (ctx->callbacks.checkSelectiveSyncNewFolderHook(ctx->callbacks.update_callback_userdata, path)) {
SAFE_FREE(st);
return 1;
}
}
goto out;
}
}
}
} else {
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Unable to open statedb" );
SAFE_FREE(st);
ctx->status_code = CSYNC_STATUS_UNSUCCESSFUL;
return -1;
}
out:
/* Set the ignored error string. */
if (st->instruction == CSYNC_INSTRUCTION_IGNORE) {
if( type == CSYNC_FTW_TYPE_SLINK ) {
st->error_status = CSYNC_STATUS_INDIVIDUAL_IS_SYMLINK; /* Symbolic links are ignored. */
} else {
if (excluded == CSYNC_FILE_EXCLUDE_LIST) {
st->error_status = CSYNC_STATUS_INDIVIDUAL_IGNORE_LIST; /* File listed on ignore list. */
} else if (excluded == CSYNC_FILE_EXCLUDE_INVALID_CHAR) {
st->error_status = CSYNC_STATUS_INDIVIDUAL_IS_INVALID_CHARS; /* File contains invalid characters. */
} else if (excluded == CSYNC_FILE_EXCLUDE_LONG_FILENAME) {
st->error_status = CSYNC_STATUS_INDIVIDUAL_EXCLUDE_LONG_FILENAME; /* File name is too long. */
} else if (excluded == CSYNC_FILE_EXCLUDE_HIDDEN ) {
st->error_status = CSYNC_STATUS_INDIVIDUAL_EXCLUDE_HIDDEN;
} else if (excluded == CSYNC_FILE_EXCLUDE_STAT_FAILED) {
st->error_status = CSYNC_STATUS_INDIVIDUAL_STAT_FAILED;
}
}
}
if (st->instruction != CSYNC_INSTRUCTION_NONE && st->instruction != CSYNC_INSTRUCTION_IGNORE
&& type != CSYNC_FTW_TYPE_DIR) {
st->child_modified = 1;
}
ctx->current_fs = st;
csync_file_stat_free(tmp);
st->inode = fs->inode;
st->mode = fs->mode;
st->size = fs->size;
st->modtime = fs->mtime;
st->type = type;
st->etag = NULL;
if( fs->etag ) {
SAFE_FREE(st->etag);
st->etag = c_strdup(fs->etag);
}
csync_vio_set_file_id(st->file_id, fs->file_id);
if (fs->fields & CSYNC_VIO_FILE_STAT_FIELDS_DIRECTDOWNLOADURL) {
SAFE_FREE(st->directDownloadUrl);
st->directDownloadUrl = c_strdup(fs->directDownloadUrl);
}
if (fs->fields & CSYNC_VIO_FILE_STAT_FIELDS_DIRECTDOWNLOADCOOKIES) {
SAFE_FREE(st->directDownloadCookies);
st->directDownloadCookies = c_strdup(fs->directDownloadCookies);
}
if (fs->fields & CSYNC_VIO_FILE_STAT_FIELDS_PERM) {
strncpy(st->remotePerm, fs->remotePerm, REMOTE_PERM_BUF_SIZE);
}
st->phash = h;
st->pathlen = len;
memcpy(st->path, (len ? path : ""), len + 1);
switch (ctx->current) {
case LOCAL_REPLICA:
if (c_rbtree_insert(ctx->local.tree, (void *) st) < 0) {
SAFE_FREE(st);
ctx->status_code = CSYNC_STATUS_TREE_ERROR;
return -1;
}
break;
case REMOTE_REPLICA:
if (c_rbtree_insert(ctx->remote.tree, (void *) st) < 0) {
SAFE_FREE(st);
ctx->status_code = CSYNC_STATUS_TREE_ERROR;
return -1;
}
break;
default:
break;
}
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "file: %s, instruction: %s <<=", st->path,
csync_instruction_str(st->instruction));
return 0;
}
static void check_csync_excluded_traversal(void **state)
{
CSYNC *csync = *state;
int rc;
_csync_exclude_add( &(csync->excludes), "/exclude" );
/* Check toplevel dir, the pattern only works for toplevel dir. */
rc = csync_excluded_traversal(csync->excludes, "/exclude", CSYNC_FTW_TYPE_DIR);
assert_int_equal(rc, CSYNC_FILE_EXCLUDE_LIST);
rc = csync_excluded_traversal(csync->excludes, "/foo/exclude", CSYNC_FTW_TYPE_DIR);
assert_int_equal(rc, CSYNC_NOT_EXCLUDED);
/* check for a file called exclude. Must still work */
rc = csync_excluded_traversal(csync->excludes, "/exclude", CSYNC_FTW_TYPE_FILE);
assert_int_equal(rc, CSYNC_FILE_EXCLUDE_LIST);
rc = csync_excluded_traversal(csync->excludes, "/foo/exclude", CSYNC_FTW_TYPE_FILE);
assert_int_equal(rc, CSYNC_NOT_EXCLUDED);
/* Add an exclude for directories only: excl/ */
_csync_exclude_add( &(csync->excludes), "excl/" );
rc = csync_excluded_traversal(csync->excludes, "/excl", CSYNC_FTW_TYPE_DIR);
assert_int_equal(rc, CSYNC_FILE_EXCLUDE_LIST);
rc = csync_excluded_traversal(csync->excludes, "meep/excl", CSYNC_FTW_TYPE_DIR);
assert_int_equal(rc, CSYNC_FILE_EXCLUDE_LIST);
rc = csync_excluded_traversal(csync->excludes, "meep/excl/file", CSYNC_FTW_TYPE_FILE);
assert_int_equal(rc, CSYNC_NOT_EXCLUDED); // because leading dirs aren't checked!
rc = csync_excluded_traversal(csync->excludes, "/excl", CSYNC_FTW_TYPE_FILE);
assert_int_equal(rc, CSYNC_NOT_EXCLUDED);
_csync_exclude_add(&csync->excludes, "/excludepath/withsubdir");
rc = csync_excluded_traversal(csync->excludes, "/excludepath/withsubdir", CSYNC_FTW_TYPE_DIR);
assert_int_equal(rc, CSYNC_FILE_EXCLUDE_LIST);
rc = csync_excluded_traversal(csync->excludes, "/excludepath/withsubdir", CSYNC_FTW_TYPE_FILE);
assert_int_equal(rc, CSYNC_FILE_EXCLUDE_LIST);
rc = csync_excluded_traversal(csync->excludes, "/excludepath/withsubdir2", CSYNC_FTW_TYPE_DIR);
assert_int_equal(rc, CSYNC_NOT_EXCLUDED);
rc = csync_excluded_traversal(csync->excludes, "/excludepath/withsubdir/foo", CSYNC_FTW_TYPE_DIR);
assert_int_equal(rc, CSYNC_NOT_EXCLUDED); // because leading dirs aren't checked!
}
