/*
* We merge replicas at the file level. The merged replica contains the
* superset of files that are on the local machine and server copies of
* the replica. In the case where the same file is in both the local
* and server copy, the file that was modified most recently is used.
* This means that new files are not deleted, and updated versions of
* existing files are not overwritten.
*
* When a file is updated, the merge algorithm compares the destination
* file with the the source file. If the destination file is newer
* (timestamp is newer), it is not overwritten. If both files, on the
* source and the destination, have been changed, the newer file wins.
*/
static int _csync_merge_algorithm_visitor(void *obj, void *data) {
csync_file_stat_t *cur = NULL;
csync_file_stat_t *other = NULL;
csync_file_stat_t *tmp = NULL;
uint64_t h = 0;
int len = 0;
CSYNC *ctx = NULL;
c_rbtree_t *tree = NULL;
c_rbnode_t *node = NULL;
cur = (csync_file_stat_t *) obj;
ctx = (CSYNC *) data;
/* we need the opposite tree! */
switch (ctx->current) {
case LOCAL_REPLICA:
tree = ctx->remote.tree;
break;
case REMOTE_REPLICA:
tree = ctx->local.tree;
break;
default:
break;
}
node = c_rbtree_find(tree, &cur->phash);
if (!node) {
/* Check the renamed path as well. */
char *renamed_path = csync_rename_adjust_path(ctx, cur->path);
if (!c_streq(renamed_path, cur->path)) {
len = strlen( renamed_path );
h = c_jhash64((uint8_t *) renamed_path, len, 0);
node = c_rbtree_find(tree, &h);
}
SAFE_FREE(renamed_path);
}
if (!node) {
/* Check if it is ignored */
node = _csync_check_ignored(tree, cur->path, cur->pathlen);
/* If it is ignored, other->instruction will be IGNORE so this one will also be ignored */
}
/* file only found on current replica */
if (node == NULL) {
switch(cur->instruction) {
/* file has been modified */
case CSYNC_INSTRUCTION_EVAL:
cur->instruction = CSYNC_INSTRUCTION_NEW;
break;
/* file has been removed on the opposite replica */
case CSYNC_INSTRUCTION_NONE:
cur->instruction = CSYNC_INSTRUCTION_REMOVE;
break;
case CSYNC_INSTRUCTION_EVAL_RENAME:
if(ctx->current == LOCAL_REPLICA ) {
/* use the old name to find the "other" node */
tmp = csync_statedb_get_stat_by_inode(ctx, cur->inode);
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Finding opposite temp through inode %" PRIu64 ": %s",
cur->inode, tmp ? "true":"false");
} else if( ctx->current == REMOTE_REPLICA ) {
tmp = csync_statedb_get_stat_by_file_id(ctx, cur->file_id);
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Finding opposite temp through file ID %s: %s",
cur->file_id, tmp ? "true":"false");
} else {
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Unknown replica...");
}
if( tmp ) {
if( tmp->path ) {
len = strlen( tmp->path );
h = c_jhash64((uint8_t *) tmp->path, len, 0);
/* First, check that the file is NOT in our tree (another file with the same name was added) */
node = c_rbtree_find(ctx->current == REMOTE_REPLICA ? ctx->remote.tree : ctx->local.tree, &h);
if (node) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Origin found in our tree : %s", tmp->path);
} else {
/* Find the temporar file in the other tree. */
node = c_rbtree_find(tree, &h);
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "PHash of temporary opposite (%s): %" PRIu64 " %s",
tmp->path , h, node ? "found": "not found" );
if (node) {
other = (csync_file_stat_t*)node->data;
} else {
/* the renamed file could not be found in the opposite tree. That is because it
* is not longer existing there, maybe because it was renamed or deleted.
* The journal is cleaned up later after propagation.
*/
}
}
}
if(!other) {
cur->instruction = CSYNC_INSTRUCTION_NEW;
} else if (other->instruction == CSYNC_INSTRUCTION_NONE
|| cur->type == CSYNC_FTW_TYPE_DIR) {
other->instruction = CSYNC_INSTRUCTION_RENAME;
other->destpath = c_strdup( cur->path );
if( !c_streq(cur->file_id, "") ) {
csync_vio_set_file_id( other->file_id, cur->file_id );
}
cur->instruction = CSYNC_INSTRUCTION_NONE;
} else if (other->instruction == CSYNC_INSTRUCTION_REMOVE) {
other->instruction = CSYNC_INSTRUCTION_RENAME;
other->destpath = c_strdup( cur->path );
if( !c_streq(cur->file_id, "") ) {
csync_vio_set_file_id( other->file_id, cur->file_id );
}
cur->instruction = CSYNC_INSTRUCTION_NONE;
} else if (other->instruction == CSYNC_INSTRUCTION_NEW) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "OOOO=> NEW detected in other tree!");
cur->instruction = CSYNC_INSTRUCTION_CONFLICT;
} else {
cur->instruction = CSYNC_INSTRUCTION_NONE;
other->instruction = CSYNC_INSTRUCTION_SYNC;
}
csync_file_stat_free(tmp);
}
break;
default:
break;
}
} else {
bool is_equal_files = false;
/*
* file found on the other replica
*/
other = (csync_file_stat_t *) node->data;
switch (cur->instruction) {
case CSYNC_INSTRUCTION_EVAL_RENAME:
/* If the file already exist on the other side, we have a conflict.
Abort the rename and consider it is a new file. */
cur->instruction = CSYNC_INSTRUCTION_NEW;
/* fall trough */
/* file on current replica is changed or new */
case CSYNC_INSTRUCTION_EVAL:
case CSYNC_INSTRUCTION_NEW:
// This operation is usually a no-op and will by default return false
if (csync_file_locked_or_open(ctx->local.uri, cur->path)) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "[Reconciler] IGNORING file %s/%s since it is locked / open", ctx->local.uri, cur->path);
cur->instruction = CSYNC_INSTRUCTION_ERROR;
if (cur->error_status == CSYNC_STATUS_OK) // don't overwrite error
cur->error_status = CYSNC_STATUS_FILE_LOCKED_OR_OPEN;
break;
} else {
//CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "[Reconciler] not ignoring file %s/%s", ctx->local.uri, cur->path);
}
switch (other->instruction) {
/* file on other replica is changed or new */
case CSYNC_INSTRUCTION_NEW:
case CSYNC_INSTRUCTION_EVAL:
if (other->type == CSYNC_VIO_FILE_TYPE_DIRECTORY &&
cur->type == CSYNC_VIO_FILE_TYPE_DIRECTORY) {
is_equal_files = (other->modtime == cur->modtime);
} else {
is_equal_files = ((other->size == cur->size) && (other->modtime == cur->modtime));
}
if (is_equal_files) {
/* The files are considered equal. */
cur->instruction = CSYNC_INSTRUCTION_NONE;
other->instruction = CSYNC_INSTRUCTION_NONE;
if( !cur->etag && other->etag ) cur->etag = c_strdup(other->etag);
cur->should_update_etag = true; /* update DB */
} else if(ctx->current == REMOTE_REPLICA) {
cur->instruction = CSYNC_INSTRUCTION_CONFLICT;
other->instruction = CSYNC_INSTRUCTION_NONE;
} else {
cur->instruction = CSYNC_INSTRUCTION_NONE;
other->instruction = CSYNC_INSTRUCTION_CONFLICT;
}
break;
/* file on the other replica has not been modified */
case CSYNC_INSTRUCTION_NONE:
cur->instruction = CSYNC_INSTRUCTION_SYNC;
break;
case CSYNC_INSTRUCTION_IGNORE:
cur->instruction = CSYNC_INSTRUCTION_IGNORE;
break;
default:
break;
}
default:
break;
}
}
//hide instruction NONE messages when log level is set to debug,
//only show these messages on log level trace
if(cur->instruction ==CSYNC_INSTRUCTION_NONE)
{
if(cur->type == CSYNC_FTW_TYPE_DIR)
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,
"%-20s dir: %s",
csync_instruction_str(cur->instruction),
cur->path);
}
else
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,
"%-20s file: %s",
csync_instruction_str(cur->instruction),
cur->path);
}
}
else
{
if(cur->type == CSYNC_FTW_TYPE_DIR)
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG,
"%-20s dir: %s",
csync_instruction_str(cur->instruction),
cur->path);
}
else
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG,
"%-20s file: %s",
csync_instruction_str(cur->instruction),
cur->path);
}
}
return 0;
}
/*
* local visitor which calls the user visitor with repacked stat info.
*/
static int _csync_treewalk_visitor(void *obj, void *data) {
int rc = 0;
csync_file_stat_t *cur = NULL;
CSYNC *ctx = NULL;
c_rbtree_visit_func *visitor = NULL;
_csync_treewalk_context *twctx = NULL;
TREE_WALK_FILE trav;
c_rbtree_t *other_tree = NULL;
c_rbnode_t *other_node = NULL;
cur = (csync_file_stat_t *) obj;
ctx = (CSYNC *) data;
if (ctx == NULL) {
return -1;
}
/* we need the opposite tree! */
switch (ctx->current) {
case LOCAL_REPLICA:
other_tree = ctx->remote.tree;
break;
case REMOTE_REPLICA:
other_tree = ctx->local.tree;
break;
default:
break;
}
other_node = c_rbtree_find(other_tree, &cur->phash);
if (!other_node) {
/* Check the renamed path as well. */
int len;
uint64_t h = 0;
char *renamed_path = csync_rename_adjust_path(ctx, cur->path);
if (!c_streq(renamed_path, cur->path)) {
len = strlen( renamed_path );
h = c_jhash64((uint8_t *) renamed_path, len, 0);
other_node = c_rbtree_find(other_tree, &h);
}
SAFE_FREE(renamed_path);
}
if (obj == NULL || data == NULL) {
ctx->status_code = CSYNC_STATUS_PARAM_ERROR;
return -1;
}
ctx->status_code = CSYNC_STATUS_OK;
twctx = (_csync_treewalk_context*) ctx->callbacks.userdata;
if (twctx == NULL) {
ctx->status_code = CSYNC_STATUS_PARAM_ERROR;
return -1;
}
if (twctx->instruction_filter > 0 &&
!(twctx->instruction_filter & cur->instruction) ) {
return 0;
}
visitor = (c_rbtree_visit_func*)(twctx->user_visitor);
if (visitor != NULL) {
trav.path = cur->path;
trav.size = cur->size;
trav.modtime = cur->modtime;
trav.uid = cur->uid;
trav.gid = cur->gid;
trav.mode = cur->mode;
trav.type = cur->type;
trav.instruction = cur->instruction;
trav.rename_path = cur->destpath;
trav.etag = cur->etag;
trav.file_id = cur->file_id;
trav.error_status = cur->error_status;
trav.should_update_etag = cur->should_update_etag;
if( other_node ) {
csync_file_stat_t *other_stat = (csync_file_stat_t*)other_node->data;
trav.other.etag = other_stat->etag;
trav.other.file_id = other_stat->file_id;
trav.other.instruction = other_stat->instruction;
trav.other.modtime = other_stat->modtime;
trav.other.size = other_stat->size;
} else {
trav.other.etag = 0;
trav.other.file_id = 0;
trav.other.instruction = CSYNC_INSTRUCTION_NONE;
trav.other.modtime = 0;
trav.other.size = 0;
}
rc = (*visitor)(&trav, twctx->userdata);
cur->instruction = trav.instruction;
if (trav.etag != cur->etag) {
SAFE_FREE(cur->etag);
cur->etag = c_strdup(trav.etag);
}
return rc;
}
ctx->status_code = CSYNC_STATUS_PARAM_ERROR;
return -1;
}
/**
* The main function in the reconcile pass.
*
* It's called for each entry in the local and remote rbtrees by
* csync_reconcile()
*
* Before the reconcile phase the trees already know about changes
* relative to the sync journal. This function's job is to spot conflicts
* between local and remote changes and adjust the nodes accordingly.
*
* See doc/dev/sync-algorithm.md for an overview.
*
*
* Older detail comment:
*
* We merge replicas at the file level. The merged replica contains the
* superset of files that are on the local machine and server copies of
* the replica. In the case where the same file is in both the local
* and server copy, the file that was modified most recently is used.
* This means that new files are not deleted, and updated versions of
* existing files are not overwritten.
*
* When a file is updated, the merge algorithm compares the destination
* file with the the source file. If the destination file is newer
* (timestamp is newer), it is not overwritten. If both files, on the
* source and the destination, have been changed, the newer file wins.
*/
static int _csync_merge_algorithm_visitor(void *obj, void *data) {
csync_file_stat_t *cur = NULL;
csync_file_stat_t *other = NULL;
csync_file_stat_t *tmp = NULL;
uint64_t h = 0;
int len = 0;
CSYNC *ctx = NULL;
c_rbtree_t *tree = NULL;
c_rbnode_t *node = NULL;
cur = (csync_file_stat_t *) obj;
ctx = (CSYNC *) data;
/* we need the opposite tree! */
switch (ctx->current) {
case LOCAL_REPLICA:
tree = ctx->remote.tree;
break;
case REMOTE_REPLICA:
tree = ctx->local.tree;
break;
default:
break;
}
node = c_rbtree_find(tree, &cur->phash);
if (!node) {
/* Check the renamed path as well. */
char *renamed_path = csync_rename_adjust_path(ctx, cur->path);
if (!c_streq(renamed_path, cur->path)) {
len = strlen( renamed_path );
h = c_jhash64((uint8_t *) renamed_path, len, 0);
node = c_rbtree_find(tree, &h);
}
SAFE_FREE(renamed_path);
}
if (!node) {
/* Check if it is ignored */
node = _csync_check_ignored(tree, cur->path, cur->pathlen);
/* If it is ignored, other->instruction will be IGNORE so this one will also be ignored */
}
/* file only found on current replica */
if (node == NULL) {
switch(cur->instruction) {
/* file has been modified */
case CSYNC_INSTRUCTION_EVAL:
cur->instruction = CSYNC_INSTRUCTION_NEW;
break;
/* file has been removed on the opposite replica */
case CSYNC_INSTRUCTION_NONE:
case CSYNC_INSTRUCTION_UPDATE_METADATA:
if (cur->has_ignored_files) {
/* Do not remove a directory that has ignored files */
break;
}
if (cur->child_modified) {
/* re-create directory that has modified contents */
cur->instruction = CSYNC_INSTRUCTION_NEW;
break;
}
cur->instruction = CSYNC_INSTRUCTION_REMOVE;
break;
case CSYNC_INSTRUCTION_EVAL_RENAME:
if(ctx->current == LOCAL_REPLICA ) {
/* use the old name to find the "other" node */
tmp = csync_statedb_get_stat_by_inode(ctx, cur->inode);
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Finding opposite temp through inode %" PRIu64 ": %s",
cur->inode, tmp ? "true":"false");
} else if( ctx->current == REMOTE_REPLICA ) {
tmp = csync_statedb_get_stat_by_file_id(ctx, cur->file_id);
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Finding opposite temp through file ID %s: %s",
cur->file_id, tmp ? "true":"false");
} else {
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Unknown replica...");
}
if( tmp ) {
len = strlen( tmp->path );
if( len > 0 ) {
h = c_jhash64((uint8_t *) tmp->path, len, 0);
/* First, check that the file is NOT in our tree (another file with the same name was added) */
node = c_rbtree_find(ctx->current == REMOTE_REPLICA ? ctx->remote.tree : ctx->local.tree, &h);
if (node) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Origin found in our tree : %s", tmp->path);
} else {
/* Find the temporar file in the other tree. */
node = c_rbtree_find(tree, &h);
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "PHash of temporary opposite (%s): %" PRIu64 " %s",
tmp->path , h, node ? "found": "not found" );
if (node) {
other = (csync_file_stat_t*)node->data;
} else {
/* the renamed file could not be found in the opposite tree. That is because it
* is not longer existing there, maybe because it was renamed or deleted.
* The journal is cleaned up later after propagation.
*/
}
}
}
if(!other) {
cur->instruction = CSYNC_INSTRUCTION_NEW;
} else if (other->instruction == CSYNC_INSTRUCTION_NONE
|| other->instruction == CSYNC_INSTRUCTION_UPDATE_METADATA
|| cur->type == CSYNC_FTW_TYPE_DIR) {
other->instruction = CSYNC_INSTRUCTION_RENAME;
other->destpath = c_strdup( cur->path );
if( !c_streq(cur->file_id, "") ) {
csync_vio_set_file_id( other->file_id, cur->file_id );
}
other->inode = cur->inode;
cur->instruction = CSYNC_INSTRUCTION_NONE;
} else if (other->instruction == CSYNC_INSTRUCTION_REMOVE) {
other->instruction = CSYNC_INSTRUCTION_RENAME;
other->destpath = c_strdup( cur->path );
if( !c_streq(cur->file_id, "") ) {
csync_vio_set_file_id( other->file_id, cur->file_id );
}
other->inode = cur->inode;
cur->instruction = CSYNC_INSTRUCTION_NONE;
} else if (other->instruction == CSYNC_INSTRUCTION_NEW) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "OOOO=> NEW detected in other tree!");
cur->instruction = CSYNC_INSTRUCTION_CONFLICT;
} else {
assert(other->type != CSYNC_FTW_TYPE_DIR);
cur->instruction = CSYNC_INSTRUCTION_NONE;
other->instruction = CSYNC_INSTRUCTION_SYNC;
}
csync_file_stat_free(tmp);
}
break;
default:
break;
}
} else {
bool is_conflict = true;
/*
* file found on the other replica
*/
other = (csync_file_stat_t *) node->data;
switch (cur->instruction) {
case CSYNC_INSTRUCTION_UPDATE_METADATA:
if (other->instruction == CSYNC_INSTRUCTION_UPDATE_METADATA && ctx->current == LOCAL_REPLICA) {
// Remote wins, the SyncEngine will pick relevant local metadata since the remote tree is walked last.
cur->instruction = CSYNC_INSTRUCTION_NONE;
}
break;
case CSYNC_INSTRUCTION_EVAL_RENAME:
/* If the file already exist on the other side, we have a conflict.
Abort the rename and consider it is a new file. */
cur->instruction = CSYNC_INSTRUCTION_NEW;
/* fall trough */
/* file on current replica is changed or new */
case CSYNC_INSTRUCTION_EVAL:
case CSYNC_INSTRUCTION_NEW:
// This operation is usually a no-op and will by default return false
if (csync_file_locked_or_open(ctx->local.uri, cur->path)) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "[Reconciler] IGNORING file %s/%s since it is locked / open", ctx->local.uri, cur->path);
cur->instruction = CSYNC_INSTRUCTION_ERROR;
if (cur->error_status == CSYNC_STATUS_OK) // don't overwrite error
cur->error_status = CYSNC_STATUS_FILE_LOCKED_OR_OPEN;
break;
} else {
//CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "[Reconciler] not ignoring file %s/%s", ctx->local.uri, cur->path);
}
switch (other->instruction) {
/* file on other replica is changed or new */
case CSYNC_INSTRUCTION_NEW:
case CSYNC_INSTRUCTION_EVAL:
if (other->type == CSYNC_FTW_TYPE_DIR &&
cur->type == CSYNC_FTW_TYPE_DIR) {
// Folders of the same path are always considered equals
is_conflict = false;
} else {
is_conflict = ((other->size != cur->size) || (other->modtime != cur->modtime));
// FIXME: do a binary comparision of the file here because of the following
// edge case:
// The files could still have different content, even though the mtime
// and size are the same.
}
if (ctx->current == REMOTE_REPLICA) {
// If the files are considered equal, only update the DB with the etag from remote
cur->instruction = is_conflict ? CSYNC_INSTRUCTION_CONFLICT : CSYNC_INSTRUCTION_UPDATE_METADATA;
other->instruction = CSYNC_INSTRUCTION_NONE;
} else {
cur->instruction = CSYNC_INSTRUCTION_NONE;
other->instruction = is_conflict ? CSYNC_INSTRUCTION_CONFLICT : CSYNC_INSTRUCTION_UPDATE_METADATA;
}
break;
/* file on the other replica has not been modified */
case CSYNC_INSTRUCTION_NONE:
case CSYNC_INSTRUCTION_UPDATE_METADATA:
if (cur->type != other->type) {
// If the type of the entity changed, it's like NEW, but
// needs to delete the other entity first.
cur->instruction = CSYNC_INSTRUCTION_TYPE_CHANGE;
other->instruction = CSYNC_INSTRUCTION_NONE;
} else if (cur->type == CSYNC_FTW_TYPE_DIR) {
cur->instruction = CSYNC_INSTRUCTION_UPDATE_METADATA;
other->instruction = CSYNC_INSTRUCTION_NONE;
} else {
cur->instruction = CSYNC_INSTRUCTION_SYNC;
other->instruction = CSYNC_INSTRUCTION_NONE;
}
break;
case CSYNC_INSTRUCTION_IGNORE:
cur->instruction = CSYNC_INSTRUCTION_IGNORE;
break;
default:
break;
}
default:
break;
}
}
//hide instruction NONE messages when log level is set to debug,
//only show these messages on log level trace
const char *repo = ctx->current == REMOTE_REPLICA ? "server" : "client";
if(cur->instruction ==CSYNC_INSTRUCTION_NONE)
{
if(cur->type == CSYNC_FTW_TYPE_DIR)
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,
"%-30s %s dir: %s",
csync_instruction_str(cur->instruction),
repo,
cur->path);
}
else
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,
"%-30s %s file: %s",
csync_instruction_str(cur->instruction),
repo,
cur->path);
}
}
else
{
if(cur->type == CSYNC_FTW_TYPE_DIR)
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG,
"%-30s %s dir: %s",
csync_instruction_str(cur->instruction),
repo,
cur->path);
}
else
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG,
"%-30s %s file: %s",
csync_instruction_str(cur->instruction),
repo,
cur->path);
}
}
return 0;
}
