/* calculate the hash of a given uri */ static uint64_t _hash_of_file(CSYNC *ctx, const char *file) { const char *path; int len; uint64_t h = 0; if( ctx && file ) { path = file; switch (ctx->current) { case LOCAL_REPLICA: if (strlen(path) <= strlen(ctx->local.uri)) { return 0; } path += strlen(ctx->local.uri) + 1; break; case REMOTE_REPLICA: if (strlen(path) <= strlen(ctx->remote.uri)) { return 0; } path += strlen(ctx->remote.uri) + 1; break; default: path = NULL; return 0; break; } len = strlen(path); h = c_jhash64((uint8_t *) path, len, 0); } return h; }
/* Check if a file is ignored because one parent is ignored. * return the node of the ignored directoy if it's the case, or NULL if it is not ignored */ static c_rbnode_t *_csync_check_ignored(c_rbtree_t *tree, const char *path, int pathlen) { uint64_t h = 0; c_rbnode_t *node = NULL; /* compute the size of the parent directory */ int parentlen = pathlen - 1; while (parentlen > 0 && path[parentlen] != '/') { parentlen--; } if (parentlen <= 0) { return NULL; } h = c_jhash64((uint8_t *) path, parentlen, 0); node = c_rbtree_find(tree, &h); if (node) { csync_file_stat_t *n = (csync_file_stat_t*)node->data; if (n->instruction == CSYNC_INSTRUCTION_IGNORE) { /* Yes, we are ignored */ return node; } else { /* Not ignored */ return NULL; } } else { /* Try if the parent itself is ignored */ return _csync_check_ignored(tree, path, parentlen); } }
qint64 SyncJournalDb::getPHash(const QString& file) const { QByteArray utf8File = file.toUtf8(); int64_t h; if( file.isEmpty() ) { return -1; } int len = utf8File.length(); h = c_jhash64((uint8_t *) utf8File.data(), len, 0); return h; }
END_TEST START_TEST (check_c_jhash64_null_strings) { uint8_t buf[1]; uint64_t h, i, t, state[HASHSTATE]; buf[0] = ~0; for (i=0; i<HASHSTATE; ++i) state[i] = 1; for (i=0, h=0; i<8; ++i) { t = h; h = c_jhash64(buf, (uint64_t)0, h); fail_if(t == h, "0-byte-string check failed: t = %.8lx, h = %.8lx", t, h); } }
/* File tree walker */ int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn, unsigned int depth) { char *filename = NULL; char *d_name = NULL; csync_vio_handle_t *dh = NULL; csync_vio_file_stat_t *dirent = NULL; csync_vio_file_stat_t *fs = NULL; csync_file_stat_t *previous_fs = NULL; int read_from_db = 0; int rc = 0; int res = 0; bool do_read_from_db = (ctx->current == REMOTE_REPLICA && ctx->remote.read_from_db); if (uri[0] == '\0') { errno = ENOENT; ctx->status_code = CSYNC_STATUS_PARAM_ERROR; goto error; } read_from_db = ctx->remote.read_from_db; // if the etag of this dir is still the same, its content is restored from the // database. if( do_read_from_db ) { if( ! fill_tree_from_db(ctx, uri) ) { errno = ENOENT; ctx->status_code = CSYNC_STATUS_OPENDIR_ERROR; goto error; } goto done; } if ((dh = csync_vio_opendir(ctx, uri)) == NULL) { int asp = 0; /* permission denied */ ctx->status_code = csync_errno_to_status(errno, CSYNC_STATUS_OPENDIR_ERROR); if (errno == EACCES) { return 0; } else if(errno == ENOENT) { asp = asprintf( &ctx->error_string, "%s", uri); if (asp < 0) { CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "asprintf failed!"); } } else { CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "opendir failed for %s - errno %d", uri, errno); } goto error; } while ((dirent = csync_vio_readdir(ctx, dh))) { const char *path = NULL; size_t ulen = 0; int flen; int flag; d_name = dirent->name; if (d_name == NULL) { ctx->status_code = CSYNC_STATUS_READDIR_ERROR; goto error; } /* skip "." and ".." */ if (d_name[0] == '.' && (d_name[1] == '\0' || (d_name[1] == '.' && d_name[2] == '\0'))) { csync_vio_file_stat_destroy(dirent); dirent = NULL; continue; } flen = asprintf(&filename, "%s/%s", uri, d_name); if (flen < 0) { csync_vio_file_stat_destroy(dirent); dirent = NULL; ctx->status_code = CSYNC_STATUS_MEMORY_ERROR; goto error; } /* Create relative path */ switch (ctx->current) { case LOCAL_REPLICA: ulen = strlen(ctx->local.uri) + 1; break; case REMOTE_REPLICA: ulen = strlen(ctx->remote.uri) + 1; break; default: break; } if (((size_t)flen) < ulen) { csync_vio_file_stat_destroy(dirent); dirent = NULL; ctx->status_code = CSYNC_STATUS_UNSUCCESSFUL; goto error; } path = filename + ulen; /* skip ".csync_journal.db" and ".csync_journal.db.ctmp" */ if (c_streq(path, ".csync_journal.db") || c_streq(path, ".csync_journal.db.ctmp") || c_streq(path, ".csync_journal.db.ctmp-journal") || c_streq(path, ".csync-progressdatabase")) { csync_vio_file_stat_destroy(dirent); dirent = NULL; SAFE_FREE(filename); continue; } /* Only for the local replica we have to stat(), for the remote one we have all data already */ if (ctx->replica == LOCAL_REPLICA) { fs = csync_vio_file_stat_new(); res = csync_vio_stat(ctx, filename, fs); } else { fs = dirent; res = 0; } if( res == 0) { switch (fs->type) { case CSYNC_VIO_FILE_TYPE_SYMBOLIC_LINK: flag = CSYNC_FTW_FLAG_SLINK; break; case CSYNC_VIO_FILE_TYPE_DIRECTORY: flag = CSYNC_FTW_FLAG_DIR; break; case CSYNC_VIO_FILE_TYPE_BLOCK_DEVICE: case CSYNC_VIO_FILE_TYPE_CHARACTER_DEVICE: case CSYNC_VIO_FILE_TYPE_SOCKET: flag = CSYNC_FTW_FLAG_SPEC; break; case CSYNC_VIO_FILE_TYPE_FIFO: flag = CSYNC_FTW_FLAG_SPEC; break; default: flag = CSYNC_FTW_FLAG_FILE; break; }; } else { flag = CSYNC_FTW_FLAG_NSTAT; } if( ctx->current == LOCAL_REPLICA ) { char *etag = NULL; int len = strlen( path ); uint64_t h = c_jhash64((uint8_t *) path, len, 0); etag = csync_statedb_get_etag( ctx, h ); if( etag ) { SAFE_FREE(fs->etag); fs->etag = etag; fs->fields |= CSYNC_VIO_FILE_STAT_FIELDS_ETAG; if( c_streq(etag, "")) { CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Uniq ID from Database is EMPTY: %s", path); } else { CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Uniq ID from Database: %s -> %s", path, fs->etag ? fs->etag : "<NULL>" ); } } } previous_fs = ctx->current_fs; /* Call walker function for each file */ rc = fn(ctx, filename, fs, flag); /* this function may update ctx->current and ctx->read_from_db */ if (ctx->current_fs && previous_fs && ctx->current_fs->child_modified) { previous_fs->child_modified = ctx->current_fs->child_modified; } /* Only for the local replica we have to destroy stat(), for the remote one it is a pointer to dirent */ if (ctx->replica == LOCAL_REPLICA) { csync_vio_file_stat_destroy(fs); } if (rc < 0) { if (CSYNC_STATUS_IS_OK(ctx->status_code)) { ctx->status_code = CSYNC_STATUS_UPDATE_ERROR; } ctx->current_fs = previous_fs; goto error; } if (flag == CSYNC_FTW_FLAG_DIR && depth && (!ctx->current_fs || ctx->current_fs->instruction != CSYNC_INSTRUCTION_IGNORE)) { rc = csync_ftw(ctx, filename, fn, depth - 1); if (rc < 0) { ctx->current_fs = previous_fs; goto error; } if (ctx->current_fs && !ctx->current_fs->child_modified && ctx->current_fs->instruction == CSYNC_INSTRUCTION_EVAL) { ctx->current_fs->instruction = CSYNC_INSTRUCTION_NONE; ctx->current_fs->should_update_etag = true; } } if (flag == CSYNC_FTW_FLAG_DIR && ctx->current_fs && (ctx->current_fs->instruction == CSYNC_INSTRUCTION_EVAL || ctx->current_fs->instruction == CSYNC_INSTRUCTION_NEW || ctx->current_fs->instruction == CSYNC_INSTRUCTION_EVAL_RENAME)) { ctx->current_fs->should_update_etag = true; } ctx->current_fs = previous_fs; ctx->remote.read_from_db = read_from_db; SAFE_FREE(filename); csync_vio_file_stat_destroy(dirent); dirent = NULL; } csync_vio_closedir(ctx, dh); CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, " <= Closing walk for %s with read_from_db %d", uri, read_from_db); done: csync_vio_file_stat_destroy(dirent); SAFE_FREE(filename); return rc; error: ctx->remote.read_from_db = read_from_db; if (dh != NULL) { csync_vio_closedir(ctx, dh); } SAFE_FREE(filename); return -1; }
/* * We merge replicas at the file level. The merged replica contains the * superset of files that are on the local machine and server copies of * the replica. In the case where the same file is in both the local * and server copy, the file that was modified most recently is used. * This means that new files are not deleted, and updated versions of * existing files are not overwritten. * * When a file is updated, the merge algorithm compares the destination * file with the the source file. If the destination file is newer * (timestamp is newer), it is not overwritten. If both files, on the * source and the destination, have been changed, the newer file wins. */ static int _csync_merge_algorithm_visitor(void *obj, void *data) { csync_file_stat_t *cur = NULL; csync_file_stat_t *other = NULL; csync_file_stat_t *tmp = NULL; uint64_t h = 0; int len = 0; CSYNC *ctx = NULL; c_rbtree_t *tree = NULL; c_rbnode_t *node = NULL; cur = (csync_file_stat_t *) obj; ctx = (CSYNC *) data; /* we need the opposite tree! */ switch (ctx->current) { case LOCAL_REPLICA: tree = ctx->remote.tree; break; case REMOTE_REPLCIA: tree = ctx->local.tree; break; default: break; } node = c_rbtree_find(tree, &cur->phash); /* file only found on current replica */ if (node == NULL) { switch(cur->instruction) { /* file has been modified */ case CSYNC_INSTRUCTION_EVAL: cur->instruction = CSYNC_INSTRUCTION_NEW; break; /* file has been removed on the opposite replica */ case CSYNC_INSTRUCTION_NONE: cur->instruction = CSYNC_INSTRUCTION_REMOVE; break; case CSYNC_INSTRUCTION_RENAME: /* rename support only on the local replica because of inode needed. */ if(ctx->current == LOCAL_REPLICA ) { /* use the old name to find the "other" node */ tmp = csync_statedb_get_stat_by_inode(ctx, cur->inode); /* Find the opposite node. */ if( tmp ) { /* We need to calculate the phash again because of the phash being stored as int in db. */ if( tmp->path ) { len = strlen( tmp->path ); h = c_jhash64((uint8_t *) tmp->path, len, 0); CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"PHash of temporar opposite: %llu", h); node = c_rbtree_find(tree, &h); } if(node) { other = (csync_file_stat_t*)node->data; other->instruction = CSYNC_INSTRUCTION_RENAME; other->destpath = c_strdup( cur->path ); cur->instruction = CSYNC_INSTRUCTION_NONE; } if( ! other ) { cur->instruction = CSYNC_INSTRUCTION_NEW; } } } break; default: break; } } else { /* * file found on the other replica */ other = (csync_file_stat_t *) node->data; switch (cur->instruction) { /* file on current replica is new */ case CSYNC_INSTRUCTION_NEW: switch (other->instruction) { /* file on other replica is new too */ case CSYNC_INSTRUCTION_NEW: if (cur->modtime > other->modtime) { if(ctx->options.with_conflict_copys) { CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"file new on both, cur is newer PATH=./%s",cur->path); cur->instruction = CSYNC_INSTRUCTION_CONFLICT; other->instruction = CSYNC_INSTRUCTION_NONE; } else { cur->instruction = CSYNC_INSTRUCTION_SYNC; other->instruction = CSYNC_INSTRUCTION_NONE; } } else if (cur->modtime < other->modtime) { if(ctx->options.with_conflict_copys) { CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"file new on both, other is newer PATH=./%s",cur->path); cur->instruction = CSYNC_INSTRUCTION_NONE; other->instruction = CSYNC_INSTRUCTION_CONFLICT; } else { cur->instruction = CSYNC_INSTRUCTION_NONE; other->instruction = CSYNC_INSTRUCTION_SYNC; } } else { /* file are equal */ cur->instruction = CSYNC_INSTRUCTION_NONE; other->instruction = CSYNC_INSTRUCTION_NONE; } break; /* file on other replica has changed too */ case CSYNC_INSTRUCTION_EVAL: /* file on current replica is newer */ if (cur->modtime > other->modtime) { if(ctx->options.with_conflict_copys) { CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"new on cur, modified on other, cur is newer PATH=./%s",cur->path); cur->instruction = CSYNC_INSTRUCTION_CONFLICT; } else { cur->instruction = CSYNC_INSTRUCTION_SYNC; } } else { /* file on opposite replica is newer */ if(ctx->options.with_conflict_copys) { CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"new on cur, modified on other, other is newer PATH=./%s",cur->path); cur->instruction = CSYNC_INSTRUCTION_NONE; } else { cur->instruction = CSYNC_INSTRUCTION_NONE; } } break; /* file on the other replica has not been modified */ case CSYNC_INSTRUCTION_NONE: cur->instruction = CSYNC_INSTRUCTION_SYNC; break; default: break; } break; /* file on current replica has been modified */ case CSYNC_INSTRUCTION_EVAL: switch (other->instruction) { /* file on other replica is new too */ case CSYNC_INSTRUCTION_NEW: if (cur->modtime > other->modtime) { if(ctx->options.with_conflict_copys) { CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"modified on cur, new on other, cur is newer PATH=./%s",cur->path); cur->instruction = CSYNC_INSTRUCTION_CONFLICT; } else { cur->instruction = CSYNC_INSTRUCTION_SYNC; } } else { if(ctx->options.with_conflict_copys) { CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"modified on cur, new on other, other is newer PATH=./%s",cur->path); cur->instruction = CSYNC_INSTRUCTION_NONE; } else { cur->instruction = CSYNC_INSTRUCTION_NONE; } } break; /* file on other replica has changed too */ case CSYNC_INSTRUCTION_EVAL: /* file on current replica is newer */ if (cur->modtime > other->modtime) { if(ctx->options.with_conflict_copys) { CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"both modified, cur is newer PATH=./%s",cur->path); cur->instruction = CSYNC_INSTRUCTION_CONFLICT; other->instruction= CSYNC_INSTRUCTION_NONE; } else { cur->instruction = CSYNC_INSTRUCTION_SYNC; } } else { /* file on opposite replica is newer */ if(ctx->options.with_conflict_copys) { CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"both modified, other is newer PATH=./%s",cur->path); cur->instruction = CSYNC_INSTRUCTION_NONE; other->instruction=CSYNC_INSTRUCTION_CONFLICT; } else { cur->instruction = CSYNC_INSTRUCTION_NONE; } } break; /* file on the other replica has not been modified */ case CSYNC_INSTRUCTION_NONE: cur->instruction = CSYNC_INSTRUCTION_SYNC; break; default: break; } break; default: break; } } //hide instruction NONE messages when log level is set to debug, //only show these messages on log level trace if(cur->instruction ==CSYNC_INSTRUCTION_NONE) { if(cur->type == CSYNC_FTW_TYPE_DIR) { CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "%-20s dir: %s", csync_instruction_str(cur->instruction), cur->path); } else { CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "%-20s file: %s", csync_instruction_str(cur->instruction), cur->path); } } else { if(cur->type == CSYNC_FTW_TYPE_DIR) { CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "%-20s dir: %s", csync_instruction_str(cur->instruction), cur->path); } else { CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "%-20s file: %s", csync_instruction_str(cur->instruction), cur->path); } } return 0; }
/* * local visitor which calls the user visitor with repacked stat info. */ static int _csync_treewalk_visitor(void *obj, void *data) { int rc = 0; csync_file_stat_t *cur = NULL; CSYNC *ctx = NULL; c_rbtree_visit_func *visitor = NULL; _csync_treewalk_context *twctx = NULL; TREE_WALK_FILE trav; c_rbtree_t *other_tree = NULL; c_rbnode_t *other_node = NULL; cur = (csync_file_stat_t *) obj; ctx = (CSYNC *) data; if (ctx == NULL) { return -1; } /* we need the opposite tree! */ switch (ctx->current) { case LOCAL_REPLICA: other_tree = ctx->remote.tree; break; case REMOTE_REPLICA: other_tree = ctx->local.tree; break; default: break; } other_node = c_rbtree_find(other_tree, &cur->phash); if (!other_node) { /* Check the renamed path as well. */ int len; uint64_t h = 0; char *renamed_path = csync_rename_adjust_path(ctx, cur->path); if (!c_streq(renamed_path, cur->path)) { len = strlen( renamed_path ); h = c_jhash64((uint8_t *) renamed_path, len, 0); other_node = c_rbtree_find(other_tree, &h); } SAFE_FREE(renamed_path); } if (obj == NULL || data == NULL) { ctx->status_code = CSYNC_STATUS_PARAM_ERROR; return -1; } ctx->status_code = CSYNC_STATUS_OK; twctx = (_csync_treewalk_context*) ctx->callbacks.userdata; if (twctx == NULL) { ctx->status_code = CSYNC_STATUS_PARAM_ERROR; return -1; } if (twctx->instruction_filter > 0 && !(twctx->instruction_filter & cur->instruction) ) { return 0; } visitor = (c_rbtree_visit_func*)(twctx->user_visitor); if (visitor != NULL) { trav.path = cur->path; trav.size = cur->size; trav.modtime = cur->modtime; trav.uid = cur->uid; trav.gid = cur->gid; trav.mode = cur->mode; trav.type = cur->type; trav.instruction = cur->instruction; trav.rename_path = cur->destpath; trav.etag = cur->etag; trav.file_id = cur->file_id; trav.error_status = cur->error_status; trav.should_update_etag = cur->should_update_etag; if( other_node ) { csync_file_stat_t *other_stat = (csync_file_stat_t*)other_node->data; trav.other.etag = other_stat->etag; trav.other.file_id = other_stat->file_id; trav.other.instruction = other_stat->instruction; trav.other.modtime = other_stat->modtime; trav.other.size = other_stat->size; } else { trav.other.etag = 0; trav.other.file_id = 0; trav.other.instruction = CSYNC_INSTRUCTION_NONE; trav.other.modtime = 0; trav.other.size = 0; } rc = (*visitor)(&trav, twctx->userdata); cur->instruction = trav.instruction; if (trav.etag != cur->etag) { SAFE_FREE(cur->etag); cur->etag = c_strdup(trav.etag); } return rc; } ctx->status_code = CSYNC_STATUS_PARAM_ERROR; return -1; }
END_TEST START_TEST (check_c_jhash64_alignment_problems) { uint8_t buf[MAXLEN+20], *b; uint64_t len; uint8_t q[] = "This is the time for all good men to come to the aid of their country"; uint8_t qq[] = "xThis is the time for all good men to come to the aid of their country"; uint8_t qqq[] = "xxThis is the time for all good men to come to the aid of their country"; uint8_t qqqq[] = "xxxThis is the time for all good men to come to the aid of their country"; uint8_t o[] = "xxxxThis is the time for all good men to come to the aid of their country"; uint8_t oo[] = "xxxxxThis is the time for all good men to come to the aid of their country"; uint8_t ooo[] = "xxxxxxThis is the time for all good men to come to the aid of their country"; uint8_t oooo[] = "xxxxxxxThis is the time for all good men to come to the aid of their country"; uint64_t h,i,j,ref,t,x,y; h = c_jhash64(q+0, (uint64_t)(sizeof(q)-1), (uint64_t)0); t = h; fail_unless(t == h, "%.8lx%.8lx\n", (uint32_t)h, (uint32_t)(h>>32)); h = c_jhash64(qq+1, (uint64_t)(sizeof(q)-1), (uint64_t)0); fail_unless(t == h, "%.8lx%.8lx\n", (uint32_t)h, (uint32_t)(h>>32)); h = c_jhash64(qqq+2, (uint64_t)(sizeof(q)-1), (uint64_t)0); fail_unless(t == h, "%.8lx%.8lx\n", (uint32_t)h, (uint32_t)(h>>32)); h = c_jhash64(qqqq+3, (uint64_t)(sizeof(q)-1), (uint64_t)0); fail_unless(t == h, "%.8lx%.8lx\n", (uint32_t)h, (uint32_t)(h>>32)); h = c_jhash64(o+4, (uint64_t)(sizeof(q)-1), (uint64_t)0); fail_unless(t == h, "%.8lx%.8lx\n", (uint32_t)h, (uint32_t)(h>>32)); h = c_jhash64(oo+5, (uint64_t)(sizeof(q)-1), (uint64_t)0); fail_unless(t == h, "%.8lx%.8lx\n", (uint32_t)h, (uint32_t)(h>>32)); h = c_jhash64(ooo+6, (uint64_t)(sizeof(q)-1), (uint64_t)0); fail_unless(t == h, "%.8lx%.8lx\n", (uint32_t)h, (uint32_t)(h>>32)); h = c_jhash64(oooo+7, (uint64_t)(sizeof(q)-1), (uint64_t)0); fail_unless(t == h, "%.8lx%.8lx\n", (uint32_t)h, (uint32_t)(h>>32)); for (h=0, b=buf+1; h<8; ++h, ++b) { for (i=0; i<MAXLEN; ++i) { len = i; for (j=0; j<i; ++j) *(b+j)=0; /* these should all be equal */ ref = c_jhash64(b, len, (uint64_t)1); *(b+i)=(uint8_t)~0; *(b-1)=(uint8_t)~0; x = c_jhash64(b, len, (uint64_t)1); y = c_jhash64(b, len, (uint64_t)1); fail_if((ref != x) || (ref != y), "alignment error: %.8lx %.8lx %.8lx %ld %ld\n", ref, x, y, h, i); } } }
END_TEST START_TEST (check_c_jhash64_trials) { uint8_t qa[MAXLEN + 1], qb[MAXLEN + 2]; uint8_t *a, *b; uint64_t c[HASHSTATE], d[HASHSTATE], i, j=0, k, l, m, z; uint64_t e[HASHSTATE],f[HASHSTATE],g[HASHSTATE],h[HASHSTATE]; uint64_t x[HASHSTATE],y[HASHSTATE]; uint64_t hlen; a = &qa[0]; b = &qb[1]; for (hlen=0; hlen < MAXLEN; ++hlen) { z=0; for (i=0; i<hlen; ++i) { /*----------------------- for each byte, */ for (j=0; j<8; ++j) { /*------------------------ for each bit, */ for (m=0; m<8; ++m) { /*-------- for serveral possible levels, */ for (l=0; l<HASHSTATE; ++l) e[l]=f[l]=g[l]=h[l]=x[l]=y[l]=~((uint64_t)0); /*---- check that every input bit affects every output bit */ for (k=0; k<MAXPAIR; k+=2) { uint64_t finished=1; /* keys have one bit different */ for (l=0; l<hlen+1; ++l) {a[l] = b[l] = (uint8_t)0;} /* have a and b be two keys differing in only one bit */ a[i] ^= (k<<j); a[i] ^= (k>>(8-j)); c[0] = c_jhash64(a, hlen, m); b[i] ^= ((k+1)<<j); b[i] ^= ((k+1)>>(8-j)); d[0] = c_jhash64(b, hlen, m); /* check every bit is 1, 0, set, and not set at least once */ for (l=0; l<HASHSTATE; ++l) { e[l] &= (c[l]^d[l]); f[l] &= ~(c[l]^d[l]); g[l] &= c[l]; h[l] &= ~c[l]; x[l] &= d[l]; y[l] &= ~d[l]; if (e[l]|f[l]|g[l]|h[l]|x[l]|y[l]) finished=0; } if (finished) break; } if (k>z) z=k; if (k==MAXPAIR) { printf("Some bit didn't change: "); printf("%.8llx %.8llx %.8llx %.8llx %.8llx %.8llx ", (long long unsigned int) e[0], (long long unsigned int) f[0], (long long unsigned int) g[0], (long long unsigned int) h[0], (long long unsigned int) x[0], (long long unsigned int) y[0]); printf("i %d j %d m %d len %d\n", (uint32_t)i,(uint32_t)j,(uint32_t)m,(uint32_t)hlen); } if (z==MAXPAIR) goto done; } } } done: if (z < MAXPAIR) { fail_unless(z < MAXPAIR, "%ld trials needed, should be less than 40", z/2); } } }
static int _csync_detect_update(CSYNC *ctx, const char *file, const csync_vio_file_stat_t *fs, const int type) { uint64_t h = 0; size_t len = 0; size_t size = 0; const char *path = NULL; csync_file_stat_t *st = NULL; csync_file_stat_t *tmp = NULL; if ((file == NULL) || (fs == NULL)) { errno = EINVAL; return -1; } path = file; switch (ctx->current) { case LOCAL_REPLICA: if (strlen(path) <= strlen(ctx->local.uri)) { return -1; } path += strlen(ctx->local.uri) + 1; break; case REMOTE_REPLCIA: if (strlen(path) <= strlen(ctx->remote.uri)) { return -1; } path += strlen(ctx->remote.uri) + 1; break; default: path = NULL; return -1; break; } len = strlen(path); h = c_jhash64((uint8_t *) path, len, 0); size = sizeof(csync_file_stat_t) + len + 1; st = c_malloc(size); if (st == NULL) { return -1; } CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "file: %s - hash %llu, st size: %zu", path, (long long unsigned int) h, size); /* Set instruction by default to none */ st->instruction = CSYNC_INSTRUCTION_NONE; /* check hardlink count */ if (type == CSYNC_FTW_TYPE_FILE && fs->nlink > 1) { st->instruction = CSYNC_INSTRUCTION_IGNORE; goto out; } /* Update detection */ if (csync_get_statedb_exists(ctx)) { tmp = csync_statedb_get_stat_by_hash(ctx, h); if (tmp && tmp->phash == h) { /* we have an update! */ if (fs->mtime > tmp->modtime) { st->instruction = CSYNC_INSTRUCTION_EVAL; goto out; } st->instruction = CSYNC_INSTRUCTION_NONE; } else { /* check if the file has been renamed */ if (ctx->current == LOCAL_REPLICA) { tmp = csync_statedb_get_stat_by_inode(ctx, fs->inode); if (tmp && tmp->inode == fs->inode) { /* inode found so the file has been renamed */ st->instruction = CSYNC_INSTRUCTION_RENAME; goto out; } else { /* file not found in statedb */ st->instruction = CSYNC_INSTRUCTION_NEW; goto out; } } /* remote and file not found in statedb */ st->instruction = CSYNC_INSTRUCTION_NEW; } } else { st->instruction = CSYNC_INSTRUCTION_NEW; } out: SAFE_FREE(tmp); st->inode = fs->inode; st->mode = fs->mode; st->size = fs->size; st->modtime = fs->mtime; st->uid = fs->uid; st->gid = fs->gid; st->nlink = fs->nlink; st->type = type; st->phash = h; st->pathlen = len; memcpy(st->path, (len ? path : ""), len + 1); switch (ctx->current) { case LOCAL_REPLICA: if (c_rbtree_insert(ctx->local.tree, (void *) st) < 0) { SAFE_FREE(st); return -1; } break; case REMOTE_REPLCIA: if (c_rbtree_insert(ctx->remote.tree, (void *) st) < 0) { SAFE_FREE(st); return -1; } break; default: break; } CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "file: %s, instruction: %s", st->path, csync_instruction_str(st->instruction)); return 0; }
/* * We merge replicas at the file level. The merged replica contains the * superset of files that are on the local machine and server copies of * the replica. In the case where the same file is in both the local * and server copy, the file that was modified most recently is used. * This means that new files are not deleted, and updated versions of * existing files are not overwritten. * * When a file is updated, the merge algorithm compares the destination * file with the the source file. If the destination file is newer * (timestamp is newer), it is not overwritten. If both files, on the * source and the destination, have been changed, the newer file wins. */ static int _csync_merge_algorithm_visitor(void *obj, void *data) { csync_file_stat_t *cur = NULL; csync_file_stat_t *other = NULL; csync_file_stat_t *tmp = NULL; uint64_t h = 0; int len = 0; CSYNC *ctx = NULL; c_rbtree_t *tree = NULL; c_rbnode_t *node = NULL; cur = (csync_file_stat_t *) obj; ctx = (CSYNC *) data; /* we need the opposite tree! */ switch (ctx->current) { case LOCAL_REPLICA: tree = ctx->remote.tree; break; case REMOTE_REPLICA: tree = ctx->local.tree; break; default: break; } node = c_rbtree_find(tree, &cur->phash); if (!node) { /* Check the renamed path as well. */ char *renamed_path = csync_rename_adjust_path(ctx, cur->path); if (!c_streq(renamed_path, cur->path)) { len = strlen( renamed_path ); h = c_jhash64((uint8_t *) renamed_path, len, 0); node = c_rbtree_find(tree, &h); } SAFE_FREE(renamed_path); } if (!node) { /* Check if it is ignored */ node = _csync_check_ignored(tree, cur->path, cur->pathlen); /* If it is ignored, other->instruction will be IGNORE so this one will also be ignored */ } /* file only found on current replica */ if (node == NULL) { switch(cur->instruction) { /* file has been modified */ case CSYNC_INSTRUCTION_EVAL: cur->instruction = CSYNC_INSTRUCTION_NEW; break; /* file has been removed on the opposite replica */ case CSYNC_INSTRUCTION_NONE: cur->instruction = CSYNC_INSTRUCTION_REMOVE; break; case CSYNC_INSTRUCTION_EVAL_RENAME: if(ctx->current == LOCAL_REPLICA ) { /* use the old name to find the "other" node */ tmp = csync_statedb_get_stat_by_inode(ctx, cur->inode); CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Finding opposite temp through inode %" PRIu64 ": %s", cur->inode, tmp ? "true":"false"); } else if( ctx->current == REMOTE_REPLICA ) { tmp = csync_statedb_get_stat_by_file_id(ctx, cur->file_id); CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Finding opposite temp through file ID %s: %s", cur->file_id, tmp ? "true":"false"); } else { CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Unknown replica..."); } if( tmp ) { if( tmp->path ) { len = strlen( tmp->path ); h = c_jhash64((uint8_t *) tmp->path, len, 0); /* First, check that the file is NOT in our tree (another file with the same name was added) */ node = c_rbtree_find(ctx->current == REMOTE_REPLICA ? ctx->remote.tree : ctx->local.tree, &h); if (node) { CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Origin found in our tree : %s", tmp->path); } else { /* Find the temporar file in the other tree. */ node = c_rbtree_find(tree, &h); CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "PHash of temporary opposite (%s): %" PRIu64 " %s", tmp->path , h, node ? "found": "not found" ); if (node) { other = (csync_file_stat_t*)node->data; } else { /* the renamed file could not be found in the opposite tree. That is because it * is not longer existing there, maybe because it was renamed or deleted. * The journal is cleaned up later after propagation. */ } } } if(!other) { cur->instruction = CSYNC_INSTRUCTION_NEW; } else if (other->instruction == CSYNC_INSTRUCTION_NONE || cur->type == CSYNC_FTW_TYPE_DIR) { other->instruction = CSYNC_INSTRUCTION_RENAME; other->destpath = c_strdup( cur->path ); if( !c_streq(cur->file_id, "") ) { csync_vio_set_file_id( other->file_id, cur->file_id ); } cur->instruction = CSYNC_INSTRUCTION_NONE; } else if (other->instruction == CSYNC_INSTRUCTION_REMOVE) { other->instruction = CSYNC_INSTRUCTION_RENAME; other->destpath = c_strdup( cur->path ); if( !c_streq(cur->file_id, "") ) { csync_vio_set_file_id( other->file_id, cur->file_id ); } cur->instruction = CSYNC_INSTRUCTION_NONE; } else if (other->instruction == CSYNC_INSTRUCTION_NEW) { CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "OOOO=> NEW detected in other tree!"); cur->instruction = CSYNC_INSTRUCTION_CONFLICT; } else { cur->instruction = CSYNC_INSTRUCTION_NONE; other->instruction = CSYNC_INSTRUCTION_SYNC; } csync_file_stat_free(tmp); } break; default: break; } } else { bool is_equal_files = false; /* * file found on the other replica */ other = (csync_file_stat_t *) node->data; switch (cur->instruction) { case CSYNC_INSTRUCTION_EVAL_RENAME: /* If the file already exist on the other side, we have a conflict. Abort the rename and consider it is a new file. */ cur->instruction = CSYNC_INSTRUCTION_NEW; /* fall trough */ /* file on current replica is changed or new */ case CSYNC_INSTRUCTION_EVAL: case CSYNC_INSTRUCTION_NEW: // This operation is usually a no-op and will by default return false if (csync_file_locked_or_open(ctx->local.uri, cur->path)) { CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "[Reconciler] IGNORING file %s/%s since it is locked / open", ctx->local.uri, cur->path); cur->instruction = CSYNC_INSTRUCTION_ERROR; if (cur->error_status == CSYNC_STATUS_OK) // don't overwrite error cur->error_status = CYSNC_STATUS_FILE_LOCKED_OR_OPEN; break; } else { //CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "[Reconciler] not ignoring file %s/%s", ctx->local.uri, cur->path); } switch (other->instruction) { /* file on other replica is changed or new */ case CSYNC_INSTRUCTION_NEW: case CSYNC_INSTRUCTION_EVAL: if (other->type == CSYNC_VIO_FILE_TYPE_DIRECTORY && cur->type == CSYNC_VIO_FILE_TYPE_DIRECTORY) { is_equal_files = (other->modtime == cur->modtime); } else { is_equal_files = ((other->size == cur->size) && (other->modtime == cur->modtime)); } if (is_equal_files) { /* The files are considered equal. */ cur->instruction = CSYNC_INSTRUCTION_NONE; other->instruction = CSYNC_INSTRUCTION_NONE; if( !cur->etag && other->etag ) cur->etag = c_strdup(other->etag); cur->should_update_etag = true; /* update DB */ } else if(ctx->current == REMOTE_REPLICA) { cur->instruction = CSYNC_INSTRUCTION_CONFLICT; other->instruction = CSYNC_INSTRUCTION_NONE; } else { cur->instruction = CSYNC_INSTRUCTION_NONE; other->instruction = CSYNC_INSTRUCTION_CONFLICT; } break; /* file on the other replica has not been modified */ case CSYNC_INSTRUCTION_NONE: cur->instruction = CSYNC_INSTRUCTION_SYNC; break; case CSYNC_INSTRUCTION_IGNORE: cur->instruction = CSYNC_INSTRUCTION_IGNORE; break; default: break; } default: break; } } //hide instruction NONE messages when log level is set to debug, //only show these messages on log level trace if(cur->instruction ==CSYNC_INSTRUCTION_NONE) { if(cur->type == CSYNC_FTW_TYPE_DIR) { CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "%-20s dir: %s", csync_instruction_str(cur->instruction), cur->path); } else { CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "%-20s file: %s", csync_instruction_str(cur->instruction), cur->path); } } else { if(cur->type == CSYNC_FTW_TYPE_DIR) { CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "%-20s dir: %s", csync_instruction_str(cur->instruction), cur->path); } else { CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "%-20s file: %s", csync_instruction_str(cur->instruction), cur->path); } } return 0; }
/** * The main function in the reconcile pass. * * It's called for each entry in the local and remote rbtrees by * csync_reconcile() * * Before the reconcile phase the trees already know about changes * relative to the sync journal. This function's job is to spot conflicts * between local and remote changes and adjust the nodes accordingly. * * See doc/dev/sync-algorithm.md for an overview. * * * Older detail comment: * * We merge replicas at the file level. The merged replica contains the * superset of files that are on the local machine and server copies of * the replica. In the case where the same file is in both the local * and server copy, the file that was modified most recently is used. * This means that new files are not deleted, and updated versions of * existing files are not overwritten. * * When a file is updated, the merge algorithm compares the destination * file with the the source file. If the destination file is newer * (timestamp is newer), it is not overwritten. If both files, on the * source and the destination, have been changed, the newer file wins. */ static int _csync_merge_algorithm_visitor(void *obj, void *data) { csync_file_stat_t *cur = NULL; csync_file_stat_t *other = NULL; csync_file_stat_t *tmp = NULL; uint64_t h = 0; int len = 0; CSYNC *ctx = NULL; c_rbtree_t *tree = NULL; c_rbnode_t *node = NULL; cur = (csync_file_stat_t *) obj; ctx = (CSYNC *) data; /* we need the opposite tree! */ switch (ctx->current) { case LOCAL_REPLICA: tree = ctx->remote.tree; break; case REMOTE_REPLICA: tree = ctx->local.tree; break; default: break; } node = c_rbtree_find(tree, &cur->phash); if (!node) { /* Check the renamed path as well. */ char *renamed_path = csync_rename_adjust_path(ctx, cur->path); if (!c_streq(renamed_path, cur->path)) { len = strlen( renamed_path ); h = c_jhash64((uint8_t *) renamed_path, len, 0); node = c_rbtree_find(tree, &h); } SAFE_FREE(renamed_path); } if (!node) { /* Check if it is ignored */ node = _csync_check_ignored(tree, cur->path, cur->pathlen); /* If it is ignored, other->instruction will be IGNORE so this one will also be ignored */ } /* file only found on current replica */ if (node == NULL) { switch(cur->instruction) { /* file has been modified */ case CSYNC_INSTRUCTION_EVAL: cur->instruction = CSYNC_INSTRUCTION_NEW; break; /* file has been removed on the opposite replica */ case CSYNC_INSTRUCTION_NONE: case CSYNC_INSTRUCTION_UPDATE_METADATA: if (cur->has_ignored_files) { /* Do not remove a directory that has ignored files */ break; } if (cur->child_modified) { /* re-create directory that has modified contents */ cur->instruction = CSYNC_INSTRUCTION_NEW; break; } cur->instruction = CSYNC_INSTRUCTION_REMOVE; break; case CSYNC_INSTRUCTION_EVAL_RENAME: if(ctx->current == LOCAL_REPLICA ) { /* use the old name to find the "other" node */ tmp = csync_statedb_get_stat_by_inode(ctx, cur->inode); CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Finding opposite temp through inode %" PRIu64 ": %s", cur->inode, tmp ? "true":"false"); } else if( ctx->current == REMOTE_REPLICA ) { tmp = csync_statedb_get_stat_by_file_id(ctx, cur->file_id); CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Finding opposite temp through file ID %s: %s", cur->file_id, tmp ? "true":"false"); } else { CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Unknown replica..."); } if( tmp ) { len = strlen( tmp->path ); if( len > 0 ) { h = c_jhash64((uint8_t *) tmp->path, len, 0); /* First, check that the file is NOT in our tree (another file with the same name was added) */ node = c_rbtree_find(ctx->current == REMOTE_REPLICA ? ctx->remote.tree : ctx->local.tree, &h); if (node) { CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Origin found in our tree : %s", tmp->path); } else { /* Find the temporar file in the other tree. */ node = c_rbtree_find(tree, &h); CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "PHash of temporary opposite (%s): %" PRIu64 " %s", tmp->path , h, node ? "found": "not found" ); if (node) { other = (csync_file_stat_t*)node->data; } else { /* the renamed file could not be found in the opposite tree. That is because it * is not longer existing there, maybe because it was renamed or deleted. * The journal is cleaned up later after propagation. */ } } } if(!other) { cur->instruction = CSYNC_INSTRUCTION_NEW; } else if (other->instruction == CSYNC_INSTRUCTION_NONE || other->instruction == CSYNC_INSTRUCTION_UPDATE_METADATA || cur->type == CSYNC_FTW_TYPE_DIR) { other->instruction = CSYNC_INSTRUCTION_RENAME; other->destpath = c_strdup( cur->path ); if( !c_streq(cur->file_id, "") ) { csync_vio_set_file_id( other->file_id, cur->file_id ); } other->inode = cur->inode; cur->instruction = CSYNC_INSTRUCTION_NONE; } else if (other->instruction == CSYNC_INSTRUCTION_REMOVE) { other->instruction = CSYNC_INSTRUCTION_RENAME; other->destpath = c_strdup( cur->path ); if( !c_streq(cur->file_id, "") ) { csync_vio_set_file_id( other->file_id, cur->file_id ); } other->inode = cur->inode; cur->instruction = CSYNC_INSTRUCTION_NONE; } else if (other->instruction == CSYNC_INSTRUCTION_NEW) { CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "OOOO=> NEW detected in other tree!"); cur->instruction = CSYNC_INSTRUCTION_CONFLICT; } else { assert(other->type != CSYNC_FTW_TYPE_DIR); cur->instruction = CSYNC_INSTRUCTION_NONE; other->instruction = CSYNC_INSTRUCTION_SYNC; } csync_file_stat_free(tmp); } break; default: break; } } else { bool is_conflict = true; /* * file found on the other replica */ other = (csync_file_stat_t *) node->data; switch (cur->instruction) { case CSYNC_INSTRUCTION_UPDATE_METADATA: if (other->instruction == CSYNC_INSTRUCTION_UPDATE_METADATA && ctx->current == LOCAL_REPLICA) { // Remote wins, the SyncEngine will pick relevant local metadata since the remote tree is walked last. cur->instruction = CSYNC_INSTRUCTION_NONE; } break; case CSYNC_INSTRUCTION_EVAL_RENAME: /* If the file already exist on the other side, we have a conflict. Abort the rename and consider it is a new file. */ cur->instruction = CSYNC_INSTRUCTION_NEW; /* fall trough */ /* file on current replica is changed or new */ case CSYNC_INSTRUCTION_EVAL: case CSYNC_INSTRUCTION_NEW: // This operation is usually a no-op and will by default return false if (csync_file_locked_or_open(ctx->local.uri, cur->path)) { CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "[Reconciler] IGNORING file %s/%s since it is locked / open", ctx->local.uri, cur->path); cur->instruction = CSYNC_INSTRUCTION_ERROR; if (cur->error_status == CSYNC_STATUS_OK) // don't overwrite error cur->error_status = CYSNC_STATUS_FILE_LOCKED_OR_OPEN; break; } else { //CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "[Reconciler] not ignoring file %s/%s", ctx->local.uri, cur->path); } switch (other->instruction) { /* file on other replica is changed or new */ case CSYNC_INSTRUCTION_NEW: case CSYNC_INSTRUCTION_EVAL: if (other->type == CSYNC_FTW_TYPE_DIR && cur->type == CSYNC_FTW_TYPE_DIR) { // Folders of the same path are always considered equals is_conflict = false; } else { is_conflict = ((other->size != cur->size) || (other->modtime != cur->modtime)); // FIXME: do a binary comparision of the file here because of the following // edge case: // The files could still have different content, even though the mtime // and size are the same. } if (ctx->current == REMOTE_REPLICA) { // If the files are considered equal, only update the DB with the etag from remote cur->instruction = is_conflict ? CSYNC_INSTRUCTION_CONFLICT : CSYNC_INSTRUCTION_UPDATE_METADATA; other->instruction = CSYNC_INSTRUCTION_NONE; } else { cur->instruction = CSYNC_INSTRUCTION_NONE; other->instruction = is_conflict ? CSYNC_INSTRUCTION_CONFLICT : CSYNC_INSTRUCTION_UPDATE_METADATA; } break; /* file on the other replica has not been modified */ case CSYNC_INSTRUCTION_NONE: case CSYNC_INSTRUCTION_UPDATE_METADATA: if (cur->type != other->type) { // If the type of the entity changed, it's like NEW, but // needs to delete the other entity first. cur->instruction = CSYNC_INSTRUCTION_TYPE_CHANGE; other->instruction = CSYNC_INSTRUCTION_NONE; } else if (cur->type == CSYNC_FTW_TYPE_DIR) { cur->instruction = CSYNC_INSTRUCTION_UPDATE_METADATA; other->instruction = CSYNC_INSTRUCTION_NONE; } else { cur->instruction = CSYNC_INSTRUCTION_SYNC; other->instruction = CSYNC_INSTRUCTION_NONE; } break; case CSYNC_INSTRUCTION_IGNORE: cur->instruction = CSYNC_INSTRUCTION_IGNORE; break; default: break; } default: break; } } //hide instruction NONE messages when log level is set to debug, //only show these messages on log level trace const char *repo = ctx->current == REMOTE_REPLICA ? "server" : "client"; if(cur->instruction ==CSYNC_INSTRUCTION_NONE) { if(cur->type == CSYNC_FTW_TYPE_DIR) { CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "%-30s %s dir: %s", csync_instruction_str(cur->instruction), repo, cur->path); } else { CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "%-30s %s file: %s", csync_instruction_str(cur->instruction), repo, cur->path); } } else { if(cur->type == CSYNC_FTW_TYPE_DIR) { CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "%-30s %s dir: %s", csync_instruction_str(cur->instruction), repo, cur->path); } else { CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "%-30s %s file: %s", csync_instruction_str(cur->instruction), repo, cur->path); } } return 0; }