/* Remove possible quotes, and also the -gzip at the end
* Remove "-gzip" at the end (cf. https://github.comowncloud/client/issues/1195)
* The caller must take ownership of the resulting string.
*/
char *csync_normalize_etag(const char *etag)
{
int len = 0;
char *buf = NULL;
if (!etag)
return NULL;
len = strlen(etag);
/* strip "XXXX-gzip" */
if(len >= 7 && etag[0] == '"' && c_streq(etag + len - 6, "-gzip\"")) {
etag++;
len -= 7;
}
/* strip leading -gzip */
if(len >= 5 && c_streq(etag + len - 5, "-gzip")) {
len -= 5;
}
/* strip normal quotes */
if (etag[0] == '"' && etag[len-1] == '"') {
etag++;
len -= 2;
}
buf = c_malloc( len+1 );
strncpy( buf, etag, len );
buf[len] = '\0';
return buf;
}
END_TEST
START_TEST (check_c_streq_null)
{
fail_if(c_streq(NULL, "test"), NULL);
fail_if(c_streq("test", NULL), NULL);
fail_if(c_streq(NULL, NULL), NULL);
}
static void install_content_reader( ne_request *req, void *userdata, const ne_status *status )
{
const char *enc = NULL;
struct transfer_context *writeCtx = userdata;
(void) status;
if( !writeCtx ) {
DEBUG_WEBDAV("Error: install_content_reader called without valid write context!");
return;
}
enc = ne_get_response_header( req, "Content-Encoding" );
if( status && status->code != 200 ) {
DEBUG_WEBDAV("Content encoding ist <%s> with status %d", enc ? enc : "empty",
status ? status->code : -1 );
}
if( enc && c_streq( enc, "gzip" )) {
writeCtx->decompress = ne_decompress_reader( req, ne_accept_2xx,
compress_reader, /* reader callback */
(void*) writeCtx ); /* userdata */
} else {
ne_add_response_body_reader( req, ne_accept_2xx,
uncompress_reader,
(void*) writeCtx );
writeCtx->decompress = NULL;
}
}
csync_file_stat_t *csync_statedb_get_stat_by_file_id( sqlite3 *db,
const char *file_id ) {
csync_file_stat_t *st = NULL;
c_strlist_t *result = NULL;
char *stmt = NULL;
size_t len = 0;
if (!file_id) {
return 0;
}
if (c_streq(file_id, "")) {
return 0;
}
stmt = sqlite3_mprintf("SELECT * FROM metadata WHERE fileid='%q'",
file_id);
if (stmt == NULL) {
return NULL;
}
result = csync_statedb_query(db, stmt);
sqlite3_free(stmt);
if (result == NULL) {
return NULL;
}
if (result->count <= 6) {
c_strlist_destroy(result);
return NULL;
}
/* phash, pathlen, path, inode, uid, gid, mode, modtime */
len = strlen(result->vector[2]);
st = c_malloc(sizeof(csync_file_stat_t) + len + 1);
if (st == NULL) {
c_strlist_destroy(result);
return NULL;
}
/* clear the whole structure */
ZERO_STRUCTP(st);
st->phash = atoll(result->vector[0]);
st->pathlen = atoi(result->vector[1]);
memcpy(st->path, (len ? result->vector[2] : ""), len + 1);
st->inode = atoll(result->vector[3]);
st->uid = atoi(result->vector[4]);
st->gid = atoi(result->vector[5]);
st->mode = atoi(result->vector[6]);
st->modtime = strtoul(result->vector[7], NULL, 10);
st->type = atoi(result->vector[8]);
if( result->vector[9] )
st->etag = c_strdup(result->vector[9]);
csync_vio_set_file_id(st->file_id, file_id);
c_strlist_destroy(result);
return st;
}
void csync_vio_file_stat_set_file_id( csync_vio_file_stat_t *dst, const char* src ) {
csync_vio_set_file_id( dst->file_id, src );
if( c_streq( dst->file_id, "" )) {
return;
}
dst->fields |= CSYNC_VIO_FILE_STAT_FIELDS_FILE_ID;
}
static int owncloud_set_property(const char *key, void *data) {
#define READ_STRING_PROPERTY(P) \
if (c_streq(key, #P)) { \
SAFE_FREE(dav_session.P); \
dav_session.P = c_strdup((const char*)data); \
return 0; \
}
#undef READ_STRING_PROPERTY
if (c_streq(key, "file_progress_callback")) {
_file_progress_cb = *(csync_file_progress_callback*)(data);
return 0;
}
return -1;
}
/* Return true if the two path have the same extension. false otherwise. */
static bool _csync_sameextension(const char *p1, const char *p2) {
/* Find pointer to the extensions */
const char *e1 = strrchr(p1, '.');
const char *e2 = strrchr(p2, '.');
/* If the found extension contains a '/', it is because the . was in the folder name
* => no extensions */
if (e1 && strchr(e1, '/')) e1 = NULL;
if (e2 && strchr(e2, '/')) e2 = NULL;
/* If none have extension, it is the same extension */
if (!e1 && !e2)
return true;
/* c_streq takes care of the rest */
return c_streq(e1, e2);
}
static
#endif
int _csync_exclude_add(c_strlist_t **inList, const char *string) {
size_t i = 0;
// We never want duplicates, so check whether the string is already
// in the list first.
if (*inList) {
for (i = 0; i < (*inList)->count; ++i) {
char *pattern = (*inList)->vector[i];
if (c_streq(pattern, string)) {
return 1;
}
}
}
return c_strlist_add_grow(inList, string);
}
static int _csync_check_db_integrity(sqlite3 *db) {
c_strlist_t *result = NULL;
int rc = -1;
result = csync_statedb_query(db, "PRAGMA quick_check;");
if (result != NULL) {
/* There is a result */
if (result->count > 0) {
if (c_streq(result->vector[0], "ok")) {
rc = 0;
}
}
c_strlist_destroy(result);
}
return rc;
}
csync_file_stat_t *csync_statedb_get_stat_by_file_id(CSYNC *ctx,
const char *file_id ) {
csync_file_stat_t *st = NULL;
int rc = 0;
if (!file_id) {
return 0;
}
if (c_streq(file_id, "")) {
return 0;
}
if( !ctx || ctx->db_is_empty ) {
return NULL;
}
if( ctx->statedb.by_fileid_stmt == NULL ) {
const char *query = "SELECT * FROM metadata WHERE fileid=?1";
SQLITE_BUSY_HANDLED(sqlite3_prepare_v2(ctx->statedb.db, query, strlen(query), &ctx->statedb.by_fileid_stmt, NULL));
ctx->statedb.lastReturnValue = rc;
if( rc != SQLITE_OK ) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "WRN: Unable to create stmt for file id query.");
return NULL;
}
}
/* bind the query value */
sqlite3_bind_text(ctx->statedb.by_fileid_stmt, 1, file_id, -1, SQLITE_STATIC);
rc = _csync_file_stat_from_metadata_table(&st, ctx->statedb.by_fileid_stmt);
ctx->statedb.lastReturnValue = rc;
if( !(rc == SQLITE_ROW || rc == SQLITE_DONE) ) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "WRN: Could not get line from metadata: %d!", rc);
}
// clear the resources used by the statement.
sqlite3_reset(ctx->statedb.by_fileid_stmt);
return st;
}
static int _csync_check_db_integrity(sqlite3 *db) {
c_strlist_t *result = NULL;
int rc = -1;
result = csync_statedb_query(db, "PRAGMA quick_check;");
if (result != NULL) {
/* There is a result */
if (result->count > 0) {
if (c_streq(result->vector[0], "ok")) {
rc = 0;
}
}
c_strlist_destroy(result);
}
if( sqlite3_threadsafe() == 0 ) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_WARN, "* WARNING: SQLite module is not threadsafe!");
rc = -1;
}
return rc;
}
static int _csync_statedb_check(const char *statedb) {
int fd = -1;
char buf[BUF_SIZE] = {0};
sqlite3 *db = NULL;
/* check db version */
fd = open(statedb, O_RDONLY);
if (fd >= 0) {
if (read(fd, (void *) buf, (size_t) BUF_SIZE - 1) >= 0) {
buf[BUF_SIZE - 1] = '\0';
close(fd);
if (c_streq(buf, "SQLite format 3")) {
if (sqlite3_open(statedb, &db ) == SQLITE_OK) {
/* everything is fine */
sqlite3_close(db);
return 0;
} else {
CSYNC_LOG(CSYNC_LOG_PRIORITY_WARN, "database corrupted, removing!");
unlink(statedb);
}
sqlite3_close(db);
} else {
CSYNC_LOG(CSYNC_LOG_PRIORITY_WARN, "sqlite version mismatch");
unlink(statedb);
}
}
}
/* create database */
if (sqlite3_open(statedb, &db) == SQLITE_OK) {
sqlite3_close(db);
return 0;
}
sqlite3_close(db);
return -1;
}
/* Configure the proxy depending on the variables */
static int configureProxy( ne_session *session )
{
int port = 8080;
int re = -1;
if( ! session ) return -1;
if( ! dav_session.proxy_type ) return 0; /* Go by NoProxy per default */
if( dav_session.proxy_port > 0 ) {
port = dav_session.proxy_port;
}
if( c_streq(dav_session.proxy_type, "NoProxy" )) {
DEBUG_WEBDAV("No proxy configured.");
re = 0;
} else if( c_streq(dav_session.proxy_type, "DefaultProxy") ||
c_streq(dav_session.proxy_type, "HttpProxy") ||
c_streq(dav_session.proxy_type, "HttpCachingProxy") ||
c_streq(dav_session.proxy_type, "Socks5Proxy")) {
if( dav_session.proxy_host ) {
DEBUG_WEBDAV("%s at %s:%d", dav_session.proxy_type, dav_session.proxy_host, port );
if (c_streq(dav_session.proxy_type, "Socks5Proxy")) {
ne_session_socks_proxy(session, NE_SOCK_SOCKSV5, dav_session.proxy_host, port,
dav_session.proxy_user, dav_session.proxy_pwd);
} else {
ne_session_proxy(session, dav_session.proxy_host, port );
}
re = 2;
} else {
DEBUG_WEBDAV("%s requested but no proxy host defined.", dav_session.proxy_type );
/* we used to try ne_system_session_proxy here, but we should rather err out
to behave exactly like the caller. */
}
} else {
DEBUG_WEBDAV( "Unsupported Proxy: %s", dav_session.proxy_type );
}
return re;
}
/*
* We merge replicas at the file level. The merged replica contains the
* superset of files that are on the local machine and server copies of
* the replica. In the case where the same file is in both the local
* and server copy, the file that was modified most recently is used.
* This means that new files are not deleted, and updated versions of
* existing files are not overwritten.
*
* When a file is updated, the merge algorithm compares the destination
* file with the the source file. If the destination file is newer
* (timestamp is newer), it is not overwritten. If both files, on the
* source and the destination, have been changed, the newer file wins.
*/
static int _csync_merge_algorithm_visitor(void *obj, void *data) {
csync_file_stat_t *cur = NULL;
csync_file_stat_t *other = NULL;
csync_file_stat_t *tmp = NULL;
uint64_t h = 0;
int len = 0;
CSYNC *ctx = NULL;
c_rbtree_t *tree = NULL;
c_rbnode_t *node = NULL;
cur = (csync_file_stat_t *) obj;
ctx = (CSYNC *) data;
/* we need the opposite tree! */
switch (ctx->current) {
case LOCAL_REPLICA:
tree = ctx->remote.tree;
break;
case REMOTE_REPLICA:
tree = ctx->local.tree;
break;
default:
break;
}
node = c_rbtree_find(tree, &cur->phash);
if (!node) {
/* Check the renamed path as well. */
char *renamed_path = csync_rename_adjust_path(ctx, cur->path);
if (!c_streq(renamed_path, cur->path)) {
len = strlen( renamed_path );
h = c_jhash64((uint8_t *) renamed_path, len, 0);
node = c_rbtree_find(tree, &h);
}
SAFE_FREE(renamed_path);
}
if (!node) {
/* Check if it is ignored */
node = _csync_check_ignored(tree, cur->path, cur->pathlen);
/* If it is ignored, other->instruction will be IGNORE so this one will also be ignored */
}
/* file only found on current replica */
if (node == NULL) {
switch(cur->instruction) {
/* file has been modified */
case CSYNC_INSTRUCTION_EVAL:
cur->instruction = CSYNC_INSTRUCTION_NEW;
break;
/* file has been removed on the opposite replica */
case CSYNC_INSTRUCTION_NONE:
cur->instruction = CSYNC_INSTRUCTION_REMOVE;
break;
case CSYNC_INSTRUCTION_EVAL_RENAME:
if(ctx->current == LOCAL_REPLICA ) {
/* use the old name to find the "other" node */
tmp = csync_statedb_get_stat_by_inode(ctx, cur->inode);
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Finding opposite temp through inode %" PRIu64 ": %s",
cur->inode, tmp ? "true":"false");
} else if( ctx->current == REMOTE_REPLICA ) {
tmp = csync_statedb_get_stat_by_file_id(ctx, cur->file_id);
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Finding opposite temp through file ID %s: %s",
cur->file_id, tmp ? "true":"false");
} else {
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Unknown replica...");
}
if( tmp ) {
if( tmp->path ) {
len = strlen( tmp->path );
h = c_jhash64((uint8_t *) tmp->path, len, 0);
/* First, check that the file is NOT in our tree (another file with the same name was added) */
node = c_rbtree_find(ctx->current == REMOTE_REPLICA ? ctx->remote.tree : ctx->local.tree, &h);
if (node) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Origin found in our tree : %s", tmp->path);
} else {
/* Find the temporar file in the other tree. */
node = c_rbtree_find(tree, &h);
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "PHash of temporary opposite (%s): %" PRIu64 " %s",
tmp->path , h, node ? "found": "not found" );
if (node) {
other = (csync_file_stat_t*)node->data;
} else {
/* the renamed file could not be found in the opposite tree. That is because it
* is not longer existing there, maybe because it was renamed or deleted.
* The journal is cleaned up later after propagation.
*/
}
}
}
if(!other) {
cur->instruction = CSYNC_INSTRUCTION_NEW;
} else if (other->instruction == CSYNC_INSTRUCTION_NONE
|| cur->type == CSYNC_FTW_TYPE_DIR) {
other->instruction = CSYNC_INSTRUCTION_RENAME;
other->destpath = c_strdup( cur->path );
if( !c_streq(cur->file_id, "") ) {
csync_vio_set_file_id( other->file_id, cur->file_id );
}
cur->instruction = CSYNC_INSTRUCTION_NONE;
} else if (other->instruction == CSYNC_INSTRUCTION_REMOVE) {
other->instruction = CSYNC_INSTRUCTION_RENAME;
other->destpath = c_strdup( cur->path );
if( !c_streq(cur->file_id, "") ) {
csync_vio_set_file_id( other->file_id, cur->file_id );
}
cur->instruction = CSYNC_INSTRUCTION_NONE;
} else if (other->instruction == CSYNC_INSTRUCTION_NEW) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "OOOO=> NEW detected in other tree!");
cur->instruction = CSYNC_INSTRUCTION_CONFLICT;
} else {
cur->instruction = CSYNC_INSTRUCTION_NONE;
other->instruction = CSYNC_INSTRUCTION_SYNC;
}
csync_file_stat_free(tmp);
}
break;
default:
break;
}
} else {
bool is_equal_files = false;
/*
* file found on the other replica
*/
other = (csync_file_stat_t *) node->data;
switch (cur->instruction) {
case CSYNC_INSTRUCTION_EVAL_RENAME:
/* If the file already exist on the other side, we have a conflict.
Abort the rename and consider it is a new file. */
cur->instruction = CSYNC_INSTRUCTION_NEW;
/* fall trough */
/* file on current replica is changed or new */
case CSYNC_INSTRUCTION_EVAL:
case CSYNC_INSTRUCTION_NEW:
// This operation is usually a no-op and will by default return false
if (csync_file_locked_or_open(ctx->local.uri, cur->path)) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "[Reconciler] IGNORING file %s/%s since it is locked / open", ctx->local.uri, cur->path);
cur->instruction = CSYNC_INSTRUCTION_ERROR;
if (cur->error_status == CSYNC_STATUS_OK) // don't overwrite error
cur->error_status = CYSNC_STATUS_FILE_LOCKED_OR_OPEN;
break;
} else {
//CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "[Reconciler] not ignoring file %s/%s", ctx->local.uri, cur->path);
}
switch (other->instruction) {
/* file on other replica is changed or new */
case CSYNC_INSTRUCTION_NEW:
case CSYNC_INSTRUCTION_EVAL:
if (other->type == CSYNC_VIO_FILE_TYPE_DIRECTORY &&
cur->type == CSYNC_VIO_FILE_TYPE_DIRECTORY) {
is_equal_files = (other->modtime == cur->modtime);
} else {
is_equal_files = ((other->size == cur->size) && (other->modtime == cur->modtime));
}
if (is_equal_files) {
/* The files are considered equal. */
cur->instruction = CSYNC_INSTRUCTION_NONE;
other->instruction = CSYNC_INSTRUCTION_NONE;
if( !cur->etag && other->etag ) cur->etag = c_strdup(other->etag);
cur->should_update_etag = true; /* update DB */
} else if(ctx->current == REMOTE_REPLICA) {
cur->instruction = CSYNC_INSTRUCTION_CONFLICT;
other->instruction = CSYNC_INSTRUCTION_NONE;
} else {
cur->instruction = CSYNC_INSTRUCTION_NONE;
other->instruction = CSYNC_INSTRUCTION_CONFLICT;
}
break;
/* file on the other replica has not been modified */
case CSYNC_INSTRUCTION_NONE:
cur->instruction = CSYNC_INSTRUCTION_SYNC;
break;
case CSYNC_INSTRUCTION_IGNORE:
cur->instruction = CSYNC_INSTRUCTION_IGNORE;
break;
default:
break;
}
default:
break;
}
}
//hide instruction NONE messages when log level is set to debug,
//only show these messages on log level trace
if(cur->instruction ==CSYNC_INSTRUCTION_NONE)
{
if(cur->type == CSYNC_FTW_TYPE_DIR)
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,
"%-20s dir: %s",
csync_instruction_str(cur->instruction),
cur->path);
}
else
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,
"%-20s file: %s",
csync_instruction_str(cur->instruction),
cur->path);
}
}
else
{
if(cur->type == CSYNC_FTW_TYPE_DIR)
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG,
"%-20s dir: %s",
csync_instruction_str(cur->instruction),
cur->path);
}
else
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG,
"%-20s file: %s",
csync_instruction_str(cur->instruction),
cur->path);
}
}
return 0;
}
/* File tree walker */
int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn,
unsigned int depth) {
char *filename = NULL;
char *d_name = NULL;
csync_vio_handle_t *dh = NULL;
csync_vio_file_stat_t *dirent = NULL;
csync_vio_file_stat_t *fs = NULL;
csync_file_stat_t *previous_fs = NULL;
int read_from_db = 0;
int rc = 0;
int res = 0;
bool do_read_from_db = (ctx->current == REMOTE_REPLICA && ctx->remote.read_from_db);
if (uri[0] == '\0') {
errno = ENOENT;
ctx->status_code = CSYNC_STATUS_PARAM_ERROR;
goto error;
}
read_from_db = ctx->remote.read_from_db;
// if the etag of this dir is still the same, its content is restored from the
// database.
if( do_read_from_db ) {
if( ! fill_tree_from_db(ctx, uri) ) {
errno = ENOENT;
ctx->status_code = CSYNC_STATUS_OPENDIR_ERROR;
goto error;
}
goto done;
}
if ((dh = csync_vio_opendir(ctx, uri)) == NULL) {
int asp = 0;
/* permission denied */
ctx->status_code = csync_errno_to_status(errno, CSYNC_STATUS_OPENDIR_ERROR);
if (errno == EACCES) {
return 0;
} else if(errno == ENOENT) {
asp = asprintf( &ctx->error_string, "%s", uri);
if (asp < 0) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "asprintf failed!");
}
} else {
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "opendir failed for %s - errno %d", uri, errno);
}
goto error;
}
while ((dirent = csync_vio_readdir(ctx, dh))) {
const char *path = NULL;
size_t ulen = 0;
int flen;
int flag;
d_name = dirent->name;
if (d_name == NULL) {
ctx->status_code = CSYNC_STATUS_READDIR_ERROR;
goto error;
}
/* skip "." and ".." */
if (d_name[0] == '.' && (d_name[1] == '\0'
|| (d_name[1] == '.' && d_name[2] == '\0'))) {
csync_vio_file_stat_destroy(dirent);
dirent = NULL;
continue;
}
flen = asprintf(&filename, "%s/%s", uri, d_name);
if (flen < 0) {
csync_vio_file_stat_destroy(dirent);
dirent = NULL;
ctx->status_code = CSYNC_STATUS_MEMORY_ERROR;
goto error;
}
/* Create relative path */
switch (ctx->current) {
case LOCAL_REPLICA:
ulen = strlen(ctx->local.uri) + 1;
break;
case REMOTE_REPLICA:
ulen = strlen(ctx->remote.uri) + 1;
break;
default:
break;
}
if (((size_t)flen) < ulen) {
csync_vio_file_stat_destroy(dirent);
dirent = NULL;
ctx->status_code = CSYNC_STATUS_UNSUCCESSFUL;
goto error;
}
path = filename + ulen;
/* skip ".csync_journal.db" and ".csync_journal.db.ctmp" */
if (c_streq(path, ".csync_journal.db")
|| c_streq(path, ".csync_journal.db.ctmp")
|| c_streq(path, ".csync_journal.db.ctmp-journal")
|| c_streq(path, ".csync-progressdatabase")) {
csync_vio_file_stat_destroy(dirent);
dirent = NULL;
SAFE_FREE(filename);
continue;
}
/* Only for the local replica we have to stat(), for the remote one we have all data already */
if (ctx->replica == LOCAL_REPLICA) {
fs = csync_vio_file_stat_new();
res = csync_vio_stat(ctx, filename, fs);
} else {
fs = dirent;
res = 0;
}
if( res == 0) {
switch (fs->type) {
case CSYNC_VIO_FILE_TYPE_SYMBOLIC_LINK:
flag = CSYNC_FTW_FLAG_SLINK;
break;
case CSYNC_VIO_FILE_TYPE_DIRECTORY:
flag = CSYNC_FTW_FLAG_DIR;
break;
case CSYNC_VIO_FILE_TYPE_BLOCK_DEVICE:
case CSYNC_VIO_FILE_TYPE_CHARACTER_DEVICE:
case CSYNC_VIO_FILE_TYPE_SOCKET:
flag = CSYNC_FTW_FLAG_SPEC;
break;
case CSYNC_VIO_FILE_TYPE_FIFO:
flag = CSYNC_FTW_FLAG_SPEC;
break;
default:
flag = CSYNC_FTW_FLAG_FILE;
break;
};
} else {
flag = CSYNC_FTW_FLAG_NSTAT;
}
if( ctx->current == LOCAL_REPLICA ) {
char *etag = NULL;
int len = strlen( path );
uint64_t h = c_jhash64((uint8_t *) path, len, 0);
etag = csync_statedb_get_etag( ctx, h );
if( etag ) {
SAFE_FREE(fs->etag);
fs->etag = etag;
fs->fields |= CSYNC_VIO_FILE_STAT_FIELDS_ETAG;
if( c_streq(etag, "")) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Uniq ID from Database is EMPTY: %s", path);
} else {
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Uniq ID from Database: %s -> %s", path, fs->etag ? fs->etag : "<NULL>" );
}
}
}
previous_fs = ctx->current_fs;
/* Call walker function for each file */
rc = fn(ctx, filename, fs, flag);
/* this function may update ctx->current and ctx->read_from_db */
if (ctx->current_fs && previous_fs && ctx->current_fs->child_modified) {
previous_fs->child_modified = ctx->current_fs->child_modified;
}
/* Only for the local replica we have to destroy stat(), for the remote one it is a pointer to dirent */
if (ctx->replica == LOCAL_REPLICA) {
csync_vio_file_stat_destroy(fs);
}
if (rc < 0) {
if (CSYNC_STATUS_IS_OK(ctx->status_code)) {
ctx->status_code = CSYNC_STATUS_UPDATE_ERROR;
}
ctx->current_fs = previous_fs;
goto error;
}
if (flag == CSYNC_FTW_FLAG_DIR && depth
&& (!ctx->current_fs || ctx->current_fs->instruction != CSYNC_INSTRUCTION_IGNORE)) {
rc = csync_ftw(ctx, filename, fn, depth - 1);
if (rc < 0) {
ctx->current_fs = previous_fs;
goto error;
}
if (ctx->current_fs && !ctx->current_fs->child_modified
&& ctx->current_fs->instruction == CSYNC_INSTRUCTION_EVAL) {
ctx->current_fs->instruction = CSYNC_INSTRUCTION_NONE;
ctx->current_fs->should_update_etag = true;
}
}
if (flag == CSYNC_FTW_FLAG_DIR && ctx->current_fs
&& (ctx->current_fs->instruction == CSYNC_INSTRUCTION_EVAL ||
ctx->current_fs->instruction == CSYNC_INSTRUCTION_NEW ||
ctx->current_fs->instruction == CSYNC_INSTRUCTION_EVAL_RENAME)) {
ctx->current_fs->should_update_etag = true;
}
ctx->current_fs = previous_fs;
ctx->remote.read_from_db = read_from_db;
SAFE_FREE(filename);
csync_vio_file_stat_destroy(dirent);
dirent = NULL;
}
csync_vio_closedir(ctx, dh);
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, " <= Closing walk for %s with read_from_db %d", uri, read_from_db);
done:
csync_vio_file_stat_destroy(dirent);
SAFE_FREE(filename);
return rc;
error:
ctx->remote.read_from_db = read_from_db;
if (dh != NULL) {
csync_vio_closedir(ctx, dh);
}
SAFE_FREE(filename);
return -1;
}
static int _csync_detect_update(CSYNC *ctx, const char *file,
const csync_vio_file_stat_t *fs, const int type) {
uint64_t h = 0;
size_t len = 0;
size_t size = 0;
const char *path = NULL;
csync_file_stat_t *st = NULL;
csync_file_stat_t *tmp = NULL;
CSYNC_EXCLUDE_TYPE excluded;
if ((file == NULL) || (fs == NULL)) {
errno = EINVAL;
ctx->status_code = CSYNC_STATUS_PARAM_ERROR;
return -1;
}
path = file;
switch (ctx->current) {
case LOCAL_REPLICA:
if (strlen(path) <= strlen(ctx->local.uri)) {
ctx->status_code = CSYNC_STATUS_PARAM_ERROR;
return -1;
}
path += strlen(ctx->local.uri) + 1;
break;
case REMOTE_REPLICA:
if (strlen(path) <= strlen(ctx->remote.uri)) {
ctx->status_code = CSYNC_STATUS_PARAM_ERROR;
return -1;
}
path += strlen(ctx->remote.uri) + 1;
break;
default:
path = NULL;
ctx->status_code = CSYNC_STATUS_PARAM_ERROR;
return -1;
}
len = strlen(path);
/* Check if file is excluded */
excluded = csync_excluded(ctx, path,type);
if (excluded != CSYNC_NOT_EXCLUDED) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "%s excluded (%d)", path, excluded);
if (excluded == CSYNC_FILE_EXCLUDE_AND_REMOVE) {
switch (ctx->current) {
case LOCAL_REPLICA:
ctx->local.ignored_cleanup = c_list_append(ctx->local.ignored_cleanup, c_strdup(path));
break;
case REMOTE_REPLICA:
ctx->remote.ignored_cleanup = c_list_append(ctx->remote.ignored_cleanup, c_strdup(path));
break;
default:
break;
}
return 0;
}
if (excluded == CSYNC_FILE_SILENTLY_EXCLUDED) {
return 0;
}
}
h = _hash_of_file(ctx, file );
if( h == 0 ) {
return -1;
}
size = sizeof(csync_file_stat_t) + len + 1;
st = c_malloc(size);
if (st == NULL) {
ctx->status_code = CSYNC_STATUS_MEMORY_ERROR;
return -1;
}
/* Set instruction by default to none */
st->instruction = CSYNC_INSTRUCTION_NONE;
st->etag = NULL;
st->child_modified = 0;
/* check hardlink count */
if (type == CSYNC_FTW_TYPE_FILE ) {
if( fs->nlink > 1) {
st->instruction = CSYNC_INSTRUCTION_IGNORE;
goto out;
}
if (fs->mtime == 0) {
tmp = csync_statedb_get_stat_by_hash(ctx, h);
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "file: %s - mtime is zero!", path);
if (tmp == NULL) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "file: %s - not found in db, IGNORE!", path);
st->instruction = CSYNC_INSTRUCTION_IGNORE;
} else {
SAFE_FREE(st);
st = tmp;
st->instruction = CSYNC_INSTRUCTION_NONE;
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "file: %s - tmp non zero, mtime %lu", path, st->modtime );
tmp = NULL;
}
goto fastout; /* Skip copying of the etag. That's an important difference to upstream
* without etags. */
}
}
/* Ignore non statable files and other strange cases. */
if (type == CSYNC_FTW_TYPE_SKIP) {
st->instruction = CSYNC_INSTRUCTION_NONE;
goto out;
}
if (excluded > CSYNC_NOT_EXCLUDED || type == CSYNC_FTW_TYPE_SLINK) {
if( type == CSYNC_FTW_TYPE_SLINK ) {
st->error_status = CSYNC_STATUS_INDIVIDUAL_IS_SYMLINK; /* Symbolic links are ignored. */
}
st->instruction = CSYNC_INSTRUCTION_IGNORE;
goto out;
}
/* Update detection: Check if a database entry exists.
* If not, the file is either new or has been renamed. To see if it is
* renamed, the db gets queried by the inode of the file as that one
* does not change on rename.
*/
if (csync_get_statedb_exists(ctx)) {
tmp = csync_statedb_get_stat_by_hash(ctx, h);
if(tmp && tmp->phash == h ) { /* there is an entry in the database */
/* we have an update! */
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Database entry found, compare: %" PRId64 " <-> %" PRId64 ", etag: %s <-> %s, inode: %" PRId64 " <-> %" PRId64,
((int64_t) fs->mtime), ((int64_t) tmp->modtime), fs->etag, tmp->etag, (uint64_t) fs->inode, (uint64_t) tmp->inode);
if( !fs->etag) {
st->instruction = CSYNC_INSTRUCTION_EVAL;
goto out;
}
if((ctx->current == REMOTE_REPLICA && !c_streq(fs->etag, tmp->etag ))
|| (ctx->current == LOCAL_REPLICA && (fs->mtime != tmp->modtime
#if 0
|| fs->inode != tmp->inode
#endif
))) {
/* Comparison of the local inode is disabled because people reported problems
* on windows with flacky inode values, see github bug #779
*
* The inode needs to be observed because:
* $> echo a > a.txt ; echo b > b.txt
* both files have the same mtime
* sync them.
* $> rm a.txt && mv b.txt a.txt
* makes b.txt appearing as a.txt yet a sync is not performed because
* both have the same modtime as mv does not change that.
*/
st->instruction = CSYNC_INSTRUCTION_EVAL;
goto out;
}
if (type == CSYNC_FTW_TYPE_DIR && ctx->current == REMOTE_REPLICA
&& c_streq(fs->file_id, tmp->file_id) && !ctx->read_from_db_disabled) {
/* If both etag and file id are equal for a directory, read all contents from
* the database.
* The comparison of file id ensure that we fetch all the file id when upgrading from
* owncloud 5 to owncloud 6.
*/
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Reading from database: %s", path);
ctx->remote.read_from_db = true;
}
if (!c_streq(fs->file_id, tmp->file_id) && ctx->current == REMOTE_REPLICA) {
/* file id has changed. Which means we need to update the DB.
* (upgrade from owncloud 5 to owncloud 6 for instence) */
st->should_update_etag = true;
}
st->instruction = CSYNC_INSTRUCTION_NONE;
} else {
enum csync_vio_file_type_e tmp_vio_type = CSYNC_VIO_FILE_TYPE_UNKNOWN;
/* check if it's a file and has been renamed */
if (ctx->current == LOCAL_REPLICA) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Checking for rename based on inode # %" PRId64 "", (uint64_t) fs->inode);
tmp = csync_statedb_get_stat_by_inode(ctx, fs->inode);
/* translate the file type between the two stat types csync has. */
if( tmp && tmp->type == 0 ) {
tmp_vio_type = CSYNC_VIO_FILE_TYPE_REGULAR;
} else if( tmp && tmp->type == 2 ) {
tmp_vio_type = CSYNC_VIO_FILE_TYPE_DIRECTORY;
} else {
tmp_vio_type = CSYNC_VIO_FILE_TYPE_UNKNOWN;
}
if (tmp && tmp->inode == fs->inode && tmp_vio_type == fs->type
&& (tmp->modtime == fs->mtime || fs->type == CSYNC_VIO_FILE_TYPE_DIRECTORY)
#ifdef NO_RENAME_EXTENSION
&& _csync_sameextension(tmp->path, path)
#endif
) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "pot rename detected based on inode # %" PRId64 "", (uint64_t) fs->inode);
/* inode found so the file has been renamed */
st->instruction = CSYNC_INSTRUCTION_EVAL_RENAME;
if (fs->type == CSYNC_VIO_FILE_TYPE_DIRECTORY) {
csync_rename_record(ctx, tmp->path, path);
}
goto out;
} else {
/* file not found in statedb */
st->instruction = CSYNC_INSTRUCTION_NEW;
goto out;
}
} else {
/* Remote Replica Rename check */
tmp = csync_statedb_get_stat_by_file_id(ctx, fs->file_id);
if(tmp ) { /* tmp existing at all */
if ((tmp->type == CSYNC_FTW_TYPE_DIR && fs->type != CSYNC_VIO_FILE_TYPE_DIRECTORY) ||
(tmp->type == CSYNC_FTW_TYPE_FILE && fs->type != CSYNC_VIO_FILE_TYPE_REGULAR)) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "WARN: file types different is not!");
st->instruction = CSYNC_INSTRUCTION_NEW;
goto out;
}
st->instruction = CSYNC_INSTRUCTION_EVAL_RENAME;
if (fs->type == CSYNC_VIO_FILE_TYPE_DIRECTORY) {
csync_rename_record(ctx, tmp->path, path);
} else {
if( !c_streq(tmp->etag, fs->etag) ) {
/* CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "ETags are different!"); */
/* File with different etag, don't do a rename, but download the file again */
st->instruction = CSYNC_INSTRUCTION_NEW;
}
}
goto out;
} else {
/* file not found in statedb */
st->instruction = CSYNC_INSTRUCTION_NEW;
goto out;
}
}
}
} else {
st->instruction = CSYNC_INSTRUCTION_NEW;
}
out:
/* Set the ignored error string. */
if (st->instruction == CSYNC_INSTRUCTION_IGNORE) {
if (excluded == CSYNC_FILE_EXCLUDE_LIST) {
st->error_status = CSYNC_STATUS_INDIVIDUAL_IGNORE_LIST; /* File listed on ignore list. */
} else if (excluded == CSYNC_FILE_EXCLUDE_INVALID_CHAR) {
st->error_status = CSYNC_STATUS_INDIVIDUAL_IS_INVALID_CHARS; /* File contains invalid characters. */
}
}
if (st->instruction != CSYNC_INSTRUCTION_NONE && st->instruction != CSYNC_INSTRUCTION_IGNORE
&& type != CSYNC_FTW_TYPE_DIR) {
st->child_modified = 1;
}
ctx->current_fs = st;
csync_file_stat_free(tmp);
st->inode = fs->inode;
st->mode = fs->mode;
st->size = fs->size;
st->modtime = fs->mtime;
st->nlink = fs->nlink;
st->type = type;
st->etag = NULL;
if( fs->etag ) {
SAFE_FREE(st->etag);
st->etag = c_strdup(fs->etag);
}
csync_vio_set_file_id(st->file_id, fs->file_id);
if (fs->fields & CSYNC_VIO_FILE_STAT_FIELDS_DIRECTDOWNLOADURL) {
SAFE_FREE(st->directDownloadUrl);
st->directDownloadUrl = c_strdup(fs->directDownloadUrl);
}
if (fs->fields & CSYNC_VIO_FILE_STAT_FIELDS_DIRECTDOWNLOADCOOKIES) {
SAFE_FREE(st->directDownloadCookies);
st->directDownloadCookies = c_strdup(fs->directDownloadCookies);
}
fastout: /* target if the file information is read from database into st */
st->phash = h;
st->pathlen = len;
memcpy(st->path, (len ? path : ""), len + 1);
switch (ctx->current) {
case LOCAL_REPLICA:
if (c_rbtree_insert(ctx->local.tree, (void *) st) < 0) {
SAFE_FREE(st);
ctx->status_code = CSYNC_STATUS_TREE_ERROR;
return -1;
}
break;
case REMOTE_REPLICA:
if (c_rbtree_insert(ctx->remote.tree, (void *) st) < 0) {
SAFE_FREE(st);
ctx->status_code = CSYNC_STATUS_TREE_ERROR;
return -1;
}
break;
default:
break;
}
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "file: %s, instruction: %s <<=", st->path,
csync_instruction_str(st->instruction));
return 0;
}
void fetch_resource_list_recursive(const char *uri, const char *curi)
{
int ret = 0;
ne_propfind_handler *hdl = NULL;
ne_request *request = NULL;
const char *content_type = NULL;
const ne_status *req_status = NULL;
int depth = NE_DEPTH_INFINITE;
DEBUG_WEBDAV("fetch_resource_list_recursive Starting recursive propfind %s %s", uri, curi);
/* do a propfind request and parse the results in the results function, set as callback */
hdl = ne_propfind_create(dav_session.ctx, curi, depth);
if(hdl) {
ret = ne_propfind_named(hdl, ls_props, propfind_results_recursive, (void*)curi);
request = ne_propfind_get_request( hdl );
req_status = ne_get_status( request );
}
if( ret == NE_OK ) {
/* Check the request status. */
if( req_status && req_status->klass != 2 ) {
set_errno_from_http_errcode(req_status->code);
DEBUG_WEBDAV("ERROR: Request failed: status %d (%s)", req_status->code,
req_status->reason_phrase);
ret = NE_CONNECT;
set_error_message(req_status->reason_phrase);
}
DEBUG_WEBDAV("Recursive propfind result code %d.", req_status ? req_status->code : 0);
} else {
if( ret == NE_ERROR && req_status->code == 404) {
errno = ENOENT;
} else {
set_errno_from_neon_errcode(ret);
}
}
if( ret == NE_OK ) {
/* Check the content type. If the server has a problem, ie. database is gone or such,
* the content type is not xml but a html error message. Stop on processing if it's
* not XML.
* FIXME: Generate user error message from the reply content.
*/
content_type = ne_get_response_header( request, "Content-Type" );
if( !(content_type && c_streq(content_type, "application/xml; charset=utf-8") ) ) {
DEBUG_WEBDAV("ERROR: Content type of propfind request not XML: %s.",
content_type ? content_type: "<empty>");
errno = ERRNO_WRONG_CONTENT;
set_error_message("Server error: PROPFIND reply is not XML formatted!");
ret = NE_CONNECT;
}
}
if( ret != NE_OK ) {
const char *err = NULL;
err = ne_get_error( dav_session.ctx );
DEBUG_WEBDAV("WRN: propfind named failed with %d, request error: %s", ret, err ? err : "<nil>");
}
if( hdl )
ne_propfind_destroy(hdl);
if( ret != NE_OK ) {
return;
}
return;
}
static csync_vio_method_handle_t *owncloud_open(const char *durl,
int flags,
mode_t mode) {
char *dir = NULL;
char getUrl[PATH_MAX];
int put = 0;
int rc = NE_OK;
#ifdef _WIN32
int gtp = 0;
char tmpname[13];
mbchar_t winTmp[PATH_MAX];
mbchar_t *winUrlMB = NULL;
char *winTmpUtf8 = NULL;
csync_stat_t sb;
#endif
struct transfer_context *writeCtx = NULL;
csync_stat_t statBuf;
memset( getUrl, '\0', PATH_MAX );
(void) mode; /* unused on webdav server */
DEBUG_WEBDAV( "=> open called for %s", durl );
if( rc == NE_OK )
dav_connect( durl );
if (flags & O_WRONLY) {
put = 1;
}
if (flags & O_RDWR) {
put = 1;
}
if (flags & O_CREAT) {
put = 1;
}
if( rc == NE_OK && put ) {
/* check if the dir name exists. Otherwise return ENOENT */
dir = c_dirname( durl );
if (dir == NULL) {
errno = ENOMEM;
return NULL;
}
DEBUG_WEBDAV("Stating directory %s", dir );
if( c_streq( dir, _lastDir )) {
DEBUG_WEBDAV("Dir %s is there, we know it already.", dir);
} else {
if( owncloud_stat( dir, (csync_vio_method_handle_t*)(&statBuf) ) == 0 ) {
DEBUG_WEBDAV("Directory of file to open exists.");
SAFE_FREE( _lastDir );
_lastDir = c_strdup(dir);
} else {
DEBUG_WEBDAV("Directory %s of file to open does NOT exist.", dir );
/* the directory does not exist. That is an ENOENT */
errno = ENOENT;
SAFE_FREE( dir );
return NULL;
}
}
}
writeCtx = c_malloc( sizeof(struct transfer_context) );
writeCtx->url = _cleanPath( durl );
if( ! writeCtx->url ) {
DEBUG_WEBDAV("Failed to clean path for %s", durl );
errno = EACCES;
rc = NE_ERROR;
}
if( rc == NE_OK && put) {
DEBUG_WEBDAV("PUT request on %s!", writeCtx->url);
writeCtx->req = ne_request_create(dav_session.ctx, "PUT", writeCtx->url);
writeCtx->method = "PUT";
}
if( rc == NE_OK && ! put ) {
writeCtx->req = 0;
writeCtx->method = "GET";
/* the download via the get function requires a full uri */
snprintf( getUrl, PATH_MAX, "%s://%s%s", ne_get_scheme( dav_session.ctx),
ne_get_server_hostport( dav_session.ctx ), writeCtx->url );
DEBUG_WEBDAV("GET request on %s", getUrl );
writeCtx->req = ne_request_create( dav_session.ctx, "GET", getUrl );
/* Call the progress callback */
if (_file_progress_cb) {
ne_set_notifier(dav_session.ctx, ne_notify_status_cb, writeCtx);
_file_progress_cb( writeCtx->url, CSYNC_NOTIFY_START_DOWNLOAD, 0 , 0, _userdata);
}
}
if( rc != NE_OK ) {
SAFE_FREE( writeCtx );
writeCtx = NULL;
}
SAFE_FREE( dir );
return (csync_vio_method_handle_t *) writeCtx;
}
static int fetch_resource_list( const char *uri,
int depth,
struct listdir_context *fetchCtx )
{
int ret = 0;
ne_propfind_handler *hdl = NULL;
ne_request *request = NULL;
const char *content_type = NULL;
char *curi = NULL;
const ne_status *req_status = NULL;
curi = _cleanPath( uri );
if (!fetchCtx) {
errno = ENOMEM;
SAFE_FREE(curi);
return -1;
}
fetchCtx->list = NULL;
fetchCtx->target = curi;
fetchCtx->currResource = NULL;
/* do a propfind request and parse the results in the results function, set as callback */
hdl = ne_propfind_create(dav_session.ctx, curi, depth);
if(hdl) {
ret = ne_propfind_named(hdl, ls_props, results, fetchCtx);
request = ne_propfind_get_request( hdl );
req_status = ne_get_status( request );
}
if( ret == NE_OK ) {
fetchCtx->currResource = fetchCtx->list;
/* Check the request status. */
if( req_status && req_status->klass != 2 ) {
set_errno_from_http_errcode(req_status->code);
DEBUG_WEBDAV("ERROR: Request failed: status %d (%s)", req_status->code,
req_status->reason_phrase);
ret = NE_CONNECT;
set_error_message(req_status->reason_phrase);
}
DEBUG_WEBDAV("Simple propfind result code %d.", req_status ? req_status->code : -1);
} else {
if( ret == NE_ERROR && req_status->code == 404) {
errno = ENOENT;
} else {
set_errno_from_neon_errcode(ret);
}
}
if( ret == NE_OK ) {
/* Check the content type. If the server has a problem, ie. database is gone or such,
* the content type is not xml but a html error message. Stop on processing if it's
* not XML.
* FIXME: Generate user error message from the reply content.
*/
content_type = ne_get_response_header( request, "Content-Type" );
if( !(content_type && c_streq(content_type, "application/xml; charset=utf-8") ) ) {
DEBUG_WEBDAV("ERROR: Content type of propfind request not XML: %s.",
content_type ? content_type: "<empty>");
errno = ERRNO_WRONG_CONTENT;
set_error_message("Server error: PROPFIND reply is not XML formatted!");
ret = NE_CONNECT;
}
}
#ifndef NDEBUG
if( ret != NE_OK ) {
const char *err = ne_get_error(dav_session.ctx);
DEBUG_WEBDAV("WRN: propfind named failed with %d, request error: %s", ret, err ? err : "<nil>");
}
#endif /* NDEBUG */
if (hdl != NULL) {
ne_propfind_destroy(hdl);
}
#ifndef NDEBUG
if (ret == NE_REDIRECT) {
const ne_uri *redir_ne_uri = ne_redirect_location(dav_session.ctx);
if (redir_ne_uri) {
char *redir_uri = ne_uri_unparse(redir_ne_uri);
DEBUG_WEBDAV("Permanently moved to %s", redir_uri);
}
}
#endif /* NDEBUG */
if( ret != NE_OK ) {
free_fetchCtx(fetchCtx);
return -1;
}
return 0;
}
CSYNC_EXCLUDE_TYPE csync_excluded(CSYNC *ctx, const char *path, int filetype) {
size_t i = 0;
const char *p = NULL;
char *bname = NULL;
char *dname = NULL;
char *prev_dname = NULL;
char *conflict = NULL;
int rc = -1;
CSYNC_EXCLUDE_TYPE match = CSYNC_NOT_EXCLUDED;
CSYNC_EXCLUDE_TYPE type = CSYNC_NOT_EXCLUDED;
for (p = path; *p; p++) {
switch (*p) {
case '\\':
case ':':
case '?':
case '*':
case '"':
case '>':
case '<':
case '|':
return CSYNC_FILE_EXCLUDE_INVALID_CHAR;
default:
break;
}
}
/* split up the path */
dname = c_dirname(path);
bname = c_basename(path);
if (bname == NULL || dname == NULL) {
match = CSYNC_NOT_EXCLUDED;
SAFE_FREE(bname);
SAFE_FREE(dname);
goto out;
}
rc = csync_fnmatch(".csync_journal.db*", bname, 0);
if (rc == 0) {
match = CSYNC_FILE_SILENTLY_EXCLUDED;
SAFE_FREE(bname);
SAFE_FREE(dname);
goto out;
}
rc = csync_fnmatch(".owncloudsync.log*", bname, 0);
if (rc == 0) {
match = CSYNC_FILE_SILENTLY_EXCLUDED;
SAFE_FREE(bname);
SAFE_FREE(dname);
goto out;
}
/* Always ignore conflict files, not only via the exclude list */
rc = csync_fnmatch("*_conflict-*", bname, 0);
if (rc == 0) {
match = CSYNC_FILE_SILENTLY_EXCLUDED;
SAFE_FREE(bname);
SAFE_FREE(dname);
goto out;
}
if (getenv("CSYNC_CONFLICT_FILE_USERNAME")) {
asprintf(&conflict, "*_conflict_%s-*", getenv("CSYNC_CONFLICT_FILE_USERNAME"));
rc = csync_fnmatch(conflict, path, 0);
if (rc == 0) {
match = CSYNC_FILE_SILENTLY_EXCLUDED;
SAFE_FREE(conflict);
SAFE_FREE(bname);
SAFE_FREE(dname);
goto out;
}
SAFE_FREE(conflict);
}
SAFE_FREE(bname);
SAFE_FREE(dname);
if (ctx->excludes == NULL) {
goto out;
}
/* Loop over all exclude patterns and evaluate the given path */
for (i = 0; match == CSYNC_NOT_EXCLUDED && i < ctx->excludes->count; i++) {
bool match_dirs_only = false;
char *pattern_stored = c_strdup(ctx->excludes->vector[i]);
char* pattern = pattern_stored;
type = CSYNC_FILE_EXCLUDE_LIST;
if (strlen(pattern) < 1) {
continue;
}
/* Ecludes starting with ']' means it can be cleanup */
if (pattern[0] == ']') {
++pattern;
if (filetype == CSYNC_FTW_TYPE_FILE) {
type = CSYNC_FILE_EXCLUDE_AND_REMOVE;
}
}
/* Check if the pattern applies to pathes only. */
if (pattern[strlen(pattern)-1] == '/') {
match_dirs_only = true;
pattern[strlen(pattern)-1] = '\0'; /* Cut off the slash */
}
/* check if the pattern contains a / and if, compare to the whole path */
if (strchr(pattern, '/')) {
rc = csync_fnmatch(pattern, path, FNM_PATHNAME);
if( rc == 0 ) {
match = type;
}
/* if the pattern requires a dir, but path is not, its still not excluded. */
if (match_dirs_only && filetype != CSYNC_FTW_TYPE_DIR) {
match = CSYNC_NOT_EXCLUDED;
}
}
/* if still not excluded, check each component of the path */
if (match == CSYNC_NOT_EXCLUDED) {
int trailing_component = 1;
dname = c_dirname(path);
bname = c_basename(path);
if (bname == NULL || dname == NULL) {
match = CSYNC_NOT_EXCLUDED;
goto out;
}
/* Check each component of the path */
do {
/* Do not check the bname if its a file and the pattern matches dirs only. */
if ( !(trailing_component == 1 /* it is the trailing component */
&& match_dirs_only /* but only directories are matched by the pattern */
&& filetype == CSYNC_FTW_TYPE_FILE) ) {
/* Check the name component against the pattern */
rc = csync_fnmatch(pattern, bname, 0);
if (rc == 0) {
match = type;
}
}
if (!(c_streq(dname, ".") || c_streq(dname, "/"))) {
rc = csync_fnmatch(pattern, dname, 0);
if (rc == 0) {
match = type;
}
}
trailing_component = 0;
prev_dname = dname;
SAFE_FREE(bname);
bname = c_basename(prev_dname);
dname = c_dirname(prev_dname);
SAFE_FREE(prev_dname);
} while( match == CSYNC_NOT_EXCLUDED && !c_streq(dname, ".")
&& !c_streq(dname, "/") );
}
SAFE_FREE(pattern_stored);
SAFE_FREE(bname);
SAFE_FREE(dname);
}
out:
return match;
}
int c_rmdirs(const char *path) {
_TDIR *d;
struct _tdirent *dp;
csync_stat_t sb;
char *fname = NULL;
mbchar_t *wfname = NULL;
mbchar_t *wpath = c_utf8_to_locale(path);
char *rd_name = NULL;
if ((d = _topendir(wpath)) != NULL) {
while( _tstat(wpath, &sb) == 0) {
/* if we can remove the directory we're done */
if (_trmdir(wpath) == 0) {
break;
}
switch (errno) {
case ENOTEMPTY:
case EEXIST:
case EBADF:
break; /* continue */
default:
_tclosedir(d);
c_free_locale_string(wpath);
return 0;
}
while ((dp = _treaddir(d)) != NULL) {
size_t len;
rd_name = c_utf8_from_locale(dp->d_name);
/* skip '.' and '..' */
if( c_streq( rd_name, "." ) || c_streq( rd_name, ".." ) ) {
c_free_locale_string(rd_name);
continue;
}
len = strlen(path) + strlen(rd_name) + 2;
fname = c_malloc(len);
if (fname == NULL) {
_tclosedir(d);
c_free_locale_string(rd_name);
c_free_locale_string(wpath);
return -1;
}
snprintf(fname, len, "%s/%s", path, rd_name);
wfname = c_utf8_to_locale(fname);
/* stat the file */
if (_tstat(wfname, &sb) != -1) {
#ifdef __unix__
if (S_ISDIR(sb.st_mode) && !S_ISLNK(sb.st_mode)) {
#else
if (S_ISDIR(sb.st_mode)) {
#endif
if (_trmdir(wfname) < 0) { /* can't be deleted */
if (errno == EACCES) {
_tclosedir(d);
SAFE_FREE(fname);
c_free_locale_string(wfname);
c_free_locale_string(rd_name);
c_free_locale_string(wpath);
return -1;
}
c_rmdirs(fname);
}
} else {
_tunlink(wfname);
}
} /* lstat */
SAFE_FREE(fname);
c_free_locale_string(wfname);
c_free_locale_string(rd_name);
} /* readdir */
_trewinddir(d);
}
} else {
c_free_locale_string(wpath);
return -1;
}
c_free_locale_string(wpath);
_tclosedir(d);
return 0;
}
/* check if path is a file */
int c_isfile(const char *path) {
csync_stat_t sb;
if (lstat (path, &sb) < 0) {
return 0;
}
#ifdef __unix__
if (S_ISREG(sb.st_mode) || S_ISLNK(sb.st_mode)) {
#else
if (S_ISREG(sb.st_mode)) {
#endif
return 1;
}
return 0;
}
/* copy file from src to dst, overwrites dst */
int c_copy(const char* src, const char *dst, mode_t mode) {
int srcfd = -1;
int dstfd = -1;
int rc = -1;
ssize_t bread, bwritten;
csync_stat_t sb;
char buf[4096];
#ifdef _WIN32
if(src && dst) {
if (CopyFile(src, dst, FALSE)) {
return 0;
}
errno = GetLastError();
return -1;
}
#endif
if (c_streq(src, dst)) {
return -1;
}
if (lstat(src, &sb) < 0) {
return -1;
}
if (S_ISDIR(sb.st_mode)) {
errno = EISDIR;
return -1;
}
if (mode == 0) {
mode = sb.st_mode;
}
if (lstat(dst, &sb) == 0) {
if (S_ISDIR(sb.st_mode)) {
errno = EISDIR;
return -1;
}
}
if ((srcfd = open(src, O_RDONLY, 0)) < 0) {
rc = -1;
goto out;
}
if ((dstfd = open(dst, O_CREAT|O_WRONLY|O_TRUNC, mode)) < 0) {
rc = -1;
goto out;
}
for (;;) {
bread = read(srcfd, buf, sizeof(buf));
if (bread == 0) {
/* done */
break;
} else if (bread < 0) {
errno = ENODATA;
rc = -1;
goto out;
}
bwritten = write(dstfd, buf, bread);
if (bwritten < 0) {
errno = ENODATA;
rc = -1;
goto out;
}
if (bread != bwritten) {
errno = EFAULT;
rc = -1;
goto out;
}
}
#ifdef __unix__
fsync(dstfd);
#endif
rc = 0;
out:
if (srcfd > 0) {
close(srcfd);
}
if (dstfd > 0) {
close(dstfd);
}
if (rc < 0) {
unlink(dst);
}
return rc;
}
/*
* This function uses the vio_opendir, vio_readdir and vio_closedir functions
* to traverse a file tree that was created before by the create_dir function.
*
* It appends a listing to the result member of the incoming struct in *state
* that can be compared later to what was expected in the calling functions.
*
* The int parameter cnt contains the number of seen files (not dirs) in the
* whole tree.
*
*/
static void traverse_dir(void **state, const char *dir, int *cnt)
{
csync_vio_handle_t *dh;
csync_vio_file_stat_t *dirent;
statevar *sv = (statevar*) *state;
CSYNC *csync = sv->csync;
char *subdir;
char *subdir_out;
int rc;
int is_dir;
/* Format: Smuggle in the C: for unix platforms as its urgently needed
* on Windows and the test can be nicely cross platform this way. */
#ifdef _WIN32
const char *format_str = "%s %s";
#else
const char *format_str = "%s C:%s";
#endif
dh = csync_vio_opendir(csync, dir);
assert_non_null(dh);
while( (dirent = csync_vio_readdir(csync, dh)) ) {
assert_non_null(dirent);
if (dirent->original_name) {
sv->ignored_dir = c_strdup(dirent->original_name);
continue;
}
assert_non_null(dirent->name);
assert_int_equal( dirent->fields & CSYNC_VIO_FILE_STAT_FIELDS_TYPE, CSYNC_VIO_FILE_STAT_FIELDS_TYPE );
if( c_streq( dirent->name, "..") || c_streq( dirent->name, "." )) {
continue;
}
is_dir = (dirent->type == CSYNC_VIO_FILE_TYPE_DIRECTORY) ? 1:0;
assert_int_not_equal( asprintf( &subdir, "%s/%s", dir, dirent->name ), -1 );
assert_int_not_equal( asprintf( &subdir_out, format_str,
is_dir ? "<DIR>":" ",
subdir), -1 );
if( is_dir ) {
if( !sv->result ) {
sv->result = c_strdup( subdir_out);
} else {
int newlen = 1+strlen(sv->result)+strlen(subdir_out);
char *tmp = sv->result;
sv->result = c_malloc(newlen);
strcpy( sv->result, tmp);
SAFE_FREE(tmp);
strcat( sv->result, subdir_out );
}
} else {
*cnt = *cnt +1;
}
output(subdir_out);
if( is_dir ) {
traverse_dir( state, subdir, cnt);
}
SAFE_FREE(subdir);
SAFE_FREE(subdir_out);
}
csync_vio_file_stat_destroy(dirent);
rc = csync_vio_closedir(csync, dh);
assert_int_equal(rc, 0);
}
/* File tree walker */
int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn,
unsigned int depth) {
char *filename = NULL;
char *d_name = NULL;
csync_vio_handle_t *dh = NULL;
csync_vio_file_stat_t *dirent = NULL;
csync_file_stat_t *previous_fs = NULL;
int read_from_db = 0;
int rc = 0;
int res = 0;
bool do_read_from_db = (ctx->current == REMOTE_REPLICA && ctx->remote.read_from_db);
if (uri[0] == '\0') {
errno = ENOENT;
ctx->status_code = CSYNC_STATUS_PARAM_ERROR;
goto error;
}
read_from_db = ctx->remote.read_from_db;
// if the etag of this dir is still the same, its content is restored from the
// database.
if( do_read_from_db ) {
if( ! fill_tree_from_db(ctx, uri) ) {
errno = ENOENT;
ctx->status_code = CSYNC_STATUS_OPENDIR_ERROR;
goto error;
}
goto done;
}
const char *uri_for_vio = uri;
if (ctx->current == REMOTE_REPLICA) {
uri_for_vio += strlen(ctx->remote.uri);
if (strlen(uri_for_vio) > 0 && uri_for_vio[0] == '/') {
uri_for_vio++; // cut leading slash
}
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "URI without fuzz for %s is \"%s\"", uri, uri_for_vio);
}
if ((dh = csync_vio_opendir(ctx, uri_for_vio)) == NULL) {
if (ctx->abort) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Aborted!");
ctx->status_code = CSYNC_STATUS_ABORTED;
goto error;
}
int asp = 0;
/* permission denied */
ctx->status_code = csync_errno_to_status(errno, CSYNC_STATUS_OPENDIR_ERROR);
if (errno == EACCES) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_WARN, "Permission denied.");
if (mark_current_item_ignored(ctx, previous_fs, CSYNC_STATUS_PERMISSION_DENIED)) {
goto done;
}
} else if(errno == ENOENT) {
asp = asprintf( &ctx->error_string, "%s", uri);
if (asp < 0) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "asprintf failed!");
}
}
// 403 Forbidden can be sent by the server if the file firewall is active.
// A file or directory should be ignored and sync must continue. See #3490
else if(errno == ERRNO_FORBIDDEN) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_WARN, "Directory access Forbidden (File Firewall?)");
if( mark_current_item_ignored(ctx, previous_fs, CSYNC_STATUS_FORBIDDEN) ) {
goto done;
}
/* if current_fs is not defined here, better throw an error */
}
// The server usually replies with the custom "503 Storage not available"
// if some path is temporarily unavailable. But in some cases a standard 503
// is returned too. Thus we can't distinguish the two and will treat any
// 503 as request to ignore the folder. See #3113 #2884.
else if(errno == ERRNO_STORAGE_UNAVAILABLE || errno == ERRNO_SERVICE_UNAVAILABLE) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_WARN, "Storage was not available!");
if( mark_current_item_ignored(ctx, previous_fs, CSYNC_STATUS_STORAGE_UNAVAILABLE ) ) {
goto done;
}
/* if current_fs is not defined here, better throw an error */
} else {
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "opendir failed for %s - errno %d", uri, errno);
}
goto error;
}
while ((dirent = csync_vio_readdir(ctx, dh))) {
const char *path = NULL;
size_t ulen = 0;
int flen;
int flag;
/* Conversion error */
if (dirent->name == NULL && dirent->original_name) {
ctx->status_code = CSYNC_STATUS_INVALID_CHARACTERS;
ctx->error_string = dirent->original_name; // take ownership
dirent->original_name = NULL;
goto error;
}
d_name = dirent->name;
if (d_name == NULL) {
ctx->status_code = CSYNC_STATUS_READDIR_ERROR;
goto error;
}
/* skip "." and ".." */
if ( (d_name[0] == '.' && d_name[1] == '\0')
|| (d_name[0] == '.' && d_name[1] == '.' && d_name[2] == '\0')) {
csync_vio_file_stat_destroy(dirent);
dirent = NULL;
continue;
}
flen = asprintf(&filename, "%s/%s", uri, d_name);
if (flen < 0) {
csync_vio_file_stat_destroy(dirent);
dirent = NULL;
ctx->status_code = CSYNC_STATUS_MEMORY_ERROR;
goto error;
}
/* Create relative path */
switch (ctx->current) {
case LOCAL_REPLICA:
ulen = strlen(ctx->local.uri) + 1;
break;
case REMOTE_REPLICA:
ulen = strlen(ctx->remote.uri) + 1;
break;
default:
break;
}
if (((size_t)flen) < ulen) {
csync_vio_file_stat_destroy(dirent);
dirent = NULL;
ctx->status_code = CSYNC_STATUS_UNSUCCESSFUL;
goto error;
}
path = filename + ulen;
/* skip ".csync_journal.db" and ".csync_journal.db.ctmp" */
/* Isn't this done via csync_exclude already? */
if (c_streq(path, ".csync_journal.db")
|| c_streq(path, ".csync_journal.db.ctmp")
|| c_streq(path, ".csync_journal.db.ctmp-journal")
|| c_streq(path, ".csync-progressdatabase")
|| c_streq(path, ".csync_journal.db-shm")
|| c_streq(path, ".csync_journal.db-wal")
|| c_streq(path, ".csync_journal.db-journal")) {
csync_vio_file_stat_destroy(dirent);
dirent = NULL;
SAFE_FREE(filename);
continue;
}
/* Only for the local replica we have to stat(), for the remote one we have all data already */
if (ctx->replica == LOCAL_REPLICA) {
res = csync_vio_stat(ctx, filename, dirent);
} else {
res = 0;
}
/* if the filename starts with a . we consider it a hidden file
* For windows, the hidden state is also discovered within the vio
* local stat function.
*/
if( d_name[0] == '.' ) {
if (strcmp(".sys.admin#recall#", d_name) != 0) { /* recall file shall not be ignored (#4420) */
dirent->flags |= CSYNC_VIO_FILE_FLAGS_HIDDEN;
}
}
if( res == 0) {
switch (dirent->type) {
case CSYNC_VIO_FILE_TYPE_SYMBOLIC_LINK:
flag = CSYNC_FTW_FLAG_SLINK;
break;
case CSYNC_VIO_FILE_TYPE_DIRECTORY:
flag = CSYNC_FTW_FLAG_DIR;
break;
case CSYNC_VIO_FILE_TYPE_BLOCK_DEVICE:
case CSYNC_VIO_FILE_TYPE_CHARACTER_DEVICE:
case CSYNC_VIO_FILE_TYPE_SOCKET:
flag = CSYNC_FTW_FLAG_SPEC;
break;
case CSYNC_VIO_FILE_TYPE_FIFO:
flag = CSYNC_FTW_FLAG_SPEC;
break;
default:
flag = CSYNC_FTW_FLAG_FILE;
break;
};
} else {
flag = CSYNC_FTW_FLAG_NSTAT;
}
previous_fs = ctx->current_fs;
/* Call walker function for each file */
rc = fn(ctx, filename, dirent, flag);
/* this function may update ctx->current and ctx->read_from_db */
if (rc < 0) {
if (CSYNC_STATUS_IS_OK(ctx->status_code)) {
ctx->status_code = CSYNC_STATUS_UPDATE_ERROR;
}
ctx->current_fs = previous_fs;
goto error;
}
if (flag == CSYNC_FTW_FLAG_DIR && depth && rc == 0
&& (!ctx->current_fs || ctx->current_fs->instruction != CSYNC_INSTRUCTION_IGNORE)) {
rc = csync_ftw(ctx, filename, fn, depth - 1);
if (rc < 0) {
ctx->current_fs = previous_fs;
goto error;
}
if (ctx->current_fs && !ctx->current_fs->child_modified
&& ctx->current_fs->instruction == CSYNC_INSTRUCTION_EVAL) {
ctx->current_fs->instruction = CSYNC_INSTRUCTION_NONE;
if (ctx->current == REMOTE_REPLICA) {
ctx->current_fs->should_update_metadata = true;
}
}
if (ctx->current_fs && previous_fs && ctx->current_fs->has_ignored_files) {
/* If a directory has ignored files, put the flag on the parent directory as well */
previous_fs->has_ignored_files = ctx->current_fs->has_ignored_files;
}
}
if (ctx->current_fs && previous_fs && ctx->current_fs->child_modified) {
/* If a directory has modified files, put the flag on the parent directory as well */
previous_fs->child_modified = ctx->current_fs->child_modified;
}
if (flag == CSYNC_FTW_FLAG_DIR && ctx->current_fs
&& (ctx->current_fs->instruction == CSYNC_INSTRUCTION_EVAL ||
ctx->current_fs->instruction == CSYNC_INSTRUCTION_NEW)) {
ctx->current_fs->should_update_metadata = true;
}
ctx->current_fs = previous_fs;
ctx->remote.read_from_db = read_from_db;
SAFE_FREE(filename);
csync_vio_file_stat_destroy(dirent);
dirent = NULL;
}
csync_vio_closedir(ctx, dh);
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, " <= Closing walk for %s with read_from_db %d", uri, read_from_db);
done:
csync_vio_file_stat_destroy(dirent);
SAFE_FREE(filename);
return rc;
error:
ctx->remote.read_from_db = read_from_db;
if (dh != NULL) {
csync_vio_closedir(ctx, dh);
}
SAFE_FREE(filename);
return -1;
}
static int _csync_detect_update(CSYNC *ctx, const char *file,
const csync_vio_file_stat_t *fs, const int type) {
uint64_t h = 0;
size_t len = 0;
size_t size = 0;
const char *path = NULL;
csync_file_stat_t *st = NULL;
csync_file_stat_t *tmp = NULL;
CSYNC_EXCLUDE_TYPE excluded;
if ((file == NULL) || (fs == NULL)) {
errno = EINVAL;
ctx->status_code = CSYNC_STATUS_PARAM_ERROR;
return -1;
}
path = file;
switch (ctx->current) {
case LOCAL_REPLICA:
if (strlen(path) <= strlen(ctx->local.uri)) {
ctx->status_code = CSYNC_STATUS_PARAM_ERROR;
return -1;
}
path += strlen(ctx->local.uri) + 1;
break;
case REMOTE_REPLICA:
if (strlen(path) <= strlen(ctx->remote.uri)) {
ctx->status_code = CSYNC_STATUS_PARAM_ERROR;
return -1;
}
path += strlen(ctx->remote.uri) + 1;
break;
default:
path = NULL;
ctx->status_code = CSYNC_STATUS_PARAM_ERROR;
return -1;
}
len = strlen(path);
if (type == CSYNC_FTW_TYPE_SKIP) {
excluded =CSYNC_FILE_EXCLUDE_STAT_FAILED;
} else {
/* Check if file is excluded */
excluded = csync_excluded_traversal(ctx->excludes, path, type);
}
if( excluded == CSYNC_NOT_EXCLUDED ) {
/* Even if it is not excluded by a pattern, maybe it is to be ignored
* because it's a hidden file that should not be synced.
* This code should probably be in csync_exclude, but it does not have the fs parameter.
* Keep it here for now */
if (ctx->ignore_hidden_files && (fs->flags & CSYNC_VIO_FILE_FLAGS_HIDDEN)) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "file excluded because it is a hidden file: %s", path);
excluded = CSYNC_FILE_EXCLUDE_HIDDEN;
}
} else {
/* File is ignored because it's matched by a user- or system exclude pattern. */
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "%s excluded (%d)", path, excluded);
if (excluded == CSYNC_FILE_EXCLUDE_AND_REMOVE) {
return 1;
}
if (excluded == CSYNC_FILE_SILENTLY_EXCLUDED) {
return 1;
}
}
if (ctx->current == REMOTE_REPLICA && ctx->callbacks.checkSelectiveSyncBlackListHook) {
if (ctx->callbacks.checkSelectiveSyncBlackListHook(ctx->callbacks.update_callback_userdata, path)) {
return 1;
}
}
h = _hash_of_file(ctx, file );
if( h == 0 ) {
return -1;
}
size = sizeof(csync_file_stat_t) + len + 1;
st = c_malloc(size);
/* Set instruction by default to none */
st->instruction = CSYNC_INSTRUCTION_NONE;
st->etag = NULL;
st->child_modified = 0;
st->has_ignored_files = 0;
if (type == CSYNC_FTW_TYPE_FILE ) {
if (fs->mtime == 0) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "file: %s - mtime is zero!", path);
}
}
if (excluded > CSYNC_NOT_EXCLUDED || type == CSYNC_FTW_TYPE_SLINK) {
st->instruction = CSYNC_INSTRUCTION_IGNORE;
if (ctx->current_fs) {
ctx->current_fs->has_ignored_files = true;
}
goto out;
}
/* Update detection: Check if a database entry exists.
* If not, the file is either new or has been renamed. To see if it is
* renamed, the db gets queried by the inode of the file as that one
* does not change on rename.
*/
if (csync_get_statedb_exists(ctx)) {
tmp = csync_statedb_get_stat_by_hash(ctx, h);
if(_last_db_return_error(ctx)) {
SAFE_FREE(st);
SAFE_FREE(tmp);
ctx->status_code = CSYNC_STATUS_UNSUCCESSFUL;
return -1;
}
if(tmp && tmp->phash == h ) { /* there is an entry in the database */
/* we have an update! */
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Database entry found, compare: %" PRId64 " <-> %" PRId64
", etag: %s <-> %s, inode: %" PRId64 " <-> %" PRId64
", size: %" PRId64 " <-> %" PRId64 ", perms: %s <-> %s, ignore: %d",
((int64_t) fs->mtime), ((int64_t) tmp->modtime),
fs->etag, tmp->etag, (uint64_t) fs->inode, (uint64_t) tmp->inode,
(uint64_t) fs->size, (uint64_t) tmp->size, fs->remotePerm, tmp->remotePerm, tmp->has_ignored_files );
if (ctx->current == REMOTE_REPLICA && !c_streq(fs->etag, tmp->etag)) {
st->instruction = CSYNC_INSTRUCTION_EVAL;
// Preserve the EVAL flag later on if the type has changed.
if (_csync_filetype_different(tmp, fs)) {
st->child_modified = 1;
}
goto out;
}
if (ctx->current == LOCAL_REPLICA &&
(!_csync_mtime_equal(fs->mtime, tmp->modtime)
// zero size in statedb can happen during migration
|| (tmp->size != 0 && fs->size != tmp->size))) {
// Checksum comparison at this stage is only enabled for .eml files,
// check #4754 #4755
bool isEmlFile = csync_fnmatch("*.eml", file, FNM_CASEFOLD) == 0;
if (isEmlFile && fs->size == tmp->size && tmp->checksumTypeId) {
if (ctx->callbacks.checksum_hook) {
st->checksum = ctx->callbacks.checksum_hook(
file, tmp->checksumTypeId,
ctx->callbacks.checksum_userdata);
}
bool checksumIdentical = false;
if (st->checksum) {
st->checksumTypeId = tmp->checksumTypeId;
checksumIdentical = strncmp(st->checksum, tmp->checksum, 1000) == 0;
}
if (checksumIdentical) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "NOTE: Checksums are identical, file did not actually change: %s", path);
st->instruction = CSYNC_INSTRUCTION_NONE;
st->should_update_metadata = true;
goto out;
}
}
// Preserve the EVAL flag later on if the type has changed.
if (_csync_filetype_different(tmp, fs)) {
st->child_modified = 1;
}
st->instruction = CSYNC_INSTRUCTION_EVAL;
goto out;
}
bool metadata_differ = (ctx->current == REMOTE_REPLICA && (!c_streq(fs->file_id, tmp->file_id)
|| !c_streq(fs->remotePerm, tmp->remotePerm)))
|| (ctx->current == LOCAL_REPLICA && fs->inode != tmp->inode);
if (type == CSYNC_FTW_TYPE_DIR && ctx->current == REMOTE_REPLICA
&& !metadata_differ && ctx->read_remote_from_db) {
/* If both etag and file id are equal for a directory, read all contents from
* the database.
* The metadata comparison ensure that we fetch all the file id or permission when
* upgrading owncloud
*/
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Reading from database: %s", path);
ctx->remote.read_from_db = true;
}
if (metadata_differ) {
/* file id or permissions has changed. Which means we need to update them in the DB. */
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Need to update metadata for: %s", path);
st->should_update_metadata = true;
}
/* If it was remembered in the db that the remote dir has ignored files, store
* that so that the reconciler can make advantage of.
*/
if( ctx->current == REMOTE_REPLICA ) {
st->has_ignored_files = tmp->has_ignored_files;
}
st->instruction = CSYNC_INSTRUCTION_NONE;
} else {
enum csync_vio_file_type_e tmp_vio_type = CSYNC_VIO_FILE_TYPE_UNKNOWN;
/* tmp might point to malloc mem, so free it here before reusing tmp */
SAFE_FREE(tmp);
/* check if it's a file and has been renamed */
if (ctx->current == LOCAL_REPLICA) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Checking for rename based on inode # %" PRId64 "", (uint64_t) fs->inode);
tmp = csync_statedb_get_stat_by_inode(ctx, fs->inode);
if(_last_db_return_error(ctx)) {
SAFE_FREE(st);
ctx->status_code = CSYNC_STATUS_UNSUCCESSFUL;
return -1;
}
/* translate the file type between the two stat types csync has. */
if( tmp && tmp->type == CSYNC_FTW_TYPE_FILE ) {
tmp_vio_type = CSYNC_VIO_FILE_TYPE_REGULAR;
} else if( tmp && tmp->type == CSYNC_FTW_TYPE_DIR) {
tmp_vio_type = CSYNC_VIO_FILE_TYPE_DIRECTORY;
} else if( tmp && tmp->type == CSYNC_FTW_TYPE_SLINK ) {
tmp_vio_type = CSYNC_VIO_FILE_TYPE_SYMBOLIC_LINK;
} else {
tmp_vio_type = CSYNC_VIO_FILE_TYPE_UNKNOWN;
}
// Default to NEW unless we're sure it's a rename.
st->instruction = CSYNC_INSTRUCTION_NEW;
bool isRename =
tmp && tmp->inode == fs->inode && tmp_vio_type == fs->type
&& (tmp->modtime == fs->mtime || fs->type == CSYNC_VIO_FILE_TYPE_DIRECTORY)
#ifdef NO_RENAME_EXTENSION
&& _csync_sameextension(tmp->path, path)
#endif
;
// Verify the checksum where possible
if (isRename && tmp->checksumTypeId && ctx->callbacks.checksum_hook
&& fs->type == CSYNC_VIO_FILE_TYPE_REGULAR) {
st->checksum = ctx->callbacks.checksum_hook(
file, tmp->checksumTypeId,
ctx->callbacks.checksum_userdata);
if (st->checksum) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "checking checksum of potential rename %s %s <-> %s", path, st->checksum, tmp->checksum);
st->checksumTypeId = tmp->checksumTypeId;
isRename = strncmp(st->checksum, tmp->checksum, 1000) == 0;
}
}
if (isRename) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "pot rename detected based on inode # %" PRId64 "", (uint64_t) fs->inode);
/* inode found so the file has been renamed */
st->instruction = CSYNC_INSTRUCTION_EVAL_RENAME;
if (fs->type == CSYNC_VIO_FILE_TYPE_DIRECTORY) {
csync_rename_record(ctx, tmp->path, path);
}
}
goto out;
} else {
/* Remote Replica Rename check */
tmp = csync_statedb_get_stat_by_file_id(ctx, fs->file_id);
if(_last_db_return_error(ctx)) {
SAFE_FREE(st);
ctx->status_code = CSYNC_STATUS_UNSUCCESSFUL;
return -1;
}
if(tmp ) { /* tmp existing at all */
if ( _csync_filetype_different(tmp, fs)) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "WARN: file types different is not!");
st->instruction = CSYNC_INSTRUCTION_NEW;
goto out;
}
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "remote rename detected based on fileid %s %s", tmp->path, file);
st->instruction = CSYNC_INSTRUCTION_EVAL_RENAME;
if (fs->type == CSYNC_VIO_FILE_TYPE_DIRECTORY) {
csync_rename_record(ctx, tmp->path, path);
} else {
if( !c_streq(tmp->etag, fs->etag) ) {
/* CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "ETags are different!"); */
/* File with different etag, don't do a rename, but download the file again */
st->instruction = CSYNC_INSTRUCTION_NEW;
}
}
goto out;
} else {
/* file not found in statedb */
st->instruction = CSYNC_INSTRUCTION_NEW;
if (fs->type == CSYNC_VIO_FILE_TYPE_DIRECTORY && ctx->current == REMOTE_REPLICA && ctx->callbacks.checkSelectiveSyncNewFolderHook) {
if (ctx->callbacks.checkSelectiveSyncNewFolderHook(ctx->callbacks.update_callback_userdata, path)) {
SAFE_FREE(st);
return 1;
}
}
goto out;
}
}
}
} else {
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Unable to open statedb" );
SAFE_FREE(st);
ctx->status_code = CSYNC_STATUS_UNSUCCESSFUL;
return -1;
}
out:
/* Set the ignored error string. */
if (st->instruction == CSYNC_INSTRUCTION_IGNORE) {
if( type == CSYNC_FTW_TYPE_SLINK ) {
st->error_status = CSYNC_STATUS_INDIVIDUAL_IS_SYMLINK; /* Symbolic links are ignored. */
} else {
if (excluded == CSYNC_FILE_EXCLUDE_LIST) {
st->error_status = CSYNC_STATUS_INDIVIDUAL_IGNORE_LIST; /* File listed on ignore list. */
} else if (excluded == CSYNC_FILE_EXCLUDE_INVALID_CHAR) {
st->error_status = CSYNC_STATUS_INDIVIDUAL_IS_INVALID_CHARS; /* File contains invalid characters. */
} else if (excluded == CSYNC_FILE_EXCLUDE_LONG_FILENAME) {
st->error_status = CSYNC_STATUS_INDIVIDUAL_EXCLUDE_LONG_FILENAME; /* File name is too long. */
} else if (excluded == CSYNC_FILE_EXCLUDE_HIDDEN ) {
st->error_status = CSYNC_STATUS_INDIVIDUAL_EXCLUDE_HIDDEN;
} else if (excluded == CSYNC_FILE_EXCLUDE_STAT_FAILED) {
st->error_status = CSYNC_STATUS_INDIVIDUAL_STAT_FAILED;
}
}
}
if (st->instruction != CSYNC_INSTRUCTION_NONE && st->instruction != CSYNC_INSTRUCTION_IGNORE
&& type != CSYNC_FTW_TYPE_DIR) {
st->child_modified = 1;
}
ctx->current_fs = st;
csync_file_stat_free(tmp);
st->inode = fs->inode;
st->mode = fs->mode;
st->size = fs->size;
st->modtime = fs->mtime;
st->type = type;
st->etag = NULL;
if( fs->etag ) {
SAFE_FREE(st->etag);
st->etag = c_strdup(fs->etag);
}
csync_vio_set_file_id(st->file_id, fs->file_id);
if (fs->fields & CSYNC_VIO_FILE_STAT_FIELDS_DIRECTDOWNLOADURL) {
SAFE_FREE(st->directDownloadUrl);
st->directDownloadUrl = c_strdup(fs->directDownloadUrl);
}
if (fs->fields & CSYNC_VIO_FILE_STAT_FIELDS_DIRECTDOWNLOADCOOKIES) {
SAFE_FREE(st->directDownloadCookies);
st->directDownloadCookies = c_strdup(fs->directDownloadCookies);
}
if (fs->fields & CSYNC_VIO_FILE_STAT_FIELDS_PERM) {
strncpy(st->remotePerm, fs->remotePerm, REMOTE_PERM_BUF_SIZE);
}
st->phash = h;
st->pathlen = len;
memcpy(st->path, (len ? path : ""), len + 1);
switch (ctx->current) {
case LOCAL_REPLICA:
if (c_rbtree_insert(ctx->local.tree, (void *) st) < 0) {
SAFE_FREE(st);
ctx->status_code = CSYNC_STATUS_TREE_ERROR;
return -1;
}
break;
case REMOTE_REPLICA:
if (c_rbtree_insert(ctx->remote.tree, (void *) st) < 0) {
SAFE_FREE(st);
ctx->status_code = CSYNC_STATUS_TREE_ERROR;
return -1;
}
break;
default:
break;
}
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "file: %s, instruction: %s <<=", st->path,
csync_instruction_str(st->instruction));
return 0;
}
/*
* local visitor which calls the user visitor with repacked stat info.
*/
static int _csync_treewalk_visitor(void *obj, void *data) {
int rc = 0;
csync_file_stat_t *cur = NULL;
CSYNC *ctx = NULL;
c_rbtree_visit_func *visitor = NULL;
_csync_treewalk_context *twctx = NULL;
TREE_WALK_FILE trav;
c_rbtree_t *other_tree = NULL;
c_rbnode_t *other_node = NULL;
cur = (csync_file_stat_t *) obj;
ctx = (CSYNC *) data;
if (ctx == NULL) {
return -1;
}
/* we need the opposite tree! */
switch (ctx->current) {
case LOCAL_REPLICA:
other_tree = ctx->remote.tree;
break;
case REMOTE_REPLICA:
other_tree = ctx->local.tree;
break;
default:
break;
}
other_node = c_rbtree_find(other_tree, &cur->phash);
if (!other_node) {
/* Check the renamed path as well. */
int len;
uint64_t h = 0;
char *renamed_path = csync_rename_adjust_path(ctx, cur->path);
if (!c_streq(renamed_path, cur->path)) {
len = strlen( renamed_path );
h = c_jhash64((uint8_t *) renamed_path, len, 0);
other_node = c_rbtree_find(other_tree, &h);
}
SAFE_FREE(renamed_path);
}
if (obj == NULL || data == NULL) {
ctx->status_code = CSYNC_STATUS_PARAM_ERROR;
return -1;
}
ctx->status_code = CSYNC_STATUS_OK;
twctx = (_csync_treewalk_context*) ctx->callbacks.userdata;
if (twctx == NULL) {
ctx->status_code = CSYNC_STATUS_PARAM_ERROR;
return -1;
}
if (twctx->instruction_filter > 0 &&
!(twctx->instruction_filter & cur->instruction) ) {
return 0;
}
visitor = (c_rbtree_visit_func*)(twctx->user_visitor);
if (visitor != NULL) {
trav.path = cur->path;
trav.size = cur->size;
trav.modtime = cur->modtime;
trav.uid = cur->uid;
trav.gid = cur->gid;
trav.mode = cur->mode;
trav.type = cur->type;
trav.instruction = cur->instruction;
trav.rename_path = cur->destpath;
trav.etag = cur->etag;
trav.file_id = cur->file_id;
trav.error_status = cur->error_status;
trav.should_update_etag = cur->should_update_etag;
if( other_node ) {
csync_file_stat_t *other_stat = (csync_file_stat_t*)other_node->data;
trav.other.etag = other_stat->etag;
trav.other.file_id = other_stat->file_id;
trav.other.instruction = other_stat->instruction;
trav.other.modtime = other_stat->modtime;
trav.other.size = other_stat->size;
} else {
trav.other.etag = 0;
trav.other.file_id = 0;
trav.other.instruction = CSYNC_INSTRUCTION_NONE;
trav.other.modtime = 0;
trav.other.size = 0;
}
rc = (*visitor)(&trav, twctx->userdata);
cur->instruction = trav.instruction;
if (trav.etag != cur->etag) {
SAFE_FREE(cur->etag);
cur->etag = c_strdup(trav.etag);
}
return rc;
}
ctx->status_code = CSYNC_STATUS_PARAM_ERROR;
return -1;
}
static int _csync_statedb_check(const char *statedb) {
int fd = -1, rc;
ssize_t r;
char buf[BUF_SIZE] = {0};
sqlite3 *db = NULL;
csync_stat_t sb;
mbchar_t *wstatedb = c_utf8_to_locale(statedb);
if (wstatedb == NULL) {
return -1;
}
/* check db version */
#ifdef _WIN32
_fmode = _O_BINARY;
#endif
fd = _topen(wstatedb, O_RDONLY);
if (fd >= 0) {
/* Check size. Size of zero is a valid database actually. */
rc = _tfstat(fd, &sb);
if (rc == 0) {
if (sb.st_size == 0) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "Database size is zero byte!");
close(fd);
} else {
r = read(fd, (void *) buf, sizeof(buf) - 1);
close(fd);
if (r >= 0) {
buf[BUF_SIZE - 1] = '\0';
if (c_streq(buf, "SQLite format 3")) {
if (sqlite3_open(statedb, &db ) == SQLITE_OK) {
rc = _csync_check_db_integrity(db);
if( sqlite3_close(db) != 0 ) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_NOTICE, "WARN: sqlite3_close error!");
}
if( rc >= 0 ) {
/* everything is fine */
c_free_locale_string(wstatedb);
return 0;
}
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "Integrity check failed!");
} else {
/* resources need to be freed even when open failed */
sqlite3_close(db);
CSYNC_LOG(CSYNC_LOG_PRIORITY_WARN, "database corrupted, removing!");
}
} else {
CSYNC_LOG(CSYNC_LOG_PRIORITY_WARN, "sqlite version mismatch");
}
}
}
}
/* if it comes here, the database is broken and should be recreated. */
_tunlink(wstatedb);
}
c_free_locale_string(wstatedb);
/* create database */
rc = sqlite3_open(statedb, &db);
if (rc == SQLITE_OK) {
sqlite3_close(db);
_csync_win32_hide_file(statedb);
return 1;
}
sqlite3_close(db);
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "sqlite3_open failed: %s %s", sqlite3_errmsg(db), statedb);
return -1;
}
END_TEST
START_TEST (check_c_streq_not_equal)
{
fail_if(c_streq("test", "test2"), NULL);
}
int csync_config_parse_file(CSYNC *ctx, const char *config)
{
unsigned int count = 0;
char line[1024] = {0};
char *s;
FILE *f;
/* copy default config, if no config exists */
if (! c_isfile(config)) {
/* check if there is still one csync.conf left over in $HOME/.csync
* and copy it over (migration path)
*/
char *home = NULL;
char *home_config = NULL;
char *config_file = NULL;
/* there is no statedb at the expected place. */
home = csync_get_user_home_dir();
if( !c_streq(home, ctx->options.config_dir) ) {
int rc = -1;
config_file = c_basename(config);
if( config_file ) {
rc = asprintf(&home_config, "%s/%s/%s", home, CSYNC_CONF_DIR, config_file);
SAFE_FREE(config_file);
}
if (rc >= 0) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_NOTICE, "config file %s not found, checking %s",
config, home_config);
/* check the home file and copy to new statedb if existing. */
if(c_isfile(home_config)) {
if (c_copy(home_config, config, 0644) < 0) {
/* copy failed, but that is not reason to die. */
CSYNC_LOG(CSYNC_LOG_PRIORITY_WARN, "Could not copy config %s => %s",
home_config, config);
} else {
CSYNC_LOG(CSYNC_LOG_PRIORITY_NOTICE, "Copied %s => %s",
home_config, config);
}
}
}
SAFE_FREE(home_config);
}
SAFE_FREE(home);
/* Still install the default one if nothing is there. */
if( ! c_isfile(config)) {
if (_csync_config_copy_default(config) < 0) {
return -1;
}
}
}
f = fopen(config, "r");
if (f == NULL) {
return 0;
}
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG,
"Reading configuration data from %s",
config);
s = fgets(line, sizeof(line), f);
while (s != NULL) {
int rc;
count++;
rc = csync_config_parse_line(ctx, line, count);
if (rc < 0) {
fclose(f);
return -1;
}
s = fgets(line, sizeof(line), f);
}
fclose(f);
return 0;
}
/*
* file functions
*/
static int owncloud_stat(const char *uri, csync_vio_file_stat_t *buf) {
/* get props:
* modtime
* creattime
* size
*/
int rc = 0;
csync_vio_file_stat_t *lfs = NULL;
struct listdir_context *fetchCtx = NULL;
char *curi = NULL;
char *decodedUri = NULL;
char strbuf[PATH_MAX +1];
int len = 0;
DEBUG_WEBDAV("owncloud_stat %s called", uri );
buf->name = c_basename(uri);
if (buf->name == NULL) {
csync_vio_file_stat_destroy(buf);
errno = ENOMEM;
return -1;
}
/* check if the data in the static 'cache' fs is for the same file.
* The cache is filled by readdir which is often called directly before
* stat. If the cache matches, a http call is saved.
*/
if( _fs.name && strcmp( buf->name, _fs.name ) == 0 ) {
buf->fields = CSYNC_VIO_FILE_STAT_FIELDS_NONE;
buf->fields |= CSYNC_VIO_FILE_STAT_FIELDS_TYPE;
buf->fields |= CSYNC_VIO_FILE_STAT_FIELDS_SIZE;
buf->fields |= CSYNC_VIO_FILE_STAT_FIELDS_MTIME;
buf->fields |= CSYNC_VIO_FILE_STAT_FIELDS_PERMISSIONS;
buf->fields = _fs.fields;
buf->type = _fs.type;
buf->mtime = _fs.mtime;
buf->size = _fs.size;
buf->mode = _stat_perms( _fs.type );
} else {
/* fetch data via a propfind call. */
fetchCtx = c_malloc( sizeof( struct listdir_context ));
if( ! fetchCtx ) {
errno = ENOMEM;
return -1;
}
curi = _cleanPath( uri );
DEBUG_WEBDAV("I have no stat cache, call propfind for %s.", curi );
fetchCtx->list = NULL;
fetchCtx->target = curi;
fetchCtx->include_target = 1;
fetchCtx->currResource = NULL;
rc = fetch_resource_list( curi, NE_DEPTH_ONE, fetchCtx );
if( rc != NE_OK ) {
if( errno != ENOENT ) {
set_errno_from_session();
}
DEBUG_WEBDAV("stat fails with errno %d", errno );
return -1;
}
if( fetchCtx ) {
struct resource *res = fetchCtx->list;
while( res ) {
/* remove trailing slashes */
len = strlen(res->uri);
while( len > 0 && res->uri[len-1] == '/' ) --len;
memset( strbuf, 0, PATH_MAX+1);
strncpy( strbuf, res->uri, len < PATH_MAX ? len : PATH_MAX );
decodedUri = ne_path_unescape( curi ); /* allocates memory */
if( c_streq(strbuf, decodedUri )) {
SAFE_FREE( decodedUri );
break;
}
res = res->next;
SAFE_FREE( decodedUri );
}
DEBUG_WEBDAV("Working on file %s", res ? res->name : "NULL");
lfs = resourceToFileStat( res );
if( lfs ) {
buf->fields = CSYNC_VIO_FILE_STAT_FIELDS_NONE;
buf->fields |= CSYNC_VIO_FILE_STAT_FIELDS_TYPE;
buf->fields |= CSYNC_VIO_FILE_STAT_FIELDS_SIZE;
buf->fields |= CSYNC_VIO_FILE_STAT_FIELDS_MTIME;
buf->fields |= CSYNC_VIO_FILE_STAT_FIELDS_PERMISSIONS;
buf->fields = lfs->fields;
buf->type = lfs->type;
buf->mtime = lfs->mtime;
buf->size = lfs->size;
buf->mode = _stat_perms( lfs->type );
csync_vio_file_stat_destroy( lfs );
}
SAFE_FREE( fetchCtx );
}
}
DEBUG_WEBDAV("STAT result: %s, type=%d", buf->name ? buf->name:"NULL",
buf->type );
return 0;
}
