static void check_csync_vio_readdir(void **state)
{
CSYNC *csync = *state;
csync_vio_handle_t *dh;
csync_vio_file_stat_t *dirent;
int rc;
dh = csync_vio_opendir(csync, CSYNC_TEST_DIR);
assert_non_null(dh);
dirent = csync_vio_readdir(csync, dh);
assert_non_null(dirent);
csync_vio_file_stat_destroy(dirent);
rc = csync_vio_closedir(csync, dh);
assert_int_equal(rc, 0);
}
/* File tree walker */
int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn,
unsigned int depth) {
char *filename = NULL;
char *d_name = NULL;
csync_vio_handle_t *dh = NULL;
csync_vio_file_stat_t *dirent = NULL;
csync_vio_file_stat_t *fs = NULL;
csync_file_stat_t *previous_fs = NULL;
int read_from_db = 0;
int rc = 0;
int res = 0;
bool do_read_from_db = (ctx->current == REMOTE_REPLICA && ctx->remote.read_from_db);
if (uri[0] == '\0') {
errno = ENOENT;
ctx->status_code = CSYNC_STATUS_PARAM_ERROR;
goto error;
}
read_from_db = ctx->remote.read_from_db;
// if the etag of this dir is still the same, its content is restored from the
// database.
if( do_read_from_db ) {
if( ! fill_tree_from_db(ctx, uri) ) {
errno = ENOENT;
ctx->status_code = CSYNC_STATUS_OPENDIR_ERROR;
goto error;
}
goto done;
}
if ((dh = csync_vio_opendir(ctx, uri)) == NULL) {
int asp = 0;
/* permission denied */
ctx->status_code = csync_errno_to_status(errno, CSYNC_STATUS_OPENDIR_ERROR);
if (errno == EACCES) {
return 0;
} else if(errno == ENOENT) {
asp = asprintf( &ctx->error_string, "%s", uri);
if (asp < 0) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "asprintf failed!");
}
} else {
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "opendir failed for %s - errno %d", uri, errno);
}
goto error;
}
while ((dirent = csync_vio_readdir(ctx, dh))) {
const char *path = NULL;
size_t ulen = 0;
int flen;
int flag;
d_name = dirent->name;
if (d_name == NULL) {
ctx->status_code = CSYNC_STATUS_READDIR_ERROR;
goto error;
}
/* skip "." and ".." */
if (d_name[0] == '.' && (d_name[1] == '\0'
|| (d_name[1] == '.' && d_name[2] == '\0'))) {
csync_vio_file_stat_destroy(dirent);
dirent = NULL;
continue;
}
flen = asprintf(&filename, "%s/%s", uri, d_name);
if (flen < 0) {
csync_vio_file_stat_destroy(dirent);
dirent = NULL;
ctx->status_code = CSYNC_STATUS_MEMORY_ERROR;
goto error;
}
/* Create relative path */
switch (ctx->current) {
case LOCAL_REPLICA:
ulen = strlen(ctx->local.uri) + 1;
break;
case REMOTE_REPLICA:
ulen = strlen(ctx->remote.uri) + 1;
break;
default:
break;
}
if (((size_t)flen) < ulen) {
csync_vio_file_stat_destroy(dirent);
dirent = NULL;
ctx->status_code = CSYNC_STATUS_UNSUCCESSFUL;
goto error;
}
path = filename + ulen;
/* skip ".csync_journal.db" and ".csync_journal.db.ctmp" */
if (c_streq(path, ".csync_journal.db")
|| c_streq(path, ".csync_journal.db.ctmp")
|| c_streq(path, ".csync_journal.db.ctmp-journal")
|| c_streq(path, ".csync-progressdatabase")) {
csync_vio_file_stat_destroy(dirent);
dirent = NULL;
SAFE_FREE(filename);
continue;
}
/* Only for the local replica we have to stat(), for the remote one we have all data already */
if (ctx->replica == LOCAL_REPLICA) {
fs = csync_vio_file_stat_new();
res = csync_vio_stat(ctx, filename, fs);
} else {
fs = dirent;
res = 0;
}
if( res == 0) {
switch (fs->type) {
case CSYNC_VIO_FILE_TYPE_SYMBOLIC_LINK:
flag = CSYNC_FTW_FLAG_SLINK;
break;
case CSYNC_VIO_FILE_TYPE_DIRECTORY:
flag = CSYNC_FTW_FLAG_DIR;
break;
case CSYNC_VIO_FILE_TYPE_BLOCK_DEVICE:
case CSYNC_VIO_FILE_TYPE_CHARACTER_DEVICE:
case CSYNC_VIO_FILE_TYPE_SOCKET:
flag = CSYNC_FTW_FLAG_SPEC;
break;
case CSYNC_VIO_FILE_TYPE_FIFO:
flag = CSYNC_FTW_FLAG_SPEC;
break;
default:
flag = CSYNC_FTW_FLAG_FILE;
break;
};
} else {
flag = CSYNC_FTW_FLAG_NSTAT;
}
if( ctx->current == LOCAL_REPLICA ) {
char *etag = NULL;
int len = strlen( path );
uint64_t h = c_jhash64((uint8_t *) path, len, 0);
etag = csync_statedb_get_etag( ctx, h );
if( etag ) {
SAFE_FREE(fs->etag);
fs->etag = etag;
fs->fields |= CSYNC_VIO_FILE_STAT_FIELDS_ETAG;
if( c_streq(etag, "")) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Uniq ID from Database is EMPTY: %s", path);
} else {
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Uniq ID from Database: %s -> %s", path, fs->etag ? fs->etag : "<NULL>" );
}
}
}
previous_fs = ctx->current_fs;
/* Call walker function for each file */
rc = fn(ctx, filename, fs, flag);
/* this function may update ctx->current and ctx->read_from_db */
if (ctx->current_fs && previous_fs && ctx->current_fs->child_modified) {
previous_fs->child_modified = ctx->current_fs->child_modified;
}
/* Only for the local replica we have to destroy stat(), for the remote one it is a pointer to dirent */
if (ctx->replica == LOCAL_REPLICA) {
csync_vio_file_stat_destroy(fs);
}
if (rc < 0) {
if (CSYNC_STATUS_IS_OK(ctx->status_code)) {
ctx->status_code = CSYNC_STATUS_UPDATE_ERROR;
}
ctx->current_fs = previous_fs;
goto error;
}
if (flag == CSYNC_FTW_FLAG_DIR && depth
&& (!ctx->current_fs || ctx->current_fs->instruction != CSYNC_INSTRUCTION_IGNORE)) {
rc = csync_ftw(ctx, filename, fn, depth - 1);
if (rc < 0) {
ctx->current_fs = previous_fs;
goto error;
}
if (ctx->current_fs && !ctx->current_fs->child_modified
&& ctx->current_fs->instruction == CSYNC_INSTRUCTION_EVAL) {
ctx->current_fs->instruction = CSYNC_INSTRUCTION_NONE;
ctx->current_fs->should_update_etag = true;
}
}
if (flag == CSYNC_FTW_FLAG_DIR && ctx->current_fs
&& (ctx->current_fs->instruction == CSYNC_INSTRUCTION_EVAL ||
ctx->current_fs->instruction == CSYNC_INSTRUCTION_NEW ||
ctx->current_fs->instruction == CSYNC_INSTRUCTION_EVAL_RENAME)) {
ctx->current_fs->should_update_etag = true;
}
ctx->current_fs = previous_fs;
ctx->remote.read_from_db = read_from_db;
SAFE_FREE(filename);
csync_vio_file_stat_destroy(dirent);
dirent = NULL;
}
csync_vio_closedir(ctx, dh);
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, " <= Closing walk for %s with read_from_db %d", uri, read_from_db);
done:
csync_vio_file_stat_destroy(dirent);
SAFE_FREE(filename);
return rc;
error:
ctx->remote.read_from_db = read_from_db;
if (dh != NULL) {
csync_vio_closedir(ctx, dh);
}
SAFE_FREE(filename);
return -1;
}
/*
* This function uses the vio_opendir, vio_readdir and vio_closedir functions
* to traverse a file tree that was created before by the create_dir function.
*
* It appends a listing to the result member of the incoming struct in *state
* that can be compared later to what was expected in the calling functions.
*
* The int parameter cnt contains the number of seen files (not dirs) in the
* whole tree.
*
*/
static void traverse_dir(void **state, const char *dir, int *cnt)
{
csync_vio_handle_t *dh;
csync_vio_file_stat_t *dirent;
statevar *sv = (statevar*) *state;
CSYNC *csync = sv->csync;
char *subdir;
char *subdir_out;
int rc;
int is_dir;
/* Format: Smuggle in the C: for unix platforms as its urgently needed
* on Windows and the test can be nicely cross platform this way. */
#ifdef _WIN32
const char *format_str = "%s %s";
#else
const char *format_str = "%s C:%s";
#endif
dh = csync_vio_opendir(csync, dir);
assert_non_null(dh);
while( (dirent = csync_vio_readdir(csync, dh)) ) {
assert_non_null(dirent);
if (dirent->original_name) {
sv->ignored_dir = c_strdup(dirent->original_name);
continue;
}
assert_non_null(dirent->name);
assert_int_equal( dirent->fields & CSYNC_VIO_FILE_STAT_FIELDS_TYPE, CSYNC_VIO_FILE_STAT_FIELDS_TYPE );
if( c_streq( dirent->name, "..") || c_streq( dirent->name, "." )) {
continue;
}
is_dir = (dirent->type == CSYNC_VIO_FILE_TYPE_DIRECTORY) ? 1:0;
assert_int_not_equal( asprintf( &subdir, "%s/%s", dir, dirent->name ), -1 );
assert_int_not_equal( asprintf( &subdir_out, format_str,
is_dir ? "<DIR>":" ",
subdir), -1 );
if( is_dir ) {
if( !sv->result ) {
sv->result = c_strdup( subdir_out);
} else {
int newlen = 1+strlen(sv->result)+strlen(subdir_out);
char *tmp = sv->result;
sv->result = c_malloc(newlen);
strcpy( sv->result, tmp);
SAFE_FREE(tmp);
strcat( sv->result, subdir_out );
}
} else {
*cnt = *cnt +1;
}
output(subdir_out);
if( is_dir ) {
traverse_dir( state, subdir, cnt);
}
SAFE_FREE(subdir);
SAFE_FREE(subdir_out);
}
csync_vio_file_stat_destroy(dirent);
rc = csync_vio_closedir(csync, dh);
assert_int_equal(rc, 0);
}
/* File tree walker */
int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn,
unsigned int depth) {
char *filename = NULL;
char *d_name = NULL;
csync_vio_handle_t *dh = NULL;
csync_vio_file_stat_t *dirent = NULL;
csync_file_stat_t *previous_fs = NULL;
int read_from_db = 0;
int rc = 0;
int res = 0;
bool do_read_from_db = (ctx->current == REMOTE_REPLICA && ctx->remote.read_from_db);
if (uri[0] == '\0') {
errno = ENOENT;
ctx->status_code = CSYNC_STATUS_PARAM_ERROR;
goto error;
}
read_from_db = ctx->remote.read_from_db;
// if the etag of this dir is still the same, its content is restored from the
// database.
if( do_read_from_db ) {
if( ! fill_tree_from_db(ctx, uri) ) {
errno = ENOENT;
ctx->status_code = CSYNC_STATUS_OPENDIR_ERROR;
goto error;
}
goto done;
}
const char *uri_for_vio = uri;
if (ctx->current == REMOTE_REPLICA) {
uri_for_vio += strlen(ctx->remote.uri);
if (strlen(uri_for_vio) > 0 && uri_for_vio[0] == '/') {
uri_for_vio++; // cut leading slash
}
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "URI without fuzz for %s is \"%s\"", uri, uri_for_vio);
}
if ((dh = csync_vio_opendir(ctx, uri_for_vio)) == NULL) {
if (ctx->abort) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Aborted!");
ctx->status_code = CSYNC_STATUS_ABORTED;
goto error;
}
int asp = 0;
/* permission denied */
ctx->status_code = csync_errno_to_status(errno, CSYNC_STATUS_OPENDIR_ERROR);
if (errno == EACCES) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_WARN, "Permission denied.");
if (mark_current_item_ignored(ctx, previous_fs, CSYNC_STATUS_PERMISSION_DENIED)) {
goto done;
}
} else if(errno == ENOENT) {
asp = asprintf( &ctx->error_string, "%s", uri);
if (asp < 0) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "asprintf failed!");
}
}
// 403 Forbidden can be sent by the server if the file firewall is active.
// A file or directory should be ignored and sync must continue. See #3490
else if(errno == ERRNO_FORBIDDEN) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_WARN, "Directory access Forbidden (File Firewall?)");
if( mark_current_item_ignored(ctx, previous_fs, CSYNC_STATUS_FORBIDDEN) ) {
goto done;
}
/* if current_fs is not defined here, better throw an error */
}
// The server usually replies with the custom "503 Storage not available"
// if some path is temporarily unavailable. But in some cases a standard 503
// is returned too. Thus we can't distinguish the two and will treat any
// 503 as request to ignore the folder. See #3113 #2884.
else if(errno == ERRNO_STORAGE_UNAVAILABLE || errno == ERRNO_SERVICE_UNAVAILABLE) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_WARN, "Storage was not available!");
if( mark_current_item_ignored(ctx, previous_fs, CSYNC_STATUS_STORAGE_UNAVAILABLE ) ) {
goto done;
}
/* if current_fs is not defined here, better throw an error */
} else {
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "opendir failed for %s - errno %d", uri, errno);
}
goto error;
}
while ((dirent = csync_vio_readdir(ctx, dh))) {
const char *path = NULL;
size_t ulen = 0;
int flen;
int flag;
/* Conversion error */
if (dirent->name == NULL && dirent->original_name) {
ctx->status_code = CSYNC_STATUS_INVALID_CHARACTERS;
ctx->error_string = dirent->original_name; // take ownership
dirent->original_name = NULL;
goto error;
}
d_name = dirent->name;
if (d_name == NULL) {
ctx->status_code = CSYNC_STATUS_READDIR_ERROR;
goto error;
}
/* skip "." and ".." */
if ( (d_name[0] == '.' && d_name[1] == '\0')
|| (d_name[0] == '.' && d_name[1] == '.' && d_name[2] == '\0')) {
csync_vio_file_stat_destroy(dirent);
dirent = NULL;
continue;
}
flen = asprintf(&filename, "%s/%s", uri, d_name);
if (flen < 0) {
csync_vio_file_stat_destroy(dirent);
dirent = NULL;
ctx->status_code = CSYNC_STATUS_MEMORY_ERROR;
goto error;
}
/* Create relative path */
switch (ctx->current) {
case LOCAL_REPLICA:
ulen = strlen(ctx->local.uri) + 1;
break;
case REMOTE_REPLICA:
ulen = strlen(ctx->remote.uri) + 1;
break;
default:
break;
}
if (((size_t)flen) < ulen) {
csync_vio_file_stat_destroy(dirent);
dirent = NULL;
ctx->status_code = CSYNC_STATUS_UNSUCCESSFUL;
goto error;
}
path = filename + ulen;
/* skip ".csync_journal.db" and ".csync_journal.db.ctmp" */
/* Isn't this done via csync_exclude already? */
if (c_streq(path, ".csync_journal.db")
|| c_streq(path, ".csync_journal.db.ctmp")
|| c_streq(path, ".csync_journal.db.ctmp-journal")
|| c_streq(path, ".csync-progressdatabase")
|| c_streq(path, ".csync_journal.db-shm")
|| c_streq(path, ".csync_journal.db-wal")
|| c_streq(path, ".csync_journal.db-journal")) {
csync_vio_file_stat_destroy(dirent);
dirent = NULL;
SAFE_FREE(filename);
continue;
}
/* Only for the local replica we have to stat(), for the remote one we have all data already */
if (ctx->replica == LOCAL_REPLICA) {
res = csync_vio_stat(ctx, filename, dirent);
} else {
res = 0;
}
/* if the filename starts with a . we consider it a hidden file
* For windows, the hidden state is also discovered within the vio
* local stat function.
*/
if( d_name[0] == '.' ) {
if (strcmp(".sys.admin#recall#", d_name) != 0) { /* recall file shall not be ignored (#4420) */
dirent->flags |= CSYNC_VIO_FILE_FLAGS_HIDDEN;
}
}
if( res == 0) {
switch (dirent->type) {
case CSYNC_VIO_FILE_TYPE_SYMBOLIC_LINK:
flag = CSYNC_FTW_FLAG_SLINK;
break;
case CSYNC_VIO_FILE_TYPE_DIRECTORY:
flag = CSYNC_FTW_FLAG_DIR;
break;
case CSYNC_VIO_FILE_TYPE_BLOCK_DEVICE:
case CSYNC_VIO_FILE_TYPE_CHARACTER_DEVICE:
case CSYNC_VIO_FILE_TYPE_SOCKET:
flag = CSYNC_FTW_FLAG_SPEC;
break;
case CSYNC_VIO_FILE_TYPE_FIFO:
flag = CSYNC_FTW_FLAG_SPEC;
break;
default:
flag = CSYNC_FTW_FLAG_FILE;
break;
};
} else {
flag = CSYNC_FTW_FLAG_NSTAT;
}
previous_fs = ctx->current_fs;
/* Call walker function for each file */
rc = fn(ctx, filename, dirent, flag);
/* this function may update ctx->current and ctx->read_from_db */
if (rc < 0) {
if (CSYNC_STATUS_IS_OK(ctx->status_code)) {
ctx->status_code = CSYNC_STATUS_UPDATE_ERROR;
}
ctx->current_fs = previous_fs;
goto error;
}
if (flag == CSYNC_FTW_FLAG_DIR && depth && rc == 0
&& (!ctx->current_fs || ctx->current_fs->instruction != CSYNC_INSTRUCTION_IGNORE)) {
rc = csync_ftw(ctx, filename, fn, depth - 1);
if (rc < 0) {
ctx->current_fs = previous_fs;
goto error;
}
if (ctx->current_fs && !ctx->current_fs->child_modified
&& ctx->current_fs->instruction == CSYNC_INSTRUCTION_EVAL) {
ctx->current_fs->instruction = CSYNC_INSTRUCTION_NONE;
if (ctx->current == REMOTE_REPLICA) {
ctx->current_fs->should_update_metadata = true;
}
}
if (ctx->current_fs && previous_fs && ctx->current_fs->has_ignored_files) {
/* If a directory has ignored files, put the flag on the parent directory as well */
previous_fs->has_ignored_files = ctx->current_fs->has_ignored_files;
}
}
if (ctx->current_fs && previous_fs && ctx->current_fs->child_modified) {
/* If a directory has modified files, put the flag on the parent directory as well */
previous_fs->child_modified = ctx->current_fs->child_modified;
}
if (flag == CSYNC_FTW_FLAG_DIR && ctx->current_fs
&& (ctx->current_fs->instruction == CSYNC_INSTRUCTION_EVAL ||
ctx->current_fs->instruction == CSYNC_INSTRUCTION_NEW)) {
ctx->current_fs->should_update_metadata = true;
}
ctx->current_fs = previous_fs;
ctx->remote.read_from_db = read_from_db;
SAFE_FREE(filename);
csync_vio_file_stat_destroy(dirent);
dirent = NULL;
}
csync_vio_closedir(ctx, dh);
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, " <= Closing walk for %s with read_from_db %d", uri, read_from_db);
done:
csync_vio_file_stat_destroy(dirent);
SAFE_FREE(filename);
return rc;
error:
ctx->remote.read_from_db = read_from_db;
if (dh != NULL) {
csync_vio_closedir(ctx, dh);
}
SAFE_FREE(filename);
return -1;
}
/* File tree walker */
int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn,
unsigned int depth) {
char errbuf[256] = {0};
char *filename = NULL;
char *d_name = NULL;
csync_vio_handle_t *dh = NULL;
csync_vio_file_stat_t *dirent = NULL;
csync_vio_file_stat_t *fs = NULL;
int rc = 0;
if (uri[0] == '\0') {
errno = ENOENT;
goto error;
}
if ((dh = csync_vio_opendir(ctx, uri)) == NULL) {
/* permission denied */
if (errno == EACCES) {
return 0;
} else {
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR,
"opendir failed for %s - %s",
uri,
strerror_r(errno, errbuf, sizeof(errbuf)));
goto error;
}
}
while ((dirent = csync_vio_readdir(ctx, dh))) {
const char *path = NULL;
int flag;
d_name = dirent->name;
if (d_name == NULL) {
goto error;
}
/* skip "." and ".." */
if (d_name[0] == '.' && (d_name[1] == '\0'
|| (d_name[1] == '.' && d_name[2] == '\0'))) {
csync_vio_file_stat_destroy(dirent);
dirent = NULL;
continue;
}
if (asprintf(&filename, "%s/%s", uri, d_name) < 0) {
csync_vio_file_stat_destroy(dirent);
dirent = NULL;
goto error;
}
/* Create relative path for checking the exclude list */
switch (ctx->current) {
case LOCAL_REPLICA:
path = filename + strlen(ctx->local.uri) + 1;
break;
case REMOTE_REPLCIA:
path = filename + strlen(ctx->remote.uri) + 1;
break;
default:
break;
}
/* Check if file is excluded */
if (csync_excluded(ctx, path)) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "%s excluded", path);
csync_vio_file_stat_destroy(dirent);
dirent = NULL;
continue;
}
fs = csync_vio_file_stat_new();
if (csync_vio_stat(ctx, filename, fs) == 0) {
switch (fs->type) {
case CSYNC_VIO_FILE_TYPE_SYMBOLIC_LINK:
flag = CSYNC_FTW_FLAG_SLINK;
break;
case CSYNC_VIO_FILE_TYPE_DIRECTORY:
flag = CSYNC_FTW_FLAG_DIR;
break;
case CSYNC_VIO_FILE_TYPE_BLOCK_DEVICE:
case CSYNC_VIO_FILE_TYPE_CHARACTER_DEVICE:
case CSYNC_VIO_FILE_TYPE_SOCKET:
flag = CSYNC_FTW_FLAG_SPEC;
break;
case CSYNC_VIO_FILE_TYPE_FIFO:
flag = CSYNC_FTW_FLAG_SPEC;
break;
default:
flag = CSYNC_FTW_FLAG_FILE;
break;
};
} else {
flag = CSYNC_FTW_FLAG_NSTAT;
}
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "walk: %s", filename);
/* Call walker function for each file */
rc = fn(ctx, filename, fs, flag);
csync_vio_file_stat_destroy(fs);
if (rc < 0) {
csync_vio_closedir(ctx, dh);
goto done;
}
if (flag == CSYNC_FTW_FLAG_DIR && depth) {
rc = csync_ftw(ctx, filename, fn, depth - 1);
if (rc < 0) {
csync_vio_closedir(ctx, dh);
goto done;
}
}
SAFE_FREE(filename);
csync_vio_file_stat_destroy(dirent);
dirent = NULL;
}
csync_vio_closedir(ctx, dh);
done:
csync_vio_file_stat_destroy(dirent);
SAFE_FREE(filename);
return rc;
error:
SAFE_FREE(filename);
return -1;
}