static DIR *kqueue_root_start_watch_dir(watchman_global_watcher_t watcher,
w_root_t *root, struct watchman_dir *dir, struct timeval now,
const char *path) {
struct kqueue_root_state *state = root->watch;
DIR *osdir;
struct stat st, osdirst;
struct kevent k;
int newwd;
unused_parameter(watcher);
osdir = opendir_nofollow(path);
if (!osdir) {
handle_open_errno(root, dir, now, "opendir", errno, NULL);
return NULL;
}
newwd = open(path, O_NOFOLLOW|O_EVTONLY|O_CLOEXEC);
if (newwd == -1) {
// directory got deleted between opendir and open
handle_open_errno(root, dir, now, "open", errno, NULL);
closedir(osdir);
return NULL;
}
if (fstat(newwd, &st) == -1 || fstat(dirfd(osdir), &osdirst) == -1) {
// whaaa?
w_log(W_LOG_ERR, "fstat on opened dir %s failed: %s\n", path,
strerror(errno));
w_root_schedule_recrawl(root, "fstat failed");
close(newwd);
closedir(osdir);
return NULL;
}
if (st.st_dev != osdirst.st_dev || st.st_ino != osdirst.st_ino) {
// directory got replaced between opendir and open -- at this point its
// parent's being watched, so we let filesystem events take care of it
handle_open_errno(root, dir, now, "open", ENOTDIR, NULL);
close(newwd);
closedir(osdir);
return NULL;
}
dir->wd = newwd;
memset(&k, 0, sizeof(k));
EV_SET(&k, dir->wd, EVFILT_VNODE, EV_ADD|EV_CLEAR,
NOTE_WRITE|NOTE_DELETE|NOTE_EXTEND|NOTE_RENAME,
0, SET_DIR_BIT(dir));
if (kevent(state->kq_fd, &k, 1, NULL, 0, 0)) {
w_log(W_LOG_DBG, "kevent EV_ADD dir %s failed: %s",
path, strerror(errno));
close(dir->wd);
dir->wd = -1;
}
return osdir;
}
static struct watchman_dir_handle *inot_root_start_watch_dir(
watchman_global_watcher_t watcher,
w_root_t *root, struct watchman_dir *dir, struct timeval now,
const char *path) {
struct inot_root_state *state = root->watch;
struct watchman_dir_handle *osdir = NULL;
int newwd, err;
unused_parameter(watcher);
// Carry out our very strict opendir first to ensure that we're not
// traversing symlinks in the context of this root
osdir = w_dir_open(path);
if (!osdir) {
handle_open_errno(root, dir, now, "opendir", errno, NULL);
return NULL;
}
// The directory might be different since the last time we looked at it, so
// call inotify_add_watch unconditionally.
newwd = inotify_add_watch(state->infd, path, WATCHMAN_INOTIFY_MASK);
if (newwd == -1) {
err = errno;
if (errno == ENOSPC || errno == ENOMEM) {
// Limits exceeded, no recovery from our perspective
set_poison_state(root, dir->path, now, "inotify-add-watch", errno,
inot_strerror(errno));
} else {
handle_open_errno(root, dir, now, "inotify_add_watch", errno,
inot_strerror(errno));
}
w_dir_close(osdir);
errno = err;
return NULL;
}
// record mapping
pthread_mutex_lock(&state->lock);
w_ht_replace(state->wd_to_name, newwd, w_ht_ptr_val(dir->path));
pthread_mutex_unlock(&state->lock);
w_log(W_LOG_DBG, "adding %d -> %s mapping\n", newwd, path);
return osdir;
}
static DIR *winwatch_root_start_watch_dir(watchman_global_watcher_t watcher,
w_root_t *root, struct watchman_dir *dir, struct timeval now,
const char *path) {
DIR *osdir;
unused_parameter(watcher);
osdir = opendir(path);
if (!osdir) {
handle_open_errno(root, dir, now, "opendir", errno, strerror(errno));
return NULL;
}
return osdir;
}
static DIR *portfs_root_start_watch_dir(watchman_global_watcher_t watcher,
w_root_t *root, struct watchman_dir *dir, struct timeval now,
const char *path) {
struct portfs_root_state *state = root->watch;
DIR *osdir;
struct stat st;
unused_parameter(watcher);
osdir = opendir_nofollow(path);
if (!osdir) {
handle_open_errno(root, dir, now, "opendir", errno, NULL);
return NULL;
}
if (fstat(dirfd(osdir), &st) == -1) {
// whaaa?
w_log(W_LOG_ERR, "fstat on opened dir %s failed: %s\n", path,
strerror(errno));
w_root_schedule_recrawl(root, "fstat failed");
closedir(osdir);
return NULL;
}
dir->port_file.fo_mtime = st.st_atim;
dir->port_file.fo_mtime = st.st_mtim;
dir->port_file.fo_ctime = st.st_ctim;
dir->port_file.fo_name = (char*)dir->path->buf;
errno = 0;
if (port_associate(state->port_fd, PORT_SOURCE_FILE,
(uintptr_t)&dir->port_file, WATCHMAN_PORT_EVENTS,
SET_DIR_BIT(dir))) {
w_log(W_LOG_ERR, "port_associate %s %s\n",
dir->port_file.fo_name, strerror(errno));
}
return osdir;
}
void InMemoryView::crawler(
const std::shared_ptr<w_root_t>& root,
SyncView::LockedPtr& view,
PendingCollection::LockedPtr& coll,
const w_string& dir_name,
struct timeval now,
bool recursive) {
struct watchman_file *file;
const watchman_dir_ent* dirent;
char path[WATCHMAN_NAME_MAX];
bool stat_all = false;
if (watcher_->flags & WATCHER_HAS_PER_FILE_NOTIFICATIONS) {
stat_all = watcher_->flags & WATCHER_COALESCED_RENAME;
} else {
// If the watcher doesn't give us per-file notifications for
// watched dirs, then we'll end up explicitly tracking them
// and will get updates for the files explicitly.
// We don't need to look at the files again when we crawl
stat_all = false;
}
auto dir = resolveDir(view, dir_name, true);
// Detect root directory replacement.
// The inode number check is handled more generally by the sister code
// in stat.cpp. We need to special case it for the root because we never
// generate a watchman_file node for the root and thus never call
// InMemoryView::statPath (we'll fault if we do!).
// Ideally the kernel would have given us a signal when we've been replaced
// but some filesystems (eg: BTRFS) do not emit appropriate inotify events
// for things like subvolume deletes. We've seen situations where the
// root has been replaced and we got no notifications at all and this has
// left the cookie sync mechanism broken forever.
if (dir_name == root->root_path) {
try {
auto st = getFileInformation(dir_name.c_str(), root->case_sensitive);
if (st.ino != view->rootInode) {
// If it still exists and the inode doesn't match, then we need
// to force recrawl to make sure we're in sync.
// We're lazily initializing the rootInode to 0 here, so we don't
// need to do this the first time through (we're already crawling
// everything in that case).
if (view->rootInode != 0) {
root->scheduleRecrawl(
"root was replaced and we didn't get notified by the kernel");
return;
}
recursive = true;
view->rootInode = st.ino;
}
} catch (const std::system_error& err) {
handle_open_errno(root, dir, now, "getFileInformation", err.code());
markDirDeleted(view, dir, now, true);
return;
}
}
memcpy(path, dir_name.data(), dir_name.size());
path[dir_name.size()] = 0;
w_log(W_LOG_DBG, "opendir(%s) recursive=%s\n",
path, recursive ? "true" : "false");
/* Start watching and open the dir for crawling.
* Whether we open the dir prior to watching or after is watcher specific,
* so the operations are rolled together in our abstraction */
std::unique_ptr<watchman_dir_handle> osdir;
try {
osdir = watcher_->startWatchDir(root, dir, path);
} catch (const std::system_error& err) {
handle_open_errno(root, dir, now, "opendir", err.code());
markDirDeleted(view, dir, now, true);
return;
}
if (dir->files.empty()) {
// Pre-size our hash(es) if we can, so that we can avoid collisions
// and re-hashing during initial crawl
uint32_t num_dirs = 0;
#ifndef _WIN32
struct stat st;
int dfd = osdir->getFd();
if (dfd != -1 && fstat(dfd, &st) == 0) {
num_dirs = (uint32_t)st.st_nlink;
}
#endif
// st.st_nlink is usually number of dirs + 2 (., ..).
// If it is less than 2 then it doesn't follow that convention.
// We just pass it through for the dir size hint and the hash
// table implementation will round that up to the next power of 2
apply_dir_size_hint(
dir,
num_dirs,
uint32_t(root->config.getInt("hint_num_files_per_dir", 64)));
}
/* flag for delete detection */
for (auto& it : dir->files) {
auto file = it.second.get();
if (file->exists) {
file->maybe_deleted = true;
}
}
try {
while ((dirent = osdir->readDir()) != nullptr) {
// Don't follow parent/self links
if (dirent->d_name[0] == '.' &&
(!strcmp(dirent->d_name, ".") || !strcmp(dirent->d_name, ".."))) {
continue;
}
// Queue it up for analysis if the file is newly existing
w_string name(dirent->d_name, W_STRING_BYTE);
file = dir->getChildFile(name);
if (file) {
file->maybe_deleted = false;
}
if (!file || !file->exists || stat_all || recursive) {
auto full_path = dir->getFullPathToChild(name);
w_log(
W_LOG_DBG,
"in crawler calling process_path on %s\n",
full_path.c_str());
processPath(
root,
view,
coll,
full_path,
now,
((recursive || !file || !file->exists) ? W_PENDING_RECURSIVE : 0),
dirent);
}
}
} catch (const std::system_error& exc) {
log(ERR,
"Error while reading dir ",
path,
": ",
exc.what(),
", re-adding to pending list to re-assess\n");
coll->add(path, now, 0);
}
osdir.reset();
// Anything still in maybe_deleted is actually deleted.
// Arrange to re-process it shortly
for (auto& it : dir->files) {
auto file = it.second.get();
if (file->exists &&
(file->maybe_deleted || (file->stat.isDir() && recursive))) {
coll->add(
dir,
file->getName().data(),
now,
recursive ? W_PENDING_RECURSIVE : 0);
}
}
}
static DIR *kqueue_root_start_watch_dir(watchman_global_watcher_t watcher,
w_root_t *root, struct watchman_dir *dir, struct timeval now,
const char *path) {
struct kqueue_root_state *state = root->watch;
DIR *osdir;
struct stat st, osdirst;
struct kevent k;
int newwd;
unused_parameter(watcher);
osdir = opendir_nofollow(path);
if (!osdir) {
handle_open_errno(root, dir, now, "opendir", errno, NULL);
return NULL;
}
newwd = open(path, O_NOFOLLOW|O_EVTONLY|O_CLOEXEC);
if (newwd == -1) {
// directory got deleted between opendir and open
handle_open_errno(root, dir, now, "open", errno, NULL);
closedir(osdir);
return NULL;
}
if (fstat(newwd, &st) == -1 || fstat(dirfd(osdir), &osdirst) == -1) {
// whaaa?
w_log(W_LOG_ERR, "fstat on opened dir %s failed: %s\n", path,
strerror(errno));
w_root_schedule_recrawl(root, "fstat failed");
close(newwd);
closedir(osdir);
return NULL;
}
if (st.st_dev != osdirst.st_dev || st.st_ino != osdirst.st_ino) {
// directory got replaced between opendir and open -- at this point its
// parent's being watched, so we let filesystem events take care of it
handle_open_errno(root, dir, now, "open", ENOTDIR, NULL);
close(newwd);
closedir(osdir);
return NULL;
}
memset(&k, 0, sizeof(k));
EV_SET(&k, newwd, EVFILT_VNODE, EV_ADD|EV_CLEAR,
NOTE_WRITE|NOTE_DELETE|NOTE_EXTEND|NOTE_RENAME,
0, SET_DIR_BIT(dir->path));
// Our mapping needs to be visible before we add it to the queue,
// otherwise we can get a wakeup and not know what it is
pthread_mutex_lock(&state->lock);
w_ht_replace(state->name_to_fd, w_ht_ptr_val(dir->path), newwd);
w_ht_replace(state->fd_to_name, newwd, w_ht_ptr_val(dir->path));
pthread_mutex_unlock(&state->lock);
if (kevent(state->kq_fd, &k, 1, NULL, 0, 0)) {
w_log(W_LOG_DBG, "kevent EV_ADD dir %s failed: %s",
path, strerror(errno));
close(newwd);
pthread_mutex_lock(&state->lock);
w_ht_del(state->name_to_fd, w_ht_ptr_val(dir->path));
w_ht_del(state->fd_to_name, newwd);
pthread_mutex_unlock(&state->lock);
} else {
w_log(W_LOG_DBG, "kevent dir %s -> %d\n", dir->path->buf, newwd);
}
return osdir;
}
