static DIR *kqueue_root_start_watch_dir(watchman_global_watcher_t watcher, w_root_t *root, struct watchman_dir *dir, struct timeval now, const char *path) { struct kqueue_root_state *state = root->watch; DIR *osdir; struct stat st, osdirst; struct kevent k; int newwd; unused_parameter(watcher); osdir = opendir_nofollow(path); if (!osdir) { handle_open_errno(root, dir, now, "opendir", errno, NULL); return NULL; } newwd = open(path, O_NOFOLLOW|O_EVTONLY|O_CLOEXEC); if (newwd == -1) { // directory got deleted between opendir and open handle_open_errno(root, dir, now, "open", errno, NULL); closedir(osdir); return NULL; } if (fstat(newwd, &st) == -1 || fstat(dirfd(osdir), &osdirst) == -1) { // whaaa? w_log(W_LOG_ERR, "fstat on opened dir %s failed: %s\n", path, strerror(errno)); w_root_schedule_recrawl(root, "fstat failed"); close(newwd); closedir(osdir); return NULL; } if (st.st_dev != osdirst.st_dev || st.st_ino != osdirst.st_ino) { // directory got replaced between opendir and open -- at this point its // parent's being watched, so we let filesystem events take care of it handle_open_errno(root, dir, now, "open", ENOTDIR, NULL); close(newwd); closedir(osdir); return NULL; } dir->wd = newwd; memset(&k, 0, sizeof(k)); EV_SET(&k, dir->wd, EVFILT_VNODE, EV_ADD|EV_CLEAR, NOTE_WRITE|NOTE_DELETE|NOTE_EXTEND|NOTE_RENAME, 0, SET_DIR_BIT(dir)); if (kevent(state->kq_fd, &k, 1, NULL, 0, 0)) { w_log(W_LOG_DBG, "kevent EV_ADD dir %s failed: %s", path, strerror(errno)); close(dir->wd); dir->wd = -1; } return osdir; }
static struct watchman_dir_handle *inot_root_start_watch_dir( watchman_global_watcher_t watcher, w_root_t *root, struct watchman_dir *dir, struct timeval now, const char *path) { struct inot_root_state *state = root->watch; struct watchman_dir_handle *osdir = NULL; int newwd, err; unused_parameter(watcher); // Carry out our very strict opendir first to ensure that we're not // traversing symlinks in the context of this root osdir = w_dir_open(path); if (!osdir) { handle_open_errno(root, dir, now, "opendir", errno, NULL); return NULL; } // The directory might be different since the last time we looked at it, so // call inotify_add_watch unconditionally. newwd = inotify_add_watch(state->infd, path, WATCHMAN_INOTIFY_MASK); if (newwd == -1) { err = errno; if (errno == ENOSPC || errno == ENOMEM) { // Limits exceeded, no recovery from our perspective set_poison_state(root, dir->path, now, "inotify-add-watch", errno, inot_strerror(errno)); } else { handle_open_errno(root, dir, now, "inotify_add_watch", errno, inot_strerror(errno)); } w_dir_close(osdir); errno = err; return NULL; } // record mapping pthread_mutex_lock(&state->lock); w_ht_replace(state->wd_to_name, newwd, w_ht_ptr_val(dir->path)); pthread_mutex_unlock(&state->lock); w_log(W_LOG_DBG, "adding %d -> %s mapping\n", newwd, path); return osdir; }
static DIR *winwatch_root_start_watch_dir(watchman_global_watcher_t watcher, w_root_t *root, struct watchman_dir *dir, struct timeval now, const char *path) { DIR *osdir; unused_parameter(watcher); osdir = opendir(path); if (!osdir) { handle_open_errno(root, dir, now, "opendir", errno, strerror(errno)); return NULL; } return osdir; }
static DIR *portfs_root_start_watch_dir(watchman_global_watcher_t watcher, w_root_t *root, struct watchman_dir *dir, struct timeval now, const char *path) { struct portfs_root_state *state = root->watch; DIR *osdir; struct stat st; unused_parameter(watcher); osdir = opendir_nofollow(path); if (!osdir) { handle_open_errno(root, dir, now, "opendir", errno, NULL); return NULL; } if (fstat(dirfd(osdir), &st) == -1) { // whaaa? w_log(W_LOG_ERR, "fstat on opened dir %s failed: %s\n", path, strerror(errno)); w_root_schedule_recrawl(root, "fstat failed"); closedir(osdir); return NULL; } dir->port_file.fo_mtime = st.st_atim; dir->port_file.fo_mtime = st.st_mtim; dir->port_file.fo_ctime = st.st_ctim; dir->port_file.fo_name = (char*)dir->path->buf; errno = 0; if (port_associate(state->port_fd, PORT_SOURCE_FILE, (uintptr_t)&dir->port_file, WATCHMAN_PORT_EVENTS, SET_DIR_BIT(dir))) { w_log(W_LOG_ERR, "port_associate %s %s\n", dir->port_file.fo_name, strerror(errno)); } return osdir; }
void InMemoryView::crawler( const std::shared_ptr<w_root_t>& root, SyncView::LockedPtr& view, PendingCollection::LockedPtr& coll, const w_string& dir_name, struct timeval now, bool recursive) { struct watchman_file *file; const watchman_dir_ent* dirent; char path[WATCHMAN_NAME_MAX]; bool stat_all = false; if (watcher_->flags & WATCHER_HAS_PER_FILE_NOTIFICATIONS) { stat_all = watcher_->flags & WATCHER_COALESCED_RENAME; } else { // If the watcher doesn't give us per-file notifications for // watched dirs, then we'll end up explicitly tracking them // and will get updates for the files explicitly. // We don't need to look at the files again when we crawl stat_all = false; } auto dir = resolveDir(view, dir_name, true); // Detect root directory replacement. // The inode number check is handled more generally by the sister code // in stat.cpp. We need to special case it for the root because we never // generate a watchman_file node for the root and thus never call // InMemoryView::statPath (we'll fault if we do!). // Ideally the kernel would have given us a signal when we've been replaced // but some filesystems (eg: BTRFS) do not emit appropriate inotify events // for things like subvolume deletes. We've seen situations where the // root has been replaced and we got no notifications at all and this has // left the cookie sync mechanism broken forever. if (dir_name == root->root_path) { try { auto st = getFileInformation(dir_name.c_str(), root->case_sensitive); if (st.ino != view->rootInode) { // If it still exists and the inode doesn't match, then we need // to force recrawl to make sure we're in sync. // We're lazily initializing the rootInode to 0 here, so we don't // need to do this the first time through (we're already crawling // everything in that case). if (view->rootInode != 0) { root->scheduleRecrawl( "root was replaced and we didn't get notified by the kernel"); return; } recursive = true; view->rootInode = st.ino; } } catch (const std::system_error& err) { handle_open_errno(root, dir, now, "getFileInformation", err.code()); markDirDeleted(view, dir, now, true); return; } } memcpy(path, dir_name.data(), dir_name.size()); path[dir_name.size()] = 0; w_log(W_LOG_DBG, "opendir(%s) recursive=%s\n", path, recursive ? "true" : "false"); /* Start watching and open the dir for crawling. * Whether we open the dir prior to watching or after is watcher specific, * so the operations are rolled together in our abstraction */ std::unique_ptr<watchman_dir_handle> osdir; try { osdir = watcher_->startWatchDir(root, dir, path); } catch (const std::system_error& err) { handle_open_errno(root, dir, now, "opendir", err.code()); markDirDeleted(view, dir, now, true); return; } if (dir->files.empty()) { // Pre-size our hash(es) if we can, so that we can avoid collisions // and re-hashing during initial crawl uint32_t num_dirs = 0; #ifndef _WIN32 struct stat st; int dfd = osdir->getFd(); if (dfd != -1 && fstat(dfd, &st) == 0) { num_dirs = (uint32_t)st.st_nlink; } #endif // st.st_nlink is usually number of dirs + 2 (., ..). // If it is less than 2 then it doesn't follow that convention. // We just pass it through for the dir size hint and the hash // table implementation will round that up to the next power of 2 apply_dir_size_hint( dir, num_dirs, uint32_t(root->config.getInt("hint_num_files_per_dir", 64))); } /* flag for delete detection */ for (auto& it : dir->files) { auto file = it.second.get(); if (file->exists) { file->maybe_deleted = true; } } try { while ((dirent = osdir->readDir()) != nullptr) { // Don't follow parent/self links if (dirent->d_name[0] == '.' && (!strcmp(dirent->d_name, ".") || !strcmp(dirent->d_name, ".."))) { continue; } // Queue it up for analysis if the file is newly existing w_string name(dirent->d_name, W_STRING_BYTE); file = dir->getChildFile(name); if (file) { file->maybe_deleted = false; } if (!file || !file->exists || stat_all || recursive) { auto full_path = dir->getFullPathToChild(name); w_log( W_LOG_DBG, "in crawler calling process_path on %s\n", full_path.c_str()); processPath( root, view, coll, full_path, now, ((recursive || !file || !file->exists) ? W_PENDING_RECURSIVE : 0), dirent); } } } catch (const std::system_error& exc) { log(ERR, "Error while reading dir ", path, ": ", exc.what(), ", re-adding to pending list to re-assess\n"); coll->add(path, now, 0); } osdir.reset(); // Anything still in maybe_deleted is actually deleted. // Arrange to re-process it shortly for (auto& it : dir->files) { auto file = it.second.get(); if (file->exists && (file->maybe_deleted || (file->stat.isDir() && recursive))) { coll->add( dir, file->getName().data(), now, recursive ? W_PENDING_RECURSIVE : 0); } } }
static DIR *kqueue_root_start_watch_dir(watchman_global_watcher_t watcher, w_root_t *root, struct watchman_dir *dir, struct timeval now, const char *path) { struct kqueue_root_state *state = root->watch; DIR *osdir; struct stat st, osdirst; struct kevent k; int newwd; unused_parameter(watcher); osdir = opendir_nofollow(path); if (!osdir) { handle_open_errno(root, dir, now, "opendir", errno, NULL); return NULL; } newwd = open(path, O_NOFOLLOW|O_EVTONLY|O_CLOEXEC); if (newwd == -1) { // directory got deleted between opendir and open handle_open_errno(root, dir, now, "open", errno, NULL); closedir(osdir); return NULL; } if (fstat(newwd, &st) == -1 || fstat(dirfd(osdir), &osdirst) == -1) { // whaaa? w_log(W_LOG_ERR, "fstat on opened dir %s failed: %s\n", path, strerror(errno)); w_root_schedule_recrawl(root, "fstat failed"); close(newwd); closedir(osdir); return NULL; } if (st.st_dev != osdirst.st_dev || st.st_ino != osdirst.st_ino) { // directory got replaced between opendir and open -- at this point its // parent's being watched, so we let filesystem events take care of it handle_open_errno(root, dir, now, "open", ENOTDIR, NULL); close(newwd); closedir(osdir); return NULL; } memset(&k, 0, sizeof(k)); EV_SET(&k, newwd, EVFILT_VNODE, EV_ADD|EV_CLEAR, NOTE_WRITE|NOTE_DELETE|NOTE_EXTEND|NOTE_RENAME, 0, SET_DIR_BIT(dir->path)); // Our mapping needs to be visible before we add it to the queue, // otherwise we can get a wakeup and not know what it is pthread_mutex_lock(&state->lock); w_ht_replace(state->name_to_fd, w_ht_ptr_val(dir->path), newwd); w_ht_replace(state->fd_to_name, newwd, w_ht_ptr_val(dir->path)); pthread_mutex_unlock(&state->lock); if (kevent(state->kq_fd, &k, 1, NULL, 0, 0)) { w_log(W_LOG_DBG, "kevent EV_ADD dir %s failed: %s", path, strerror(errno)); close(newwd); pthread_mutex_lock(&state->lock); w_ht_del(state->name_to_fd, w_ht_ptr_val(dir->path)); w_ht_del(state->fd_to_name, newwd); pthread_mutex_unlock(&state->lock); } else { w_log(W_LOG_DBG, "kevent dir %s -> %d\n", dir->path->buf, newwd); } return osdir; }