static w_query_expr *suffix_parser(w_query *query, json_t *term) { const char *ignore, *suffix; char *arg; w_string_t *str; int i, l; if (json_unpack(term, "[s,s]", &ignore, &suffix) != 0) { query->errmsg = strdup("must use [\"suffix\", \"suffixstring\"]"); return NULL; } arg = strdup(suffix); if (!arg) { query->errmsg = strdup("out of memory"); return NULL; } l = strlen_uint32(arg); for (i = 0; i < l; i++) { arg[i] = (char)tolower((uint8_t)arg[i]); } str = w_string_new_typed(arg, W_STRING_BYTE); free(arg); if (!str) { query->errmsg = strdup("out of memory"); return NULL; } return w_query_expr_new(eval_suffix, dispose_suffix, str); }
// Compute the basename of path, return that as a string w_string_t *w_string_new_basename_typed(const char *path, w_string_type_t type) { const char *base; base = path + strlen(path); while (base > path && !is_slash(base[-1])) { base--; } return w_string_new_typed(base, type); }
// Return the normalized (lowercase) filename suffix w_string_t *w_string_suffix(w_string_t *str) { int end; char name_buf[128]; char *buf; /* can't use libc strXXX functions because we may be operating * on a slice */ for (end = str->len - 1; end >= 0; end--) { if (str->buf[end] == '.') { if (str->len - end > sizeof(name_buf)) { // Too long return NULL; } buf = name_buf; end++; while ((unsigned)end < str->len) { *buf = (char)tolower((uint8_t)str->buf[end]); end++; buf++; } *buf = '\0'; return w_string_new_typed(name_buf, str->type); } else if (str->len - end >= sizeof(name_buf)) { // We haven't found the '.' yet but the suffix will never fit in our local // buffer return nullptr; } if (is_slash(str->buf[end])) { // No suffix return NULL; } } // Has no suffix return NULL; }
void FSEventsWatcher::FSEventsThread(const std::shared_ptr<w_root_t>& root) { CFFileDescriptorRef fdref; CFFileDescriptorContext fdctx; w_set_thread_name("fsevents %s", root->root_path.c_str()); { // Block until fsevents_root_start is waiting for our initialization auto wlock = items_.wlock(); attempt_resync_on_drop = root->config.getBool("fsevents_try_resync", true); memset(&fdctx, 0, sizeof(fdctx)); fdctx.info = root.get(); fdref = CFFileDescriptorCreate( nullptr, fse_pipe.read.fd(), true, fse_pipe_callback, &fdctx); CFFileDescriptorEnableCallBacks(fdref, kCFFileDescriptorReadCallBack); { CFRunLoopSourceRef fdsrc; fdsrc = CFFileDescriptorCreateRunLoopSource(nullptr, fdref, 0); if (!fdsrc) { root->failure_reason = w_string_new_typed( "CFFileDescriptorCreateRunLoopSource failed", W_STRING_UNICODE); goto done; } CFRunLoopAddSource(CFRunLoopGetCurrent(), fdsrc, kCFRunLoopDefaultMode); CFRelease(fdsrc); } stream = fse_stream_make( root, kFSEventStreamEventIdSinceNow, root->failure_reason); if (!stream) { goto done; } if (!FSEventStreamStart(stream->stream)) { root->failure_reason = w_string::printf( "FSEventStreamStart failed, look at your log file %s for " "lines mentioning FSEvents and see %s#fsevents for more information\n", log_name, cfg_get_trouble_url()); goto done; } // Signal to fsevents_root_start that we're done initializing fse_cond.notify_one(); } // Process the events stream until we get signalled to quit CFRunLoopRun(); done: if (stream) { delete stream; } if (fdref) { CFRelease(fdref); } w_log(W_LOG_DBG, "fse_thread done\n"); }
// Simulate static void bench_pending(void) { // These parameters give us 262140 items to track const size_t tree_depth = 7; const size_t num_files_per_dir = 8; const size_t num_dirs_per_dir = 4; w_string_t *root_name = w_string_new_typed("/some/path", W_STRING_BYTE); struct pending_list list; const size_t alloc_size = 280000; struct timeval start, end; list.pending = calloc(alloc_size, sizeof(struct watchman_pending_fs)); list.avail = list.pending; list.end = list.pending + alloc_size; // Build a list ordered from the root (top) down to the leaves. build_list(&list, root_name, tree_depth, num_files_per_dir, num_dirs_per_dir); diag("built list with %u items", list.avail - list.pending); // Benchmark insertion in top-down order. { struct watchman_pending_collection coll; struct watchman_pending_fs *item; size_t drained = 0; w_pending_coll_init(&coll); gettimeofday(&start, NULL); for (item = list.pending; item < list.avail; item++) { w_pending_coll_add(&coll, item->path, item->now, item->flags); } drained = process_items(&coll); gettimeofday(&end, NULL); diag("took %.3fs to insert %u items into pending coll", w_timeval_diff(start, end), drained); w_pending_coll_destroy(&coll); } // and now in reverse order; this is from the leaves of the filesystem // tree up to the root, or bottom-up. This simulates the workload of // a recursive delete of a filesystem tree. { struct watchman_pending_collection coll; struct watchman_pending_fs *item; size_t drained = 0; w_pending_coll_init(&coll); gettimeofday(&start, NULL); for (item = list.avail - 1; item >= list.pending; item--) { w_pending_coll_add(&coll, item->path, item->now, item->flags); } drained = process_items(&coll); gettimeofday(&end, NULL); diag("took %.3fs to reverse insert %u items into pending coll", w_timeval_diff(start, end), drained); w_pending_coll_destroy(&coll); } { struct watchman_pending_fs *item; for (item = list.pending; item < list.avail; item++) { w_string_delref(item->path); } free(list.pending); w_string_delref(root_name); } }
static void spawn_command( const std::shared_ptr<w_root_t>& root, struct watchman_trigger_command* cmd, w_query_res* res, struct w_clockspec* since_spec) { char **envp = NULL; uint32_t i = 0; int ret; char **argv = NULL; uint32_t env_size; posix_spawn_file_actions_t actions; posix_spawnattr_t attr; #ifndef _WIN32 sigset_t mask; #endif long arg_max; size_t argspace_remaining; bool file_overflow = false; int result_log_level; w_string_t *working_dir = NULL; #ifdef _WIN32 arg_max = 32*1024; #else arg_max = sysconf(_SC_ARG_MAX); #endif if (arg_max <= 0) { argspace_remaining = UINT_MAX; } else { argspace_remaining = (uint32_t)arg_max; } // Allow some misc working overhead argspace_remaining -= 32; // Record an overflow before we call prepare_stdin(), which mutates // and resizes the results to fit the specified limit. if (cmd->max_files_stdin > 0 && res->resultsArray.array().size() > cmd->max_files_stdin) { file_overflow = true; } auto stdin_file = prepare_stdin(cmd, res); if (!stdin_file) { w_log( W_LOG_ERR, "trigger %s:%s %s\n", root->root_path.c_str(), cmd->triggername.c_str(), strerror(errno)); return; } // Assumption: that only one thread will be executing on a given // cmd instance so that mutation of cmd->envht is safe. // This is guaranteed in the current architecture. // It is way too much of a hassle to try to recreate the clock value if it's // not a relative clock spec, and it's only going to happen on the first run // anyway, so just skip doing that entirely. if (since_spec && since_spec->tag == w_cs_clock) { w_envp_set_cstring( cmd->envht, "WATCHMAN_SINCE", since_spec->clock.position.toClockString().c_str()); } else { w_envp_unset(cmd->envht, "WATCHMAN_SINCE"); } w_envp_set_cstring( cmd->envht, "WATCHMAN_CLOCK", res->clockAtStartOfQuery.toClockString().c_str()); if (cmd->query->relative_root) { w_envp_set(cmd->envht, "WATCHMAN_RELATIVE_ROOT", cmd->query->relative_root); } else { w_envp_unset(cmd->envht, "WATCHMAN_RELATIVE_ROOT"); } // Compute args auto args = json_deep_copy(cmd->command); if (cmd->append_files) { // Measure how much space the base args take up for (i = 0; i < json_array_size(args); i++) { const char *ele = json_string_value(json_array_get(args, i)); argspace_remaining -= strlen(ele) + 1 + sizeof(char*); } // Dry run with env to compute space envp = w_envp_make_from_ht(cmd->envht, &env_size); free(envp); envp = NULL; argspace_remaining -= env_size; for (const auto& item : res->dedupedFileNames) { // also: NUL terminator and entry in argv uint32_t size = item.size() + 1 + sizeof(char*); if (argspace_remaining < size) { file_overflow = true; break; } argspace_remaining -= size; json_array_append_new(args, w_string_to_json(item)); } } argv = w_argv_copy_from_json(args, 0); args = nullptr; w_envp_set_bool(cmd->envht, "WATCHMAN_FILES_OVERFLOW", file_overflow); envp = w_envp_make_from_ht(cmd->envht, &env_size); posix_spawnattr_init(&attr); #ifndef _WIN32 sigemptyset(&mask); posix_spawnattr_setsigmask(&attr, &mask); #endif posix_spawnattr_setflags(&attr, POSIX_SPAWN_SETSIGMASK| #ifdef POSIX_SPAWN_CLOEXEC_DEFAULT // Darwin: close everything except what we put in file actions POSIX_SPAWN_CLOEXEC_DEFAULT| #endif POSIX_SPAWN_SETPGROUP); posix_spawn_file_actions_init(&actions); #ifndef _WIN32 posix_spawn_file_actions_adddup2( &actions, stdin_file->getFileDescriptor(), STDIN_FILENO); #else posix_spawn_file_actions_adddup2_handle_np( &actions, stdin_file->getWindowsHandle(), STDIN_FILENO); #endif if (cmd->stdout_name) { posix_spawn_file_actions_addopen(&actions, STDOUT_FILENO, cmd->stdout_name, cmd->stdout_flags, 0666); } else { posix_spawn_file_actions_adddup2(&actions, STDOUT_FILENO, STDOUT_FILENO); } if (cmd->stderr_name) { posix_spawn_file_actions_addopen(&actions, STDERR_FILENO, cmd->stderr_name, cmd->stderr_flags, 0666); } else { posix_spawn_file_actions_adddup2(&actions, STDERR_FILENO, STDERR_FILENO); } // Figure out the appropriate cwd { const char *cwd = NULL; working_dir = NULL; if (cmd->query->relative_root) { working_dir = cmd->query->relative_root; } else { working_dir = root->root_path; } w_string_addref(working_dir); json_unpack(cmd->definition, "{s:s}", "chdir", &cwd); if (cwd) { w_string_t *cwd_str = w_string_new_typed(cwd, W_STRING_BYTE); if (w_is_path_absolute_cstr(cwd)) { w_string_delref(working_dir); working_dir = cwd_str; } else { w_string_t *joined; joined = w_string_path_cat(working_dir, cwd_str); w_string_delref(cwd_str); w_string_delref(working_dir); working_dir = joined; } } w_log(W_LOG_DBG, "using %.*s for working dir\n", working_dir->len, working_dir->buf); } #ifndef _WIN32 // This mutex is present to avoid fighting over the cwd when multiple // triggers run at the same time. It doesn't coordinate with all // possible chdir() calls, but this is the only place that we do this // in the watchman server process. static std::mutex cwdMutex; { std::unique_lock<std::mutex> lock(cwdMutex); ignore_result(chdir(working_dir->buf)); #else posix_spawnattr_setcwd_np(&attr, working_dir->buf); #endif w_string_delref(working_dir); working_dir = nullptr; ret = posix_spawnp(&cmd->current_proc, argv[0], &actions, &attr, argv, envp); if (ret != 0) { // On Darwin (at least), posix_spawn can fail but will still populate the // pid. Since we use the pid to gate future spawns, we need to ensure // that we clear out the pid on failure, otherwise the trigger would be // effectively disabled for the rest of the watch lifetime cmd->current_proc = 0; } #ifndef _WIN32 ignore_result(chdir("/")); } #endif // If failed, we want to make sure we log enough info to figure out why result_log_level = res == 0 ? W_LOG_DBG : W_LOG_ERR; w_log(result_log_level, "posix_spawnp: %s\n", cmd->triggername.c_str()); for (i = 0; argv[i]; i++) { w_log(result_log_level, "argv[%d] %s\n", i, argv[i]); } for (i = 0; envp[i]; i++) { w_log(result_log_level, "envp[%d] %s\n", i, envp[i]); } w_log( result_log_level, "trigger %s:%s pid=%d ret=%d %s\n", root->root_path.c_str(), cmd->triggername.c_str(), (int)cmd->current_proc, ret, strerror(ret)); free(argv); free(envp); posix_spawnattr_destroy(&attr); posix_spawn_file_actions_destroy(&actions); }
static void process_inotify_event( w_root_t *root, struct watchman_pending_collection *coll, struct inotify_event *ine, struct timeval now) { struct inot_root_state *state = root->watch; char flags_label[128]; w_expand_flags(inflags, ine->mask, flags_label, sizeof(flags_label)); w_log(W_LOG_DBG, "notify: wd=%d mask=0x%x %s %s\n", ine->wd, ine->mask, flags_label, ine->len > 0 ? ine->name : ""); if (ine->wd == -1 && (ine->mask & IN_Q_OVERFLOW)) { /* we missed something, will need to re-crawl */ w_root_schedule_recrawl(root, "IN_Q_OVERFLOW"); } else if (ine->wd != -1) { w_string_t *dir_name = NULL; w_string_t *name = NULL; char buf[WATCHMAN_NAME_MAX]; int pending_flags = W_PENDING_VIA_NOTIFY; pthread_mutex_lock(&state->lock); dir_name = w_ht_val_ptr(w_ht_get(state->wd_to_name, ine->wd)); if (dir_name) { w_string_addref(dir_name); } pthread_mutex_unlock(&state->lock); if (dir_name) { if (ine->len > 0) { snprintf(buf, sizeof(buf), "%.*s/%s", dir_name->len, dir_name->buf, ine->name); name = w_string_new_typed(buf, W_STRING_BYTE); } else { name = dir_name; w_string_addref(name); } } if (ine->len > 0 && (ine->mask & (IN_MOVED_FROM|IN_ISDIR)) == (IN_MOVED_FROM|IN_ISDIR)) { struct pending_move mv; // record this as a pending move, so that we can automatically // watch the target when we get the other side of it. mv.created = now.tv_sec; mv.name = name; pthread_mutex_lock(&state->lock); if (!w_ht_replace(state->move_map, ine->cookie, w_ht_ptr_val(&mv))) { w_log(W_LOG_FATAL, "failed to store %" PRIx32 " -> %s in move map\n", ine->cookie, name->buf); } pthread_mutex_unlock(&state->lock); w_log(W_LOG_DBG, "recording move_from %" PRIx32 " %s\n", ine->cookie, name->buf); } if (ine->len > 0 && (ine->mask & (IN_MOVED_TO|IN_ISDIR)) == (IN_MOVED_FROM|IN_ISDIR)) { struct pending_move *old; pthread_mutex_lock(&state->lock); old = w_ht_val_ptr(w_ht_get(state->move_map, ine->cookie)); if (old) { int wd = inotify_add_watch(state->infd, name->buf, WATCHMAN_INOTIFY_MASK); if (wd == -1) { if (errno == ENOSPC || errno == ENOMEM) { // Limits exceeded, no recovery from our perspective set_poison_state(root, name, now, "inotify-add-watch", errno, inot_strerror(errno)); } else { w_log(W_LOG_DBG, "add_watch: %s %s\n", name->buf, inot_strerror(errno)); } } else { w_log(W_LOG_DBG, "moved %s -> %s\n", old->name->buf, name->buf); w_ht_replace(state->wd_to_name, wd, w_ht_ptr_val(name)); } } else { w_log(W_LOG_DBG, "move: cookie=%" PRIx32 " not found in move map %s\n", ine->cookie, name->buf); } pthread_mutex_unlock(&state->lock); } if (dir_name) { if ((ine->mask & (IN_UNMOUNT|IN_IGNORED|IN_DELETE_SELF|IN_MOVE_SELF))) { w_string_t *pname; if (w_string_equal(root->root_path, name)) { w_log(W_LOG_ERR, "root dir %s has been (re)moved, canceling watch\n", root->root_path->buf); w_string_delref(name); w_string_delref(dir_name); w_root_cancel(root); return; } // We need to examine the parent and crawl down pname = w_string_dirname(name); w_log(W_LOG_DBG, "mask=%x, focus on parent: %.*s\n", ine->mask, pname->len, pname->buf); w_string_delref(name); name = pname; pending_flags |= W_PENDING_RECURSIVE; } if (ine->mask & (IN_CREATE|IN_DELETE)) { pending_flags |= W_PENDING_RECURSIVE; } w_log(W_LOG_DBG, "add_pending for inotify mask=%x %.*s\n", ine->mask, name->len, name->buf); w_pending_coll_add(coll, name, now, pending_flags); w_string_delref(name); // The kernel removed the wd -> name mapping, so let's update // our state here also if ((ine->mask & IN_IGNORED) != 0) { w_log(W_LOG_DBG, "mask=%x: remove watch %d %.*s\n", ine->mask, ine->wd, dir_name->len, dir_name->buf); pthread_mutex_lock(&state->lock); w_ht_del(state->wd_to_name, ine->wd); pthread_mutex_unlock(&state->lock); } w_string_delref(dir_name); } else if ((ine->mask & (IN_MOVE_SELF|IN_IGNORED)) == 0) { // If we can't resolve the dir, and this isn't notification // that it has gone away, then we want to recrawl to fix // up our state. w_log(W_LOG_ERR, "wanted dir %d for mask %x but not found %.*s\n", ine->wd, ine->mask, ine->len, ine->name); w_root_schedule_recrawl(root, "dir missing from internal state"); } } }
struct watchman_trigger_command *w_build_trigger_from_def( w_root_t *root, json_t *trig, char **errmsg) { struct watchman_trigger_command *cmd; json_t *ele, *query, *relative_root; json_int_t jint; const char *name = NULL; cmd = calloc(1, sizeof(*cmd)); if (!cmd) { *errmsg = strdup("no memory"); return NULL; } cmd->definition = trig; json_incref(cmd->definition); query = json_pack("{s:O}", "expression", json_object_get(cmd->definition, "expression")); relative_root = json_object_get(cmd->definition, "relative_root"); if (relative_root) { json_object_set_nocheck(query, "relative_root", relative_root); } cmd->query = w_query_parse(root, query, errmsg); json_decref(query); if (!cmd->query) { w_trigger_command_free(cmd); return NULL; } json_unpack(trig, "{s:u}", "name", &name); if (!name) { *errmsg = strdup("invalid or missing name"); w_trigger_command_free(cmd); return NULL; } cmd->triggername = w_string_new_typed(name, W_STRING_UNICODE); cmd->command = json_object_get(trig, "command"); if (cmd->command) { json_incref(cmd->command); } if (!cmd->command || !json_is_array(cmd->command) || !json_array_size(cmd->command)) { *errmsg = strdup("invalid command array"); w_trigger_command_free(cmd); return NULL; } json_unpack(trig, "{s:b}", "append_files", &cmd->append_files); ele = json_object_get(trig, "stdin"); if (!ele) { cmd->stdin_style = input_dev_null; } else if (json_is_array(ele)) { cmd->stdin_style = input_json; if (!parse_field_list(ele, &cmd->field_list, errmsg)) { w_trigger_command_free(cmd); return NULL; } } else if (json_is_string(ele)) { const char *str = json_string_value(ele); if (!strcmp(str, "/dev/null")) { cmd->stdin_style = input_dev_null; } else if (!strcmp(str, "NAME_PER_LINE")) { cmd->stdin_style = input_name_list; } else { ignore_result(asprintf(errmsg, "invalid stdin value %s", str)); w_trigger_command_free(cmd); return NULL; } } else { *errmsg = strdup("invalid value for stdin"); w_trigger_command_free(cmd); return NULL; } jint = 0; // unlimited unless specified json_unpack(trig, "{s:I}", "max_files_stdin", &jint); if (jint < 0) { *errmsg = strdup("max_files_stdin must be >= 0"); w_trigger_command_free(cmd); return NULL; } cmd->max_files_stdin = (uint32_t)jint; json_unpack(trig, "{s:s}", "stdout", &cmd->stdout_name); json_unpack(trig, "{s:s}", "stderr", &cmd->stderr_name); if (!parse_redirection(&cmd->stdout_name, &cmd->stdout_flags, "stdout", errmsg)) { w_trigger_command_free(cmd); return NULL; } if (!parse_redirection(&cmd->stderr_name, &cmd->stderr_flags, "stderr", errmsg)) { w_trigger_command_free(cmd); return NULL; } // Copy current environment cmd->envht = w_envp_make_ht(); // Set some standard vars w_envp_set(cmd->envht, "WATCHMAN_ROOT", root->root_path); w_envp_set_cstring(cmd->envht, "WATCHMAN_SOCK", get_sock_name()); w_envp_set(cmd->envht, "WATCHMAN_TRIGGER", cmd->triggername); return cmd; }