static w_query_expr *suffix_parser(w_query *query, json_t *term)
{
const char *ignore, *suffix;
char *arg;
w_string_t *str;
int i, l;
if (json_unpack(term, "[s,s]", &ignore, &suffix) != 0) {
query->errmsg = strdup("must use [\"suffix\", \"suffixstring\"]");
return NULL;
}
arg = strdup(suffix);
if (!arg) {
query->errmsg = strdup("out of memory");
return NULL;
}
l = strlen_uint32(arg);
for (i = 0; i < l; i++) {
arg[i] = (char)tolower((uint8_t)arg[i]);
}
str = w_string_new_typed(arg, W_STRING_BYTE);
free(arg);
if (!str) {
query->errmsg = strdup("out of memory");
return NULL;
}
return w_query_expr_new(eval_suffix, dispose_suffix, str);
}
// Compute the basename of path, return that as a string
w_string_t *w_string_new_basename_typed(const char *path,
w_string_type_t type) {
const char *base;
base = path + strlen(path);
while (base > path && !is_slash(base[-1])) {
base--;
}
return w_string_new_typed(base, type);
}
// Return the normalized (lowercase) filename suffix
w_string_t *w_string_suffix(w_string_t *str)
{
int end;
char name_buf[128];
char *buf;
/* can't use libc strXXX functions because we may be operating
* on a slice */
for (end = str->len - 1; end >= 0; end--) {
if (str->buf[end] == '.') {
if (str->len - end > sizeof(name_buf)) {
// Too long
return NULL;
}
buf = name_buf;
end++;
while ((unsigned)end < str->len) {
*buf = (char)tolower((uint8_t)str->buf[end]);
end++;
buf++;
}
*buf = '\0';
return w_string_new_typed(name_buf, str->type);
} else if (str->len - end >= sizeof(name_buf)) {
// We haven't found the '.' yet but the suffix will never fit in our local
// buffer
return nullptr;
}
if (is_slash(str->buf[end])) {
// No suffix
return NULL;
}
}
// Has no suffix
return NULL;
}
void FSEventsWatcher::FSEventsThread(const std::shared_ptr<w_root_t>& root) {
CFFileDescriptorRef fdref;
CFFileDescriptorContext fdctx;
w_set_thread_name("fsevents %s", root->root_path.c_str());
{
// Block until fsevents_root_start is waiting for our initialization
auto wlock = items_.wlock();
attempt_resync_on_drop = root->config.getBool("fsevents_try_resync", true);
memset(&fdctx, 0, sizeof(fdctx));
fdctx.info = root.get();
fdref = CFFileDescriptorCreate(
nullptr, fse_pipe.read.fd(), true, fse_pipe_callback, &fdctx);
CFFileDescriptorEnableCallBacks(fdref, kCFFileDescriptorReadCallBack);
{
CFRunLoopSourceRef fdsrc;
fdsrc = CFFileDescriptorCreateRunLoopSource(nullptr, fdref, 0);
if (!fdsrc) {
root->failure_reason = w_string_new_typed(
"CFFileDescriptorCreateRunLoopSource failed", W_STRING_UNICODE);
goto done;
}
CFRunLoopAddSource(CFRunLoopGetCurrent(), fdsrc, kCFRunLoopDefaultMode);
CFRelease(fdsrc);
}
stream = fse_stream_make(
root, kFSEventStreamEventIdSinceNow, root->failure_reason);
if (!stream) {
goto done;
}
if (!FSEventStreamStart(stream->stream)) {
root->failure_reason = w_string::printf(
"FSEventStreamStart failed, look at your log file %s for "
"lines mentioning FSEvents and see %s#fsevents for more information\n",
log_name,
cfg_get_trouble_url());
goto done;
}
// Signal to fsevents_root_start that we're done initializing
fse_cond.notify_one();
}
// Process the events stream until we get signalled to quit
CFRunLoopRun();
done:
if (stream) {
delete stream;
}
if (fdref) {
CFRelease(fdref);
}
w_log(W_LOG_DBG, "fse_thread done\n");
}
// Simulate
static void bench_pending(void) {
// These parameters give us 262140 items to track
const size_t tree_depth = 7;
const size_t num_files_per_dir = 8;
const size_t num_dirs_per_dir = 4;
w_string_t *root_name = w_string_new_typed("/some/path", W_STRING_BYTE);
struct pending_list list;
const size_t alloc_size = 280000;
struct timeval start, end;
list.pending = calloc(alloc_size, sizeof(struct watchman_pending_fs));
list.avail = list.pending;
list.end = list.pending + alloc_size;
// Build a list ordered from the root (top) down to the leaves.
build_list(&list, root_name, tree_depth, num_files_per_dir, num_dirs_per_dir);
diag("built list with %u items", list.avail - list.pending);
// Benchmark insertion in top-down order.
{
struct watchman_pending_collection coll;
struct watchman_pending_fs *item;
size_t drained = 0;
w_pending_coll_init(&coll);
gettimeofday(&start, NULL);
for (item = list.pending; item < list.avail; item++) {
w_pending_coll_add(&coll, item->path, item->now, item->flags);
}
drained = process_items(&coll);
gettimeofday(&end, NULL);
diag("took %.3fs to insert %u items into pending coll",
w_timeval_diff(start, end), drained);
w_pending_coll_destroy(&coll);
}
// and now in reverse order; this is from the leaves of the filesystem
// tree up to the root, or bottom-up. This simulates the workload of
// a recursive delete of a filesystem tree.
{
struct watchman_pending_collection coll;
struct watchman_pending_fs *item;
size_t drained = 0;
w_pending_coll_init(&coll);
gettimeofday(&start, NULL);
for (item = list.avail - 1; item >= list.pending; item--) {
w_pending_coll_add(&coll, item->path, item->now, item->flags);
}
drained = process_items(&coll);
gettimeofday(&end, NULL);
diag("took %.3fs to reverse insert %u items into pending coll",
w_timeval_diff(start, end), drained);
w_pending_coll_destroy(&coll);
}
{
struct watchman_pending_fs *item;
for (item = list.pending; item < list.avail; item++) {
w_string_delref(item->path);
}
free(list.pending);
w_string_delref(root_name);
}
}
static void spawn_command(
const std::shared_ptr<w_root_t>& root,
struct watchman_trigger_command* cmd,
w_query_res* res,
struct w_clockspec* since_spec) {
char **envp = NULL;
uint32_t i = 0;
int ret;
char **argv = NULL;
uint32_t env_size;
posix_spawn_file_actions_t actions;
posix_spawnattr_t attr;
#ifndef _WIN32
sigset_t mask;
#endif
long arg_max;
size_t argspace_remaining;
bool file_overflow = false;
int result_log_level;
w_string_t *working_dir = NULL;
#ifdef _WIN32
arg_max = 32*1024;
#else
arg_max = sysconf(_SC_ARG_MAX);
#endif
if (arg_max <= 0) {
argspace_remaining = UINT_MAX;
} else {
argspace_remaining = (uint32_t)arg_max;
}
// Allow some misc working overhead
argspace_remaining -= 32;
// Record an overflow before we call prepare_stdin(), which mutates
// and resizes the results to fit the specified limit.
if (cmd->max_files_stdin > 0 &&
res->resultsArray.array().size() > cmd->max_files_stdin) {
file_overflow = true;
}
auto stdin_file = prepare_stdin(cmd, res);
if (!stdin_file) {
w_log(
W_LOG_ERR,
"trigger %s:%s %s\n",
root->root_path.c_str(),
cmd->triggername.c_str(),
strerror(errno));
return;
}
// Assumption: that only one thread will be executing on a given
// cmd instance so that mutation of cmd->envht is safe.
// This is guaranteed in the current architecture.
// It is way too much of a hassle to try to recreate the clock value if it's
// not a relative clock spec, and it's only going to happen on the first run
// anyway, so just skip doing that entirely.
if (since_spec && since_spec->tag == w_cs_clock) {
w_envp_set_cstring(
cmd->envht,
"WATCHMAN_SINCE",
since_spec->clock.position.toClockString().c_str());
} else {
w_envp_unset(cmd->envht, "WATCHMAN_SINCE");
}
w_envp_set_cstring(
cmd->envht,
"WATCHMAN_CLOCK",
res->clockAtStartOfQuery.toClockString().c_str());
if (cmd->query->relative_root) {
w_envp_set(cmd->envht, "WATCHMAN_RELATIVE_ROOT", cmd->query->relative_root);
} else {
w_envp_unset(cmd->envht, "WATCHMAN_RELATIVE_ROOT");
}
// Compute args
auto args = json_deep_copy(cmd->command);
if (cmd->append_files) {
// Measure how much space the base args take up
for (i = 0; i < json_array_size(args); i++) {
const char *ele = json_string_value(json_array_get(args, i));
argspace_remaining -= strlen(ele) + 1 + sizeof(char*);
}
// Dry run with env to compute space
envp = w_envp_make_from_ht(cmd->envht, &env_size);
free(envp);
envp = NULL;
argspace_remaining -= env_size;
for (const auto& item : res->dedupedFileNames) {
// also: NUL terminator and entry in argv
uint32_t size = item.size() + 1 + sizeof(char*);
if (argspace_remaining < size) {
file_overflow = true;
break;
}
argspace_remaining -= size;
json_array_append_new(args, w_string_to_json(item));
}
}
argv = w_argv_copy_from_json(args, 0);
args = nullptr;
w_envp_set_bool(cmd->envht, "WATCHMAN_FILES_OVERFLOW", file_overflow);
envp = w_envp_make_from_ht(cmd->envht, &env_size);
posix_spawnattr_init(&attr);
#ifndef _WIN32
sigemptyset(&mask);
posix_spawnattr_setsigmask(&attr, &mask);
#endif
posix_spawnattr_setflags(&attr,
POSIX_SPAWN_SETSIGMASK|
#ifdef POSIX_SPAWN_CLOEXEC_DEFAULT
// Darwin: close everything except what we put in file actions
POSIX_SPAWN_CLOEXEC_DEFAULT|
#endif
POSIX_SPAWN_SETPGROUP);
posix_spawn_file_actions_init(&actions);
#ifndef _WIN32
posix_spawn_file_actions_adddup2(
&actions, stdin_file->getFileDescriptor(), STDIN_FILENO);
#else
posix_spawn_file_actions_adddup2_handle_np(
&actions, stdin_file->getWindowsHandle(), STDIN_FILENO);
#endif
if (cmd->stdout_name) {
posix_spawn_file_actions_addopen(&actions, STDOUT_FILENO,
cmd->stdout_name, cmd->stdout_flags, 0666);
} else {
posix_spawn_file_actions_adddup2(&actions, STDOUT_FILENO, STDOUT_FILENO);
}
if (cmd->stderr_name) {
posix_spawn_file_actions_addopen(&actions, STDERR_FILENO,
cmd->stderr_name, cmd->stderr_flags, 0666);
} else {
posix_spawn_file_actions_adddup2(&actions, STDERR_FILENO, STDERR_FILENO);
}
// Figure out the appropriate cwd
{
const char *cwd = NULL;
working_dir = NULL;
if (cmd->query->relative_root) {
working_dir = cmd->query->relative_root;
} else {
working_dir = root->root_path;
}
w_string_addref(working_dir);
json_unpack(cmd->definition, "{s:s}", "chdir", &cwd);
if (cwd) {
w_string_t *cwd_str = w_string_new_typed(cwd, W_STRING_BYTE);
if (w_is_path_absolute_cstr(cwd)) {
w_string_delref(working_dir);
working_dir = cwd_str;
} else {
w_string_t *joined;
joined = w_string_path_cat(working_dir, cwd_str);
w_string_delref(cwd_str);
w_string_delref(working_dir);
working_dir = joined;
}
}
w_log(W_LOG_DBG, "using %.*s for working dir\n", working_dir->len,
working_dir->buf);
}
#ifndef _WIN32
// This mutex is present to avoid fighting over the cwd when multiple
// triggers run at the same time. It doesn't coordinate with all
// possible chdir() calls, but this is the only place that we do this
// in the watchman server process.
static std::mutex cwdMutex;
{
std::unique_lock<std::mutex> lock(cwdMutex);
ignore_result(chdir(working_dir->buf));
#else
posix_spawnattr_setcwd_np(&attr, working_dir->buf);
#endif
w_string_delref(working_dir);
working_dir = nullptr;
ret =
posix_spawnp(&cmd->current_proc, argv[0], &actions, &attr, argv, envp);
if (ret != 0) {
// On Darwin (at least), posix_spawn can fail but will still populate the
// pid. Since we use the pid to gate future spawns, we need to ensure
// that we clear out the pid on failure, otherwise the trigger would be
// effectively disabled for the rest of the watch lifetime
cmd->current_proc = 0;
}
#ifndef _WIN32
ignore_result(chdir("/"));
}
#endif
// If failed, we want to make sure we log enough info to figure out why
result_log_level = res == 0 ? W_LOG_DBG : W_LOG_ERR;
w_log(result_log_level, "posix_spawnp: %s\n", cmd->triggername.c_str());
for (i = 0; argv[i]; i++) {
w_log(result_log_level, "argv[%d] %s\n", i, argv[i]);
}
for (i = 0; envp[i]; i++) {
w_log(result_log_level, "envp[%d] %s\n", i, envp[i]);
}
w_log(
result_log_level,
"trigger %s:%s pid=%d ret=%d %s\n",
root->root_path.c_str(),
cmd->triggername.c_str(),
(int)cmd->current_proc,
ret,
strerror(ret));
free(argv);
free(envp);
posix_spawnattr_destroy(&attr);
posix_spawn_file_actions_destroy(&actions);
}
static void process_inotify_event(
w_root_t *root,
struct watchman_pending_collection *coll,
struct inotify_event *ine,
struct timeval now)
{
struct inot_root_state *state = root->watch;
char flags_label[128];
w_expand_flags(inflags, ine->mask, flags_label, sizeof(flags_label));
w_log(W_LOG_DBG, "notify: wd=%d mask=0x%x %s %s\n", ine->wd, ine->mask,
flags_label, ine->len > 0 ? ine->name : "");
if (ine->wd == -1 && (ine->mask & IN_Q_OVERFLOW)) {
/* we missed something, will need to re-crawl */
w_root_schedule_recrawl(root, "IN_Q_OVERFLOW");
} else if (ine->wd != -1) {
w_string_t *dir_name = NULL;
w_string_t *name = NULL;
char buf[WATCHMAN_NAME_MAX];
int pending_flags = W_PENDING_VIA_NOTIFY;
pthread_mutex_lock(&state->lock);
dir_name = w_ht_val_ptr(w_ht_get(state->wd_to_name, ine->wd));
if (dir_name) {
w_string_addref(dir_name);
}
pthread_mutex_unlock(&state->lock);
if (dir_name) {
if (ine->len > 0) {
snprintf(buf, sizeof(buf), "%.*s/%s",
dir_name->len, dir_name->buf,
ine->name);
name = w_string_new_typed(buf, W_STRING_BYTE);
} else {
name = dir_name;
w_string_addref(name);
}
}
if (ine->len > 0 && (ine->mask & (IN_MOVED_FROM|IN_ISDIR))
== (IN_MOVED_FROM|IN_ISDIR)) {
struct pending_move mv;
// record this as a pending move, so that we can automatically
// watch the target when we get the other side of it.
mv.created = now.tv_sec;
mv.name = name;
pthread_mutex_lock(&state->lock);
if (!w_ht_replace(state->move_map, ine->cookie, w_ht_ptr_val(&mv))) {
w_log(W_LOG_FATAL,
"failed to store %" PRIx32 " -> %s in move map\n",
ine->cookie, name->buf);
}
pthread_mutex_unlock(&state->lock);
w_log(W_LOG_DBG,
"recording move_from %" PRIx32 " %s\n", ine->cookie,
name->buf);
}
if (ine->len > 0 && (ine->mask & (IN_MOVED_TO|IN_ISDIR))
== (IN_MOVED_FROM|IN_ISDIR)) {
struct pending_move *old;
pthread_mutex_lock(&state->lock);
old = w_ht_val_ptr(w_ht_get(state->move_map, ine->cookie));
if (old) {
int wd = inotify_add_watch(state->infd, name->buf,
WATCHMAN_INOTIFY_MASK);
if (wd == -1) {
if (errno == ENOSPC || errno == ENOMEM) {
// Limits exceeded, no recovery from our perspective
set_poison_state(root, name, now, "inotify-add-watch", errno,
inot_strerror(errno));
} else {
w_log(W_LOG_DBG, "add_watch: %s %s\n",
name->buf, inot_strerror(errno));
}
} else {
w_log(W_LOG_DBG, "moved %s -> %s\n", old->name->buf, name->buf);
w_ht_replace(state->wd_to_name, wd, w_ht_ptr_val(name));
}
} else {
w_log(W_LOG_DBG, "move: cookie=%" PRIx32 " not found in move map %s\n",
ine->cookie, name->buf);
}
pthread_mutex_unlock(&state->lock);
}
if (dir_name) {
if ((ine->mask & (IN_UNMOUNT|IN_IGNORED|IN_DELETE_SELF|IN_MOVE_SELF))) {
w_string_t *pname;
if (w_string_equal(root->root_path, name)) {
w_log(W_LOG_ERR,
"root dir %s has been (re)moved, canceling watch\n",
root->root_path->buf);
w_string_delref(name);
w_string_delref(dir_name);
w_root_cancel(root);
return;
}
// We need to examine the parent and crawl down
pname = w_string_dirname(name);
w_log(W_LOG_DBG, "mask=%x, focus on parent: %.*s\n",
ine->mask, pname->len, pname->buf);
w_string_delref(name);
name = pname;
pending_flags |= W_PENDING_RECURSIVE;
}
if (ine->mask & (IN_CREATE|IN_DELETE)) {
pending_flags |= W_PENDING_RECURSIVE;
}
w_log(W_LOG_DBG, "add_pending for inotify mask=%x %.*s\n",
ine->mask, name->len, name->buf);
w_pending_coll_add(coll, name, now, pending_flags);
w_string_delref(name);
// The kernel removed the wd -> name mapping, so let's update
// our state here also
if ((ine->mask & IN_IGNORED) != 0) {
w_log(W_LOG_DBG, "mask=%x: remove watch %d %.*s\n", ine->mask,
ine->wd, dir_name->len, dir_name->buf);
pthread_mutex_lock(&state->lock);
w_ht_del(state->wd_to_name, ine->wd);
pthread_mutex_unlock(&state->lock);
}
w_string_delref(dir_name);
} else if ((ine->mask & (IN_MOVE_SELF|IN_IGNORED)) == 0) {
// If we can't resolve the dir, and this isn't notification
// that it has gone away, then we want to recrawl to fix
// up our state.
w_log(W_LOG_ERR, "wanted dir %d for mask %x but not found %.*s\n",
ine->wd, ine->mask, ine->len, ine->name);
w_root_schedule_recrawl(root, "dir missing from internal state");
}
}
}
struct watchman_trigger_command *w_build_trigger_from_def(
w_root_t *root, json_t *trig, char **errmsg)
{
struct watchman_trigger_command *cmd;
json_t *ele, *query, *relative_root;
json_int_t jint;
const char *name = NULL;
cmd = calloc(1, sizeof(*cmd));
if (!cmd) {
*errmsg = strdup("no memory");
return NULL;
}
cmd->definition = trig;
json_incref(cmd->definition);
query = json_pack("{s:O}", "expression",
json_object_get(cmd->definition, "expression"));
relative_root = json_object_get(cmd->definition, "relative_root");
if (relative_root) {
json_object_set_nocheck(query, "relative_root", relative_root);
}
cmd->query = w_query_parse(root, query, errmsg);
json_decref(query);
if (!cmd->query) {
w_trigger_command_free(cmd);
return NULL;
}
json_unpack(trig, "{s:u}", "name", &name);
if (!name) {
*errmsg = strdup("invalid or missing name");
w_trigger_command_free(cmd);
return NULL;
}
cmd->triggername = w_string_new_typed(name, W_STRING_UNICODE);
cmd->command = json_object_get(trig, "command");
if (cmd->command) {
json_incref(cmd->command);
}
if (!cmd->command || !json_is_array(cmd->command) ||
!json_array_size(cmd->command)) {
*errmsg = strdup("invalid command array");
w_trigger_command_free(cmd);
return NULL;
}
json_unpack(trig, "{s:b}", "append_files", &cmd->append_files);
ele = json_object_get(trig, "stdin");
if (!ele) {
cmd->stdin_style = input_dev_null;
} else if (json_is_array(ele)) {
cmd->stdin_style = input_json;
if (!parse_field_list(ele, &cmd->field_list, errmsg)) {
w_trigger_command_free(cmd);
return NULL;
}
} else if (json_is_string(ele)) {
const char *str = json_string_value(ele);
if (!strcmp(str, "/dev/null")) {
cmd->stdin_style = input_dev_null;
} else if (!strcmp(str, "NAME_PER_LINE")) {
cmd->stdin_style = input_name_list;
} else {
ignore_result(asprintf(errmsg, "invalid stdin value %s", str));
w_trigger_command_free(cmd);
return NULL;
}
} else {
*errmsg = strdup("invalid value for stdin");
w_trigger_command_free(cmd);
return NULL;
}
jint = 0; // unlimited unless specified
json_unpack(trig, "{s:I}", "max_files_stdin", &jint);
if (jint < 0) {
*errmsg = strdup("max_files_stdin must be >= 0");
w_trigger_command_free(cmd);
return NULL;
}
cmd->max_files_stdin = (uint32_t)jint;
json_unpack(trig, "{s:s}", "stdout", &cmd->stdout_name);
json_unpack(trig, "{s:s}", "stderr", &cmd->stderr_name);
if (!parse_redirection(&cmd->stdout_name, &cmd->stdout_flags,
"stdout", errmsg)) {
w_trigger_command_free(cmd);
return NULL;
}
if (!parse_redirection(&cmd->stderr_name, &cmd->stderr_flags,
"stderr", errmsg)) {
w_trigger_command_free(cmd);
return NULL;
}
// Copy current environment
cmd->envht = w_envp_make_ht();
// Set some standard vars
w_envp_set(cmd->envht, "WATCHMAN_ROOT", root->root_path);
w_envp_set_cstring(cmd->envht, "WATCHMAN_SOCK", get_sock_name());
w_envp_set(cmd->envht, "WATCHMAN_TRIGGER", cmd->triggername);
return cmd;
}
