void w_envp_set_bool(
std::unordered_map<w_string, w_string>& envht,
const char* key,
bool val) {
if (val) {
w_envp_set_cstring(envht, key, "true");
} else {
w_envp_unset(envht, key);
}
}
static void spawn_command(w_root_t *root,
struct watchman_trigger_command *cmd,
w_query_res *res,
struct w_clockspec *since_spec)
{
char **envp = NULL;
uint32_t i = 0;
int ret;
int stdin_fd = -1;
json_t *args;
char **argv = NULL;
uint32_t env_size;
posix_spawn_file_actions_t actions;
posix_spawnattr_t attr;
sigset_t mask;
long arg_max;
uint32_t argspace_remaining;
bool file_overflow = false;
int result_log_level;
char clockbuf[128];
const char *cwd = NULL;
arg_max = sysconf(_SC_ARG_MAX);
if (arg_max <= 0) {
argspace_remaining = UINT_MAX;
} else {
argspace_remaining = (uint32_t)arg_max;
}
// Allow some misc working overhead
argspace_remaining -= 32;
stdin_fd = prepare_stdin(cmd, res);
// Assumption: that only one thread will be executing on a given
// cmd instance so that mutation of cmd->envht is safe.
// This is guaranteed in the current architecture.
if (cmd->max_files_stdin > 0 && res->num_results > cmd->max_files_stdin) {
file_overflow = true;
}
// It is way too much of a hassle to try to recreate the clock value if it's
// not a relative clock spec, and it's only going to happen on the first run
// anyway, so just skip doing that entirely.
if (since_spec && since_spec->tag == w_cs_clock &&
clock_id_string(since_spec->clock.root_number, since_spec->clock.ticks,
clockbuf, sizeof(clockbuf))) {
w_envp_set_cstring(cmd->envht, "WATCHMAN_SINCE", clockbuf);
} else {
w_envp_unset(cmd->envht, "WATCHMAN_SINCE");
}
if (clock_id_string(res->root_number, res->ticks,
clockbuf, sizeof(clockbuf))) {
w_envp_set_cstring(cmd->envht, "WATCHMAN_CLOCK", clockbuf);
} else {
w_envp_unset(cmd->envht, "WATCHMAN_CLOCK");
}
if (cmd->query->relative_root) {
w_envp_set(cmd->envht, "WATCHMAN_RELATIVE_ROOT", cmd->query->relative_root);
} else {
w_envp_unset(cmd->envht, "WATCHMAN_RELATIVE_ROOT");
}
// Compute args
args = json_deep_copy(cmd->command);
if (cmd->append_files) {
// Measure how much space the base args take up
for (i = 0; i < json_array_size(args); i++) {
const char *ele = json_string_value(json_array_get(args, i));
argspace_remaining -= strlen(ele) + 1 + sizeof(char*);
}
// Dry run with env to compute space
envp = w_envp_make_from_ht(cmd->envht, &env_size);
free(envp);
envp = NULL;
argspace_remaining -= env_size;
for (i = 0; i < res->num_results; i++) {
// also: NUL terminator and entry in argv
uint32_t size = res->results[i].relname->len + 1 + sizeof(char*);
if (argspace_remaining < size) {
file_overflow = true;
break;
}
argspace_remaining -= size;
json_array_append_new(
args,
json_string_nocheck(res->results[i].relname->buf)
);
}
}
argv = w_argv_copy_from_json(args, 0);
json_decref(args);
args = NULL;
w_envp_set_bool(cmd->envht, "WATCHMAN_FILES_OVERFLOW", file_overflow);
envp = w_envp_make_from_ht(cmd->envht, &env_size);
posix_spawnattr_init(&attr);
sigemptyset(&mask);
posix_spawnattr_setsigmask(&attr, &mask);
posix_spawnattr_setflags(&attr,
POSIX_SPAWN_SETSIGMASK|
#ifdef POSIX_SPAWN_CLOEXEC_DEFAULT
// Darwin: close everything except what we put in file actions
POSIX_SPAWN_CLOEXEC_DEFAULT|
#endif
POSIX_SPAWN_SETPGROUP);
posix_spawn_file_actions_init(&actions);
posix_spawn_file_actions_adddup2(&actions, stdin_fd, STDIN_FILENO);
if (cmd->stdout_name) {
posix_spawn_file_actions_addopen(&actions, STDOUT_FILENO,
cmd->stdout_name, cmd->stdout_flags, 0666);
} else {
posix_spawn_file_actions_adddup2(&actions, STDOUT_FILENO, STDOUT_FILENO);
}
if (cmd->stderr_name) {
posix_spawn_file_actions_addopen(&actions, STDERR_FILENO,
cmd->stderr_name, cmd->stderr_flags, 0666);
} else {
posix_spawn_file_actions_adddup2(&actions, STDERR_FILENO, STDERR_FILENO);
}
pthread_mutex_lock(&spawn_lock);
if (cmd->query->relative_root) {
ignore_result(chdir(cmd->query->relative_root->buf));
} else {
ignore_result(chdir(root->root_path->buf));
}
json_unpack(cmd->definition, "{s:s}", "chdir", &cwd);
if (cwd) {
ignore_result(chdir(cwd));
}
ret = posix_spawnp(&cmd->current_proc, argv[0], &actions, &attr, argv, envp);
if (ret == 0) {
w_root_addref(root);
insert_running_pid(cmd->current_proc, root);
} else {
// On Darwin (at least), posix_spawn can fail but will still populate the
// pid. Since we use the pid to gate future spawns, we need to ensure
// that we clear out the pid on failure, otherwise the trigger would be
// effectively disabled for the rest of the watch lifetime
cmd->current_proc = 0;
}
ignore_result(chdir("/"));
pthread_mutex_unlock(&spawn_lock);
// If failed, we want to make sure we log enough info to figure out why
result_log_level = res == 0 ? W_LOG_DBG : W_LOG_ERR;
w_log(result_log_level, "posix_spawnp:\n");
for (i = 0; argv[i]; i++) {
w_log(result_log_level, "argv[%d] %s\n", i, argv[i]);
}
for (i = 0; envp[i]; i++) {
w_log(result_log_level, "envp[%d] %s\n", i, envp[i]);
}
w_log(result_log_level, "trigger %.*s:%s pid=%d ret=%d %s\n",
(int)root->root_path->len,
root->root_path->buf,
cmd->triggername->buf, (int)cmd->current_proc, ret, strerror(ret));
free(argv);
free(envp);
posix_spawnattr_destroy(&attr);
posix_spawn_file_actions_destroy(&actions);
if (stdin_fd != -1) {
close(stdin_fd);
}
}
struct watchman_trigger_command *w_build_trigger_from_def(
w_root_t *root, json_t *trig, char **errmsg)
{
struct watchman_trigger_command *cmd;
json_t *ele, *query;
json_int_t jint;
const char *name = NULL;
cmd = calloc(1, sizeof(*cmd));
if (!cmd) {
*errmsg = strdup("no memory");
return NULL;
}
cmd->definition = trig;
json_incref(cmd->definition);
query = json_pack("{s:O}", "expression",
json_object_get(cmd->definition, "expression"));
cmd->query = w_query_parse(query, errmsg);
json_decref(query);
if (!cmd->query) {
w_trigger_command_free(cmd);
return NULL;
}
json_unpack(trig, "{s:s}", "name", &name);
if (!name) {
*errmsg = strdup("invalid or missing name");
w_trigger_command_free(cmd);
return NULL;
}
cmd->triggername = w_string_new(name);
cmd->command = json_object_get(trig, "command");
if (cmd->command) {
json_incref(cmd->command);
}
if (!cmd->command || !json_is_array(cmd->command) ||
!json_array_size(cmd->command)) {
*errmsg = strdup("invalid command array");
w_trigger_command_free(cmd);
return NULL;
}
json_unpack(trig, "{s:b}", "append_files", &cmd->append_files);
ele = json_object_get(trig, "stdin");
if (!ele) {
cmd->stdin_style = input_dev_null;
} else if (json_is_array(ele)) {
cmd->stdin_style = input_json;
if (!parse_field_list(ele, &cmd->field_list, errmsg)) {
w_trigger_command_free(cmd);
return NULL;
}
} else if (json_is_string(ele)) {
const char *str = json_string_value(ele);
if (!strcmp(str, "/dev/null")) {
cmd->stdin_style = input_dev_null;
} else if (!strcmp(str, "NAME_PER_LINE")) {
cmd->stdin_style = input_name_list;
} else {
ignore_result(asprintf(errmsg, "invalid stdin value %s", str));
w_trigger_command_free(cmd);
return NULL;
}
} else {
*errmsg = strdup("invalid value for stdin");
w_trigger_command_free(cmd);
return NULL;
}
jint = 0; // unlimited unless specified
json_unpack(trig, "{s:I}", "max_files_stdin", &jint);
if (jint < 0) {
*errmsg = strdup("max_files_stdin must be >= 0");
w_trigger_command_free(cmd);
return NULL;
}
cmd->max_files_stdin = jint;
json_unpack(trig, "{s:s}", "stdout", &cmd->stdout_name);
json_unpack(trig, "{s:s}", "stderr", &cmd->stderr_name);
if (!parse_redirection(&cmd->stdout_name, &cmd->stdout_flags,
"stdout", errmsg)) {
w_trigger_command_free(cmd);
return NULL;
}
if (!parse_redirection(&cmd->stderr_name, &cmd->stderr_flags,
"stderr", errmsg)) {
w_trigger_command_free(cmd);
return NULL;
}
// Copy current environment
cmd->envht = w_envp_make_ht();
// Set some standard vars
w_envp_set(cmd->envht, "WATCHMAN_ROOT", root->root_path);
w_envp_set_cstring(cmd->envht, "WATCHMAN_SOCK", get_sock_name());
w_envp_set(cmd->envht, "WATCHMAN_TRIGGER", cmd->triggername);
return cmd;
}
static void *perf_log_thread(void *unused) {
json_t *samples = NULL;
char **envp;
json_t *perf_cmd;
int64_t sample_batch;
unused_parameter(unused);
w_set_thread_name("perflog");
// Prep some things that we'll need each time we run a command
{
uint32_t env_size;
w_ht_t *envpht = w_envp_make_ht();
char *statedir = dirname(strdup(watchman_state_file));
w_envp_set_cstring(envpht, "WATCHMAN_STATE_DIR", statedir);
w_envp_set_cstring(envpht, "WATCHMAN_SOCK", get_sock_name());
envp = w_envp_make_from_ht(envpht, &env_size);
}
perf_cmd = cfg_get_json(NULL, "perf_logger_command");
if (json_is_string(perf_cmd)) {
perf_cmd = json_pack("[O]", perf_cmd);
}
if (!json_is_array(perf_cmd)) {
w_log(
W_LOG_FATAL,
"perf_logger_command must be either a string or an array of strings\n");
}
sample_batch =
cfg_get_int(NULL, "perf_logger_command_max_samples_per_call", 4);
while (true) {
pthread_mutex_lock(&perf_log_lock);
if (!perf_log_samples) {
pthread_cond_wait(&perf_log_cond, &perf_log_lock);
}
samples = perf_log_samples;
perf_log_samples = NULL;
pthread_mutex_unlock(&perf_log_lock);
if (samples) {
while (json_array_size(samples) > 0) {
int i = 0;
json_t *cmd = json_array();
posix_spawnattr_t attr;
posix_spawn_file_actions_t actions;
pid_t pid;
char **argv = NULL;
json_array_extend(cmd, perf_cmd);
while (i < sample_batch && json_array_size(samples) > 0) {
char *stringy = json_dumps(json_array_get(samples, 0), 0);
json_array_append(cmd, typed_string_to_json(stringy, W_STRING_MIXED));
free(stringy);
json_array_remove(samples, 0);
i++;
}
argv = w_argv_copy_from_json(cmd, 0);
if (!argv) {
char *dumped = json_dumps(cmd, 0);
w_log(W_LOG_FATAL, "error converting %s to an argv array\n", dumped);
}
posix_spawnattr_init(&attr);
#ifdef POSIX_SPAWN_CLOEXEC_DEFAULT
posix_spawnattr_setflags(&attr, POSIX_SPAWN_CLOEXEC_DEFAULT);
#endif
posix_spawn_file_actions_init(&actions);
posix_spawn_file_actions_addopen(&actions, STDIN_FILENO, "/dev/null",
O_RDONLY, 0666);
posix_spawn_file_actions_addopen(&actions, STDOUT_FILENO, "/dev/null",
O_WRONLY, 0666);
posix_spawn_file_actions_addopen(&actions, STDERR_FILENO, "/dev/null",
O_WRONLY, 0666);
if (posix_spawnp(&pid, argv[0], &actions, &attr, argv, envp) == 0) {
// There's no sense waiting here, because w_reap_children is called
// by the reaper thread.
} else {
int err = errno;
w_log(W_LOG_ERR, "failed to spawn %s: %s\n", argv[0],
strerror(err));
}
posix_spawnattr_destroy(&attr);
posix_spawn_file_actions_destroy(&actions);
free(argv);
json_decref(cmd);
}
json_decref(samples);
}
}
return NULL;
}
void PerfLogThread::loop() {
json_ref samples;
char **envp;
json_ref perf_cmd;
int64_t sample_batch;
w_set_thread_name("perflog");
// Prep some things that we'll need each time we run a command
{
uint32_t env_size;
auto envpht = w_envp_make_ht();
char *statedir = dirname(strdup(watchman_state_file));
w_envp_set_cstring(envpht, "WATCHMAN_STATE_DIR", statedir);
w_envp_set_cstring(envpht, "WATCHMAN_SOCK", get_sock_name());
envp = w_envp_make_from_ht(envpht, &env_size);
}
perf_cmd = cfg_get_json("perf_logger_command");
if (json_is_string(perf_cmd)) {
perf_cmd = json_array({perf_cmd});
}
if (!json_is_array(perf_cmd)) {
w_log(
W_LOG_FATAL,
"perf_logger_command must be either a string or an array of strings\n");
}
sample_batch = cfg_get_int("perf_logger_command_max_samples_per_call", 4);
while (!w_is_stopping()) {
{
auto wlock = samples_.wlock();
if (!*wlock) {
cond_.wait(wlock.getUniqueLock());
}
samples = nullptr;
std::swap(samples, *wlock);
}
if (samples) {
while (json_array_size(samples) > 0) {
int i = 0;
auto cmd = json_array();
posix_spawnattr_t attr;
posix_spawn_file_actions_t actions;
pid_t pid;
char **argv = NULL;
json_array_extend(cmd, perf_cmd);
while (i < sample_batch && json_array_size(samples) > 0) {
char *stringy = json_dumps(json_array_get(samples, 0), 0);
if (stringy) {
json_array_append_new(
cmd, typed_string_to_json(stringy, W_STRING_MIXED));
free(stringy);
}
json_array_remove(samples, 0);
i++;
}
argv = w_argv_copy_from_json(cmd, 0);
if (!argv) {
char *dumped = json_dumps(cmd, 0);
w_log(W_LOG_FATAL, "error converting %s to an argv array\n", dumped);
}
posix_spawnattr_init(&attr);
#ifdef POSIX_SPAWN_CLOEXEC_DEFAULT
posix_spawnattr_setflags(&attr, POSIX_SPAWN_CLOEXEC_DEFAULT);
#endif
posix_spawn_file_actions_init(&actions);
posix_spawn_file_actions_addopen(&actions, STDIN_FILENO, "/dev/null",
O_RDONLY, 0666);
posix_spawn_file_actions_addopen(&actions, STDOUT_FILENO, "/dev/null",
O_WRONLY, 0666);
posix_spawn_file_actions_addopen(&actions, STDERR_FILENO, "/dev/null",
O_WRONLY, 0666);
if (posix_spawnp(&pid, argv[0], &actions, &attr, argv, envp) == 0) {
int status;
while (waitpid(pid, &status, 0) != pid) {
if (errno != EINTR) {
break;
}
}
} else {
int err = errno;
w_log(W_LOG_ERR, "failed to spawn %s: %s\n", argv[0],
strerror(err));
}
posix_spawnattr_destroy(&attr);
posix_spawn_file_actions_destroy(&actions);
free(argv);
}
}
}
}
static void spawn_command(
const std::shared_ptr<w_root_t>& root,
struct watchman_trigger_command* cmd,
w_query_res* res,
struct w_clockspec* since_spec) {
char **envp = NULL;
uint32_t i = 0;
int ret;
char **argv = NULL;
uint32_t env_size;
posix_spawn_file_actions_t actions;
posix_spawnattr_t attr;
#ifndef _WIN32
sigset_t mask;
#endif
long arg_max;
size_t argspace_remaining;
bool file_overflow = false;
int result_log_level;
w_string_t *working_dir = NULL;
#ifdef _WIN32
arg_max = 32*1024;
#else
arg_max = sysconf(_SC_ARG_MAX);
#endif
if (arg_max <= 0) {
argspace_remaining = UINT_MAX;
} else {
argspace_remaining = (uint32_t)arg_max;
}
// Allow some misc working overhead
argspace_remaining -= 32;
// Record an overflow before we call prepare_stdin(), which mutates
// and resizes the results to fit the specified limit.
if (cmd->max_files_stdin > 0 &&
res->resultsArray.array().size() > cmd->max_files_stdin) {
file_overflow = true;
}
auto stdin_file = prepare_stdin(cmd, res);
if (!stdin_file) {
w_log(
W_LOG_ERR,
"trigger %s:%s %s\n",
root->root_path.c_str(),
cmd->triggername.c_str(),
strerror(errno));
return;
}
// Assumption: that only one thread will be executing on a given
// cmd instance so that mutation of cmd->envht is safe.
// This is guaranteed in the current architecture.
// It is way too much of a hassle to try to recreate the clock value if it's
// not a relative clock spec, and it's only going to happen on the first run
// anyway, so just skip doing that entirely.
if (since_spec && since_spec->tag == w_cs_clock) {
w_envp_set_cstring(
cmd->envht,
"WATCHMAN_SINCE",
since_spec->clock.position.toClockString().c_str());
} else {
w_envp_unset(cmd->envht, "WATCHMAN_SINCE");
}
w_envp_set_cstring(
cmd->envht,
"WATCHMAN_CLOCK",
res->clockAtStartOfQuery.toClockString().c_str());
if (cmd->query->relative_root) {
w_envp_set(cmd->envht, "WATCHMAN_RELATIVE_ROOT", cmd->query->relative_root);
} else {
w_envp_unset(cmd->envht, "WATCHMAN_RELATIVE_ROOT");
}
// Compute args
auto args = json_deep_copy(cmd->command);
if (cmd->append_files) {
// Measure how much space the base args take up
for (i = 0; i < json_array_size(args); i++) {
const char *ele = json_string_value(json_array_get(args, i));
argspace_remaining -= strlen(ele) + 1 + sizeof(char*);
}
// Dry run with env to compute space
envp = w_envp_make_from_ht(cmd->envht, &env_size);
free(envp);
envp = NULL;
argspace_remaining -= env_size;
for (const auto& item : res->dedupedFileNames) {
// also: NUL terminator and entry in argv
uint32_t size = item.size() + 1 + sizeof(char*);
if (argspace_remaining < size) {
file_overflow = true;
break;
}
argspace_remaining -= size;
json_array_append_new(args, w_string_to_json(item));
}
}
argv = w_argv_copy_from_json(args, 0);
args = nullptr;
w_envp_set_bool(cmd->envht, "WATCHMAN_FILES_OVERFLOW", file_overflow);
envp = w_envp_make_from_ht(cmd->envht, &env_size);
posix_spawnattr_init(&attr);
#ifndef _WIN32
sigemptyset(&mask);
posix_spawnattr_setsigmask(&attr, &mask);
#endif
posix_spawnattr_setflags(&attr,
POSIX_SPAWN_SETSIGMASK|
#ifdef POSIX_SPAWN_CLOEXEC_DEFAULT
// Darwin: close everything except what we put in file actions
POSIX_SPAWN_CLOEXEC_DEFAULT|
#endif
POSIX_SPAWN_SETPGROUP);
posix_spawn_file_actions_init(&actions);
#ifndef _WIN32
posix_spawn_file_actions_adddup2(
&actions, stdin_file->getFileDescriptor(), STDIN_FILENO);
#else
posix_spawn_file_actions_adddup2_handle_np(
&actions, stdin_file->getWindowsHandle(), STDIN_FILENO);
#endif
if (cmd->stdout_name) {
posix_spawn_file_actions_addopen(&actions, STDOUT_FILENO,
cmd->stdout_name, cmd->stdout_flags, 0666);
} else {
posix_spawn_file_actions_adddup2(&actions, STDOUT_FILENO, STDOUT_FILENO);
}
if (cmd->stderr_name) {
posix_spawn_file_actions_addopen(&actions, STDERR_FILENO,
cmd->stderr_name, cmd->stderr_flags, 0666);
} else {
posix_spawn_file_actions_adddup2(&actions, STDERR_FILENO, STDERR_FILENO);
}
// Figure out the appropriate cwd
{
const char *cwd = NULL;
working_dir = NULL;
if (cmd->query->relative_root) {
working_dir = cmd->query->relative_root;
} else {
working_dir = root->root_path;
}
w_string_addref(working_dir);
json_unpack(cmd->definition, "{s:s}", "chdir", &cwd);
if (cwd) {
w_string_t *cwd_str = w_string_new_typed(cwd, W_STRING_BYTE);
if (w_is_path_absolute_cstr(cwd)) {
w_string_delref(working_dir);
working_dir = cwd_str;
} else {
w_string_t *joined;
joined = w_string_path_cat(working_dir, cwd_str);
w_string_delref(cwd_str);
w_string_delref(working_dir);
working_dir = joined;
}
}
w_log(W_LOG_DBG, "using %.*s for working dir\n", working_dir->len,
working_dir->buf);
}
#ifndef _WIN32
// This mutex is present to avoid fighting over the cwd when multiple
// triggers run at the same time. It doesn't coordinate with all
// possible chdir() calls, but this is the only place that we do this
// in the watchman server process.
static std::mutex cwdMutex;
{
std::unique_lock<std::mutex> lock(cwdMutex);
ignore_result(chdir(working_dir->buf));
#else
posix_spawnattr_setcwd_np(&attr, working_dir->buf);
#endif
w_string_delref(working_dir);
working_dir = nullptr;
ret =
posix_spawnp(&cmd->current_proc, argv[0], &actions, &attr, argv, envp);
if (ret != 0) {
// On Darwin (at least), posix_spawn can fail but will still populate the
// pid. Since we use the pid to gate future spawns, we need to ensure
// that we clear out the pid on failure, otherwise the trigger would be
// effectively disabled for the rest of the watch lifetime
cmd->current_proc = 0;
}
#ifndef _WIN32
ignore_result(chdir("/"));
}
#endif
// If failed, we want to make sure we log enough info to figure out why
result_log_level = res == 0 ? W_LOG_DBG : W_LOG_ERR;
w_log(result_log_level, "posix_spawnp: %s\n", cmd->triggername.c_str());
for (i = 0; argv[i]; i++) {
w_log(result_log_level, "argv[%d] %s\n", i, argv[i]);
}
for (i = 0; envp[i]; i++) {
w_log(result_log_level, "envp[%d] %s\n", i, envp[i]);
}
w_log(
result_log_level,
"trigger %s:%s pid=%d ret=%d %s\n",
root->root_path.c_str(),
cmd->triggername.c_str(),
(int)cmd->current_proc,
ret,
strerror(ret));
free(argv);
free(envp);
posix_spawnattr_destroy(&attr);
posix_spawn_file_actions_destroy(&actions);
}
