static void free_fd(int job_id, HANDLE h) { char buf[1024]; HANDLE target = GetStdHandle(STD_OUTPUT_HANDLE); SetFilePointer(h, 0, NULL, FILE_BEGIN); // Wait until we can take the TTY. td_mutex_lock_or_die(&fd_mutex); while (tty_owner != -1) pthread_cond_wait(&tty_free, &fd_mutex); tty_owner = job_id; td_mutex_unlock_or_die(&fd_mutex); // Dump the contents of the temporary file to the TTY. for (;;) { DWORD rb, wb; if (!ReadFile(h, buf, sizeof(buf), &rb, NULL) || rb == 0) break; if (!WriteFile(target, buf, rb, &wb, NULL)) break; } // Mark the TTY as free again and wake waiters. td_mutex_lock_or_die(&fd_mutex); tty_owner = -1; pthread_cond_signal(&tty_free); td_mutex_unlock_or_die(&fd_mutex); // Truncate the temporary file for reuse SetFilePointer(h, 0, NULL, FILE_BEGIN); SetEndOfFile(h); }
void tty_job_exit(int job_id) { /* Find any queued buffers for this job and flush them. */ td_mutex_lock_or_die(&linelock); TTY_PRINTF(("exit job %d\n", job_id)); if (job_id != printing_job) { /* Wait until we can grab the TTY */ while (-1 != printing_job) { pthread_cond_wait(&can_print, &linelock); } } printing_job = job_id; flush_output_queue(job_id); printing_job = -1; td_mutex_unlock_or_die(&linelock); pthread_cond_broadcast(&can_print); }
static int scan_implicit_deps(td_job_queue *queue, td_node *node) { int collect_stats; double t1 = 0.0, t2 = 0.0; td_engine *engine = queue->engine; td_scanner *scanner = node->scanner; int result; if (!scanner) return 0; collect_stats = td_debug_check(engine, TD_DEBUG_STATS); td_mutex_unlock_or_die(&queue->mutex); if (collect_stats) t1 = td_timestamp(); if (!queue->engine->settings.dry_run) result = td_scan_includes(queue->engine, node, scanner); else result = 0; td_mutex_lock_or_die(&queue->mutex); if (collect_stats) { t2 = td_timestamp(); td_mutex_lock_or_die(engine->stats_lock); engine->stats.scan_time += t2 - t1; td_mutex_unlock_or_die(engine->stats_lock); } return result; }
int td_scan_includes(td_engine *engine, td_node *node, td_scanner *state) { td_alloc scratch; int i, count; td_scanner *config = (td_scanner *) state; unsigned int salt = relation_salt_cpp(config); int set_cursor; include_set *set; td_alloc_init(&scratch, 10, 1024 * 1024); set = (include_set *) td_page_alloc(&scratch, sizeof(include_set)); set->count = 0; for (i = 0, count = node->input_count; i < count; ++i) push_include(set, node->inputs[i]); set_cursor = 0; while (set_cursor < set->count) { td_file *input = set->files[set_cursor++]; scan_file(engine, &scratch, input, config, salt, set); } node->job.idep_count = set->count - node->input_count; td_mutex_lock_or_die(engine->lock); node->job.ideps = (td_file **) td_page_alloc(&engine->alloc, sizeof(td_file*) * node->job.idep_count); td_mutex_unlock_or_die(engine->lock); memcpy(&node->job.ideps[0], &set->files[node->input_count], sizeof(td_file*) * node->job.idep_count); td_alloc_cleanup(&scratch); return 0; }
static int is_up_to_date(td_job_queue *queue, td_node *node) { double t1 = 0.0, t2 = 0.0; int collect_stats; int i, count; const td_digest *prev_signature = NULL; td_engine *engine = queue->engine; const td_ancestor_data *ancestor; int up_to_date = 0; collect_stats = td_debug_check(engine, TD_DEBUG_STATS); if (collect_stats) t1 = td_timestamp(); /* We can safely drop the build queue lock in here as no job state is accessed. */ td_mutex_unlock_or_die(&queue->mutex); /* rebuild if any output files are missing */ for (i = 0, count = node->output_count; i < count; ++i) { td_file *file = node->outputs[i]; const td_stat *stat = td_stat_file(engine, file); if (0 == (stat->flags & TD_STAT_EXISTS)) { if (td_debug_check(engine, TD_DEBUG_REASON)) printf("%s: output file %s is missing\n", node->annotation, file->path); goto leave; } } if (NULL != (ancestor = node->ancestor_data)) prev_signature = &ancestor->input_signature; /* rebuild if there is no stored signature */ if (!prev_signature) { if (td_debug_check(engine, TD_DEBUG_REASON)) printf("%s: no previous input signature\n", node->annotation); goto leave; } /* rebuild if the job failed last time */ if (TD_JOB_FAILED == ancestor->job_result) { if (td_debug_check(engine, TD_DEBUG_REASON)) printf("%s: build failed last time\n", node->annotation); goto leave; } /* rebuild if the input signatures have changed */ if (0 != memcmp(prev_signature->data, node->job.input_signature.data, sizeof(td_digest))) { if (td_debug_check(engine, TD_DEBUG_REASON)) printf("%s: input signature differs\n", node->annotation); goto leave; } /* otherwise, the node is up to date */ up_to_date = 1; leave: if (collect_stats) { t2 = td_timestamp(); td_mutex_lock_or_die(engine->lock); engine->stats.up2date_check_time += t2 - t1; td_mutex_unlock_or_die(engine->lock); } td_mutex_lock_or_die(&queue->mutex); return up_to_date; }
static void update_input_signature(td_job_queue *queue, td_node *node) { static unsigned char zero_byte = 0; td_engine *engine = queue->engine; FILE* sign_debug_file = (FILE*) engine->sign_debug_file; int i, count; MD5_CTX context; td_mutex_unlock_or_die(&queue->mutex); MD5_Init(&context); if (sign_debug_file) fprintf(sign_debug_file, "begin signing \"%s\"\n", node->annotation); /* Add the command line */ if (node->action) { if (sign_debug_file) fprintf(sign_debug_file, "action = \"%s\"\n", node->action); MD5_Update(&context, (char*) node->action, (unsigned long) (strlen(node->action) + 1)); } else MD5_Update(&context, "", 1); for (i = 0, count = node->input_count; i < count; ++i) { td_file *input_file = node->inputs[i]; td_digest *digest = td_get_signature(engine, input_file); MD5_Update(&context, digest->data, sizeof(digest->data)); if (sign_debug_file) { char buffer[33]; td_digest_to_string(digest, buffer); fprintf(sign_debug_file, "input[%d] = %s (\"%s\")\n", i, buffer, input_file->path); } } /* add a separator between the inputs and implicit deps */ MD5_Update(&context, &zero_byte, 1); /* We technically invalidate the threading rules here and read the idep array. * * This is OK for the following reasons: * - The implicit dependencies for the node have already been scanned. * - They will never be scanned again (the DAG ensures this) * - The memory view of this array is already guaranteed consistent as we just released the lock. */ for (i = 0, count = node->job.idep_count; i < count; ++i) { td_file *dep = node->job.ideps[i]; td_digest *digest = td_get_signature(engine, dep); MD5_Update(&context, digest->data, sizeof(digest->data)); if (sign_debug_file) { char buffer[33]; td_digest_to_string(digest, buffer); fprintf(sign_debug_file, "implicit_input[%d] = %s (\"%s\")\n", i, buffer, dep->path); } } /* Grab the queue lock again before we publish the input signature */ td_mutex_lock_or_die(&queue->mutex); MD5_Final(node->job.input_signature.data, &context); if (sign_debug_file) { char buffer[33]; td_digest_to_string(&node->job.input_signature, buffer); fprintf(sign_debug_file, "resulting input signature = %s\n\n", buffer); } }
static int run_job(td_job_queue *queue, td_node *node, int job_id) { double t1, mkdir_time, cmd_time; td_engine *engine = queue->engine; int i, count, result, was_signalled = 0; const char *command = node->action; if (!command || '\0' == command[0]) return 0; ++queue->jobs_run; pthread_mutex_unlock(&queue->mutex); t1 = td_timestamp(); /* ensure directories for output files exist */ for (i = 0, count = node->output_count; i < count; ++i) { td_file *dir = td_parent_dir(engine, node->outputs[i]); if (!dir) continue; if (0 != (result = ensure_dir_exists(engine, dir))) goto leave; } mkdir_time = td_timestamp() - t1; t1 = td_timestamp(); /* If the outputs of this node can't be overwritten; delete them now */ if (0 == (TD_NODE_OVERWRITE & node->flags)) { delete_outputs(node); touch_outputs(engine, node); } if (!engine->settings.dry_run) { result = td_exec( command, node->env_count, node->env, &was_signalled, job_id, td_verbosity_check(engine, 2), td_verbosity_check(engine, 1) ? node->annotation : NULL); } else result = 0; cmd_time = td_timestamp() - t1; if (0 != result) { td_mutex_lock_or_die(&queue->mutex); /* Maintain a fail count so we can track why we stopped building if * we're stopping after the first error. Otherwise it might appear as * we succeeded. */ ++queue->fail_count; /* If the command failed or was signalled (e.g. Ctrl+C), abort the build */ if (was_signalled) queue->siginfo.flag = -1; else if (!engine->settings.continue_on_error) queue->siginfo.flag = 1; td_mutex_unlock_or_die(&queue->mutex); } /* If the build failed, and the node isn't set to keep all its output files * in all possible cases (precious), then delete all output files as we * can't assume anything about their state. */ if (0 != result && 0 == (TD_NODE_PRECIOUS & node->flags)) delete_outputs(node); /* Mark all output files as dirty regardless of whether the build succeeded * or not. If it succeeded, we must assume the build overwrote them. * Otherwise, it's likely we've deleted them. In any case, touching them * again isn't going to hurt anything.*/ touch_outputs(engine, node); /* Update engine stats. */ if (td_debug_check(engine, TD_DEBUG_STATS)) { td_mutex_lock_or_die(engine->stats_lock); engine->stats.mkdir_time += mkdir_time; engine->stats.build_time += cmd_time; td_mutex_unlock_or_die(engine->stats_lock); } leave: td_mutex_lock_or_die(&queue->mutex); return result; }
td_file * td_engine_get_file(td_engine *engine, const char *input_path, td_get_file_mode mode) { unsigned int hash; int slot; td_file *chain; td_file *f; int path_len; char path[1024]; const char *out_path; size_t input_path_len = strlen(input_path); if (input_path_len >= sizeof(path)) td_croak("path too long: %s", input_path); if (TD_COPY_STRING == mode) { if (isabspath(input_path)) { strcpy(path, input_path); } else { snprintf(path, sizeof path, "%s%s", cwd, input_path); } sanitize_path(path, sizeof(path), input_path_len); hash = (unsigned int) djb2_hash(path); out_path = path; } else { hash = (unsigned int) djb2_hash(input_path); out_path = input_path; } td_mutex_lock_or_die(engine->lock); slot = (int) (hash % engine->file_hash_size); chain = engine->file_hash[slot]; while (chain) { if (chain->hash == hash && 0 == strcmp(out_path, chain->path)) { td_mutex_unlock_or_die(engine->lock); return chain; } chain = chain->bucket_next; } ++engine->stats.file_count; f = td_page_alloc(&engine->alloc, sizeof(td_file)); memset(f, 0, sizeof(td_file)); f->path_len = path_len = (int) strlen(out_path); if (TD_COPY_STRING == mode) f->path = td_page_strdup(&engine->alloc, out_path, path_len); else f->path = out_path; f->hash = hash; f->name = find_basename(f->path, path_len); f->bucket_next = engine->file_hash[slot]; f->signer = engine->default_signer; f->stat_dirty = 1; f->signature_dirty = 1; f->frozen_relstring_index = ~(0u); engine->file_hash[slot] = f; td_mutex_unlock_or_die(engine->lock); return f; }
void tty_emit(int job_id, int is_stderr, int sort_key, const char *data, int len) { line_buffer *buf = NULL; td_mutex_lock_or_die(&linelock); while (len > 0) { /* Wait for a line buffer to become available, or for the tty to become free. */ for (;;) { /* If we already have the tty, or we can get it: break */ if (-1 == printing_job || job_id == printing_job) break; /* If we can allocate a buffer: break */ if (NULL != (buf = alloc_line_buffer())) break; /* Otherwise wait */ pthread_cond_wait(&can_print, &linelock); } if (-1 == printing_job) { /* Let this job own the output channel */ TTY_PRINTF(("job %d is taking the tty\n", job_id)); printing_job = job_id; flush_output_queue(job_id); } else if (job_id != printing_job) { assert(buf); } if (!buf) { /* This thread owns the TTY, so just print. We don't need to keep the * mutex locked as this job will not finsh and reset the currently * printing job until later. Releasing the mutex now means other * threads can come in and buffer their data. */ td_mutex_unlock_or_die(&linelock); TTY_PRINTF(("copying %d bytes of data from job_id %d, stderr=%d, sort_key %d\n", len, job_id, is_stderr, sort_key)); write(is_stderr ? STDERR_FILENO : STDOUT_FILENO, data, strlen(data)); return; /* finish the loop immediately */ } else { /* We can't print this data as we don't own the TTY so we buffer it. */ TTY_PRINTF(("buffering %d bytes of data from job_id %d, stderr=%d, sort_key %d\n", len, job_id, is_stderr, sort_key)); /* PERF: Could release mutex around this memcpy, not sure if it's a win. */ buf->job_id = job_id; buf->sort_key = sort_key; buf->len = len > LINEBUF_SIZE - 1 ? LINEBUF_SIZE - 1 : len; buf->is_stderr = is_stderr; memcpy(buf->data, data, buf->len); buf->data[buf->len] = '\0'; len -= buf->len; data += buf->len; /* Queue this line for output */ queued_linebufs[queued_linebuf_count++] = buf; } } td_mutex_unlock_or_die(&linelock); return; }