void td_load_ancestors(td_engine *engine) { FILE* f; int i, count; long file_size; size_t read_count; if (NULL == (f = fopen(TD_ANCESTOR_FILE, "rb"))) { if (td_debug_check(engine, TD_DEBUG_ANCESTORS)) printf("couldn't open %s; no ancestor information present\n", TD_ANCESTOR_FILE); return; } fseek(f, 0, SEEK_END); file_size = ftell(f); rewind(f); if (file_size % sizeof(td_ancestor_data) != 0) td_croak("illegal ancestor file: %d not a multiple of %d bytes", (int) file_size, sizeof(td_ancestor_data)); engine->ancestor_count = count = (int) (file_size / sizeof(td_ancestor_data)); engine->ancestors = malloc(file_size); engine->ancestor_used = calloc(sizeof(td_node *), count); read_count = fread(engine->ancestors, sizeof(td_ancestor_data), count, f); if (td_debug_check(engine, TD_DEBUG_ANCESTORS)) printf("read %d ancestors\n", count); if (read_count != (size_t) count) td_croak("only read %d items, wanted %d", read_count, count); for (i = 1; i < count; ++i) { int cmp = td_compare_ancestors(&engine->ancestors[i-1], &engine->ancestors[i]); if (cmp == 0) td_croak("bad ancestor file; duplicate item (%d/%d)", i, count); if (cmp > 0) td_croak("bad ancestor file; bad sort order on item (%d/%d)", i, count); } if (td_debug_check(engine, TD_DEBUG_ANCESTORS)) { printf("full ancestor dump on load:\n"); for (i = 0; i < count; ++i) { char guid[33], sig[33]; td_digest_to_string(&engine->ancestors[i].guid, guid); td_digest_to_string(&engine->ancestors[i].input_signature, sig); printf("%s %s %ld %d\n", guid, sig, (long) engine->ancestors[i].access_time, engine->ancestors[i].job_result); } } fclose(f); }
static void scan_file( td_engine *engine, td_alloc *scratch, td_file *file, td_scanner *scanner, unsigned int salt, include_set *set) { int i, count; td_file **files; int found_count = 0; td_file* found_files[TD_MAX_INCLUDES_IN_FILE]; td_include_data includes[TD_MAX_INCLUDES_IN_FILE]; /* see if there is a cached include set for this file */ files = td_engine_get_relations(engine, file, salt, &count); if (files) { if (td_debug_check(engine, TD_DEBUG_SCAN)) printf("%s: hit relation cache; %d entries\n", file->path, count); for (i = 0; i < count; ++i) push_include(set, files[i]); return; } if (td_debug_check(engine, TD_DEBUG_SCAN)) printf("%s: scanning\n", file->path); count = scan_file_data(scratch, file, &includes[0], sizeof(includes)/sizeof(includes[0]), scanner); for (i = 0; i < count; ++i) { if (NULL != (found_files[found_count] = find_file(file, engine, &includes[i], scanner))) ++found_count; } for (i = 0; i < found_count; ++i) push_include(set, found_files[i]); if (td_debug_check(engine, TD_DEBUG_SCAN)) printf("%s: inserting %d entries in relation cache\n", file->path, found_count); td_engine_set_relations(engine, file, salt, found_count, found_files); }
static void transition_job(td_job_queue *queue, td_node *node, td_jobstate new_state) { if (td_debug_check(queue->engine, TD_DEBUG_QUEUE)) { printf("[%s] %s -> %s { %d blockers }\n", node->annotation, jobstate_name(node->job.state), jobstate_name(new_state), node->job.block_count); } node->job.state = new_state; }
static void enqueue(td_job_queue *queue, td_node *node) { assert(is_root(node) || !is_queued(node)); node->job.flags |= TD_JOBF_QUEUED; assert((queue->tail - queue->head) < queue->array_size); if (td_debug_check(queue->engine, TD_DEBUG_QUEUE)) printf("enqueueing %s\n", node->annotation); queue->array[queue->tail % queue->array_size] = node; ++queue->tail; }
static void compute_node_guid(td_engine *engine, td_node *node) { MD5_CTX context; MD5_Init(&context); md5_string(&context, node->action); md5_string(&context, node->annotation); md5_string(&context, node->salt); MD5_Final(node->guid.data, &context); if (td_debug_check(engine, TD_DEBUG_NODES)) { char guidstr[33]; td_digest_to_string(&node->guid, guidstr); printf("%s with guid %s\n", node->annotation, guidstr); } }
void td_setup_ancestor_data(td_engine *engine, td_node *node) { compute_node_guid(engine, node); ++engine->stats.ancestor_checks; if (engine->ancestors) { td_ancestor_data key; key.guid = node->guid; /* only key field is relevant */ node->ancestor_data = (td_ancestor_data *) bsearch(&key, engine->ancestors, engine->ancestor_count, sizeof(td_ancestor_data), td_compare_ancestors); if (node->ancestor_data) { int index = (int) (node->ancestor_data - engine->ancestors); td_node *other; if (NULL != (other = engine->ancestor_used[index])) td_croak("node error: nodes \"%s\" and \"%s\" share the same ancestor", node->annotation, other->annotation); engine->ancestor_used[index] = node; ++engine->stats.ancestor_nodes; } else { if (td_debug_check(engine, TD_DEBUG_ANCESTORS)) { char guidstr[33]; td_digest_to_string(&node->guid, guidstr); printf("no ancestor for %s with guid %s\n", node->annotation, guidstr); } } } else { /* We didn't load any ancestor data, just set the ancestor to NULL. * Everything will rebuild without ancestry. */ node->ancestor_data = NULL; } }
static int scan_implicit_deps(td_job_queue *queue, td_node *node) { int collect_stats; double t1 = 0.0, t2 = 0.0; td_engine *engine = queue->engine; td_scanner *scanner = node->scanner; int result; if (!scanner) return 0; collect_stats = td_debug_check(engine, TD_DEBUG_STATS); td_mutex_unlock_or_die(&queue->mutex); if (collect_stats) t1 = td_timestamp(); if (!queue->engine->settings.dry_run) result = td_scan_includes(queue->engine, node, scanner); else result = 0; td_mutex_lock_or_die(&queue->mutex); if (collect_stats) { t2 = td_timestamp(); td_mutex_lock_or_die(engine->stats_lock); engine->stats.scan_time += t2 - t1; td_mutex_unlock_or_die(engine->stats_lock); } return result; }
void td_save_ancestors(td_engine *engine, td_node *root) { FILE* f; int i, count, max_count; int output_cursor, write_count; td_ancestor_data *output; unsigned char *visited; time_t now = time(NULL); const int dbg = td_debug_check(engine, TD_DEBUG_ANCESTORS); if (NULL == (f = fopen(TD_ANCESTOR_FILE ".tmp", "wb"))) { fprintf(stderr, "warning: couldn't save ancestors\n"); return; } max_count = engine->node_count + engine->ancestor_count; output = (td_ancestor_data *) malloc(sizeof(td_ancestor_data) * max_count); visited = (unsigned char *) calloc(engine->ancestor_count, 1); output_cursor = 0; update_ancestors(engine, root, now, &output_cursor, output, visited); if (dbg) printf("refreshed %d ancestors\n", output_cursor); for (i = 0, count = engine->ancestor_count; i < count; ++i) { const td_ancestor_data *a = &engine->ancestors[i]; if (!visited[i] && !ancestor_timed_out(a, now)) output[output_cursor++] = *a; } if (dbg) printf("%d ancestors to save in total\n", output_cursor); qsort(output, output_cursor, sizeof(td_ancestor_data), td_compare_ancestors); if (dbg) { printf("full ancestor dump on save:\n"); for (i = 0; i < output_cursor; ++i) { char guid[33], sig[33]; td_digest_to_string(&output[i].guid, guid); td_digest_to_string(&output[i].input_signature, sig); printf("%s %s %ld %d\n", guid, sig, (long) output[i].access_time, output[i].job_result); } } write_count = (int) fwrite(output, sizeof(td_ancestor_data), output_cursor, f); fclose(f); free(visited); free(output); if (write_count != output_cursor) td_croak("couldn't write %d entries; only wrote %d", output_cursor, write_count); if (0 != td_move_file(TD_ANCESTOR_FILE ".tmp", TD_ANCESTOR_FILE)) td_croak("couldn't rename %s to %s", TD_ANCESTOR_FILE ".tmp", TD_ANCESTOR_FILE); }
td_build_result td_build(td_engine *engine, td_node *node, int *jobs_run) { int i; int thread_count; td_job_queue queue; if (engine->stats.build_called) td_croak("build() called more than once on same engine"); if (0 != td_init_exec()) td_croak("couldn't initialize command execution"); engine->stats.build_called = 1; memset(&queue, 0, sizeof(queue)); queue.engine = engine; pthread_mutex_init(&queue.mutex, NULL); pthread_cond_init(&queue.work_avail, NULL); queue.array_size = engine->node_count; queue.array = (td_node **) calloc(engine->node_count, sizeof(td_node*)); queue.siginfo.mutex = &queue.mutex; queue.siginfo.cond = &queue.work_avail; thread_count = engine->settings.thread_count; if (thread_count > TD_MAX_THREADS) thread_count = TD_MAX_THREADS; if (td_debug_check(engine, TD_DEBUG_QUEUE)) printf("using %d build threads\n", thread_count); td_block_signals(1); td_install_sighandler(&queue.siginfo); for (i = 0; i < thread_count-1; ++i) { int rc; if (td_debug_check(engine, TD_DEBUG_QUEUE)) printf("starting thread %d\n", i); thread_arg[i].job_id = i + 2; thread_arg[i].queue = &queue; rc = pthread_create(&threads[i], NULL, build_worker, &thread_arg[i]); if (0 != rc) td_croak("couldn't start thread %d: %s", i, strerror(rc)); } pthread_mutex_lock(&queue.mutex); node->job.flags |= TD_JOBF_ROOT; enqueue(&queue, node); pthread_mutex_unlock(&queue.mutex); pthread_cond_broadcast(&queue.work_avail); { thread_start_arg main_arg; main_arg.job_id = 1; main_arg.queue = &queue; build_worker(&main_arg); } if (td_verbosity_check(engine, 2) && -1 == queue.siginfo.flag) printf("*** aborted on signal %s\n", queue.siginfo.reason); for (i = thread_count - 2; i >= 0; --i) { void *res; int rc = pthread_join(threads[i], &res); if (0 != rc) td_croak("couldn't join thread %d: %s", i, strerror(rc)); } if (0 != queue.thread_count) td_croak("threads are still running"); /* there is a tiny race condition here if a user presses Ctrl-C just * before the write to the static pointer occurs, but that's in nanosecond * land */ td_remove_sighandler(); free(queue.array); pthread_cond_destroy(&queue.work_avail); pthread_mutex_destroy(&queue.mutex); *jobs_run = queue.jobs_run; if (queue.siginfo.flag < 0) return TD_BUILD_ABORTED; else if (queue.fail_count) return TD_BUILD_FAILED; else return TD_BUILD_SUCCESS; }
static void advance_job(td_job_queue *queue, td_node *node, int job_id) { td_jobstate state; while ((state = node->job.state) < TD_JOB_COMPLETED) { switch (state) { case TD_JOB_INITIAL: if (node->job.block_count > 0) { /* enqueue any blocking jobs and transition to the blocked state */ int i, count, bc = 0; td_node **deps = node->deps; for (i = 0, count = node->dep_count; i < count; ++i) { td_node *dep = deps[i]; if (!is_completed(dep)) { ++bc; if (!is_queued(dep) && dep->job.state < TD_JOB_BLOCKED) enqueue(queue, dep); } } assert(bc == node->job.block_count); transition_job(queue, node, TD_JOB_BLOCKED); pthread_cond_broadcast(&queue->work_avail); return; } else { /* nothing is blocking this job, so scan implicit deps immediately */ transition_job(queue, node, TD_JOB_SCANNING); } break; case TD_JOB_BLOCKED: assert(0 == node->job.block_count); if (0 == node->job.failed_deps) transition_job(queue, node, TD_JOB_SCANNING); else transition_job(queue, node, TD_JOB_FAILED); break; case TD_JOB_SCANNING: if (0 == scan_implicit_deps(queue, node)) { update_input_signature(queue, node); if (is_up_to_date(queue, node)) transition_job(queue, node, TD_JOB_UPTODATE); else transition_job(queue, node, TD_JOB_RUNNING); } else { /* implicit dependency scanning failed */ transition_job(queue, node, TD_JOB_FAILED); } break; case TD_JOB_RUNNING: if (0 != run_job(queue, node, job_id)) transition_job(queue, node, TD_JOB_FAILED); else transition_job(queue, node, TD_JOB_COMPLETED); break; default: assert(0); td_croak("can't get here"); break; } } if (is_completed(node)) { int qcount = 0; td_job_chain *chain = node->job.pending_jobs; if (td_debug_check(queue->engine, TD_DEBUG_QUEUE)) printf("%s completed - enqueing blocked jobs\n", node->annotation); /* unblock all jobs that are waiting for this job and enqueue them */ while (chain) { td_node *n = chain->node; if (is_failed(node)) n->job.failed_deps++; /* nodes blocked on this node can't be completed yet */ assert(!is_completed(n)); if (0 == --n->job.block_count) { if (!is_queued(n)) enqueue(queue, n); ++qcount; } chain = chain->next; } if (1 < qcount) pthread_cond_broadcast(&queue->work_avail); else if (1 == qcount) pthread_cond_signal(&queue->work_avail); } }
static int is_up_to_date(td_job_queue *queue, td_node *node) { double t1 = 0.0, t2 = 0.0; int collect_stats; int i, count; const td_digest *prev_signature = NULL; td_engine *engine = queue->engine; const td_ancestor_data *ancestor; int up_to_date = 0; collect_stats = td_debug_check(engine, TD_DEBUG_STATS); if (collect_stats) t1 = td_timestamp(); /* We can safely drop the build queue lock in here as no job state is accessed. */ td_mutex_unlock_or_die(&queue->mutex); /* rebuild if any output files are missing */ for (i = 0, count = node->output_count; i < count; ++i) { td_file *file = node->outputs[i]; const td_stat *stat = td_stat_file(engine, file); if (0 == (stat->flags & TD_STAT_EXISTS)) { if (td_debug_check(engine, TD_DEBUG_REASON)) printf("%s: output file %s is missing\n", node->annotation, file->path); goto leave; } } if (NULL != (ancestor = node->ancestor_data)) prev_signature = &ancestor->input_signature; /* rebuild if there is no stored signature */ if (!prev_signature) { if (td_debug_check(engine, TD_DEBUG_REASON)) printf("%s: no previous input signature\n", node->annotation); goto leave; } /* rebuild if the job failed last time */ if (TD_JOB_FAILED == ancestor->job_result) { if (td_debug_check(engine, TD_DEBUG_REASON)) printf("%s: build failed last time\n", node->annotation); goto leave; } /* rebuild if the input signatures have changed */ if (0 != memcmp(prev_signature->data, node->job.input_signature.data, sizeof(td_digest))) { if (td_debug_check(engine, TD_DEBUG_REASON)) printf("%s: input signature differs\n", node->annotation); goto leave; } /* otherwise, the node is up to date */ up_to_date = 1; leave: if (collect_stats) { t2 = td_timestamp(); td_mutex_lock_or_die(engine->lock); engine->stats.up2date_check_time += t2 - t1; td_mutex_unlock_or_die(engine->lock); } td_mutex_lock_or_die(&queue->mutex); return up_to_date; }
static int run_job(td_job_queue *queue, td_node *node, int job_id) { double t1, mkdir_time, cmd_time; td_engine *engine = queue->engine; int i, count, result, was_signalled = 0; const char *command = node->action; if (!command || '\0' == command[0]) return 0; ++queue->jobs_run; pthread_mutex_unlock(&queue->mutex); t1 = td_timestamp(); /* ensure directories for output files exist */ for (i = 0, count = node->output_count; i < count; ++i) { td_file *dir = td_parent_dir(engine, node->outputs[i]); if (!dir) continue; if (0 != (result = ensure_dir_exists(engine, dir))) goto leave; } mkdir_time = td_timestamp() - t1; t1 = td_timestamp(); /* If the outputs of this node can't be overwritten; delete them now */ if (0 == (TD_NODE_OVERWRITE & node->flags)) { delete_outputs(node); touch_outputs(engine, node); } if (!engine->settings.dry_run) { result = td_exec( command, node->env_count, node->env, &was_signalled, job_id, td_verbosity_check(engine, 2), td_verbosity_check(engine, 1) ? node->annotation : NULL); } else result = 0; cmd_time = td_timestamp() - t1; if (0 != result) { td_mutex_lock_or_die(&queue->mutex); /* Maintain a fail count so we can track why we stopped building if * we're stopping after the first error. Otherwise it might appear as * we succeeded. */ ++queue->fail_count; /* If the command failed or was signalled (e.g. Ctrl+C), abort the build */ if (was_signalled) queue->siginfo.flag = -1; else if (!engine->settings.continue_on_error) queue->siginfo.flag = 1; td_mutex_unlock_or_die(&queue->mutex); } /* If the build failed, and the node isn't set to keep all its output files * in all possible cases (precious), then delete all output files as we * can't assume anything about their state. */ if (0 != result && 0 == (TD_NODE_PRECIOUS & node->flags)) delete_outputs(node); /* Mark all output files as dirty regardless of whether the build succeeded * or not. If it succeeded, we must assume the build overwrote them. * Otherwise, it's likely we've deleted them. In any case, touching them * again isn't going to hurt anything.*/ touch_outputs(engine, node); /* Update engine stats. */ if (td_debug_check(engine, TD_DEBUG_STATS)) { td_mutex_lock_or_die(engine->stats_lock); engine->stats.mkdir_time += mkdir_time; engine->stats.build_time += cmd_time; td_mutex_unlock_or_die(engine->stats_lock); } leave: td_mutex_lock_or_die(&queue->mutex); return result; }