void
td_load_ancestors(td_engine *engine)
{
FILE* f;
int i, count;
long file_size;
size_t read_count;
if (NULL == (f = fopen(TD_ANCESTOR_FILE, "rb")))
{
if (td_debug_check(engine, TD_DEBUG_ANCESTORS))
printf("couldn't open %s; no ancestor information present\n", TD_ANCESTOR_FILE);
return;
}
fseek(f, 0, SEEK_END);
file_size = ftell(f);
rewind(f);
if (file_size % sizeof(td_ancestor_data) != 0)
td_croak("illegal ancestor file: %d not a multiple of %d bytes",
(int) file_size, sizeof(td_ancestor_data));
engine->ancestor_count = count = (int) (file_size / sizeof(td_ancestor_data));
engine->ancestors = malloc(file_size);
engine->ancestor_used = calloc(sizeof(td_node *), count);
read_count = fread(engine->ancestors, sizeof(td_ancestor_data), count, f);
if (td_debug_check(engine, TD_DEBUG_ANCESTORS))
printf("read %d ancestors\n", count);
if (read_count != (size_t) count)
td_croak("only read %d items, wanted %d", read_count, count);
for (i = 1; i < count; ++i)
{
int cmp = td_compare_ancestors(&engine->ancestors[i-1], &engine->ancestors[i]);
if (cmp == 0)
td_croak("bad ancestor file; duplicate item (%d/%d)", i, count);
if (cmp > 0)
td_croak("bad ancestor file; bad sort order on item (%d/%d)", i, count);
}
if (td_debug_check(engine, TD_DEBUG_ANCESTORS))
{
printf("full ancestor dump on load:\n");
for (i = 0; i < count; ++i)
{
char guid[33], sig[33];
td_digest_to_string(&engine->ancestors[i].guid, guid);
td_digest_to_string(&engine->ancestors[i].input_signature, sig);
printf("%s %s %ld %d\n", guid, sig, (long) engine->ancestors[i].access_time, engine->ancestors[i].job_result);
}
}
fclose(f);
}
static void
scan_file(
td_engine *engine,
td_alloc *scratch,
td_file *file,
td_scanner *scanner,
unsigned int salt,
include_set *set)
{
int i, count;
td_file **files;
int found_count = 0;
td_file* found_files[TD_MAX_INCLUDES_IN_FILE];
td_include_data includes[TD_MAX_INCLUDES_IN_FILE];
/* see if there is a cached include set for this file */
files = td_engine_get_relations(engine, file, salt, &count);
if (files)
{
if (td_debug_check(engine, TD_DEBUG_SCAN))
printf("%s: hit relation cache; %d entries\n", file->path, count);
for (i = 0; i < count; ++i)
push_include(set, files[i]);
return;
}
if (td_debug_check(engine, TD_DEBUG_SCAN))
printf("%s: scanning\n", file->path);
count = scan_file_data(scratch, file, &includes[0], sizeof(includes)/sizeof(includes[0]), scanner);
for (i = 0; i < count; ++i)
{
if (NULL != (found_files[found_count] = find_file(file, engine, &includes[i], scanner)))
++found_count;
}
for (i = 0; i < found_count; ++i)
push_include(set, found_files[i]);
if (td_debug_check(engine, TD_DEBUG_SCAN))
printf("%s: inserting %d entries in relation cache\n", file->path, found_count);
td_engine_set_relations(engine, file, salt, found_count, found_files);
}
static void
transition_job(td_job_queue *queue, td_node *node, td_jobstate new_state)
{
if (td_debug_check(queue->engine, TD_DEBUG_QUEUE))
{
printf("[%s] %s -> %s { %d blockers }\n",
node->annotation,
jobstate_name(node->job.state),
jobstate_name(new_state),
node->job.block_count);
}
node->job.state = new_state;
}
static void
enqueue(td_job_queue *queue, td_node *node)
{
assert(is_root(node) || !is_queued(node));
node->job.flags |= TD_JOBF_QUEUED;
assert((queue->tail - queue->head) < queue->array_size);
if (td_debug_check(queue->engine, TD_DEBUG_QUEUE))
printf("enqueueing %s\n", node->annotation);
queue->array[queue->tail % queue->array_size] = node;
++queue->tail;
}
static void
compute_node_guid(td_engine *engine, td_node *node)
{
MD5_CTX context;
MD5_Init(&context);
md5_string(&context, node->action);
md5_string(&context, node->annotation);
md5_string(&context, node->salt);
MD5_Final(node->guid.data, &context);
if (td_debug_check(engine, TD_DEBUG_NODES))
{
char guidstr[33];
td_digest_to_string(&node->guid, guidstr);
printf("%s with guid %s\n", node->annotation, guidstr);
}
}
void
td_setup_ancestor_data(td_engine *engine, td_node *node)
{
compute_node_guid(engine, node);
++engine->stats.ancestor_checks;
if (engine->ancestors)
{
td_ancestor_data key;
key.guid = node->guid; /* only key field is relevant */
node->ancestor_data = (td_ancestor_data *)
bsearch(&key, engine->ancestors, engine->ancestor_count, sizeof(td_ancestor_data), td_compare_ancestors);
if (node->ancestor_data)
{
int index = (int) (node->ancestor_data - engine->ancestors);
td_node *other;
if (NULL != (other = engine->ancestor_used[index]))
td_croak("node error: nodes \"%s\" and \"%s\" share the same ancestor", node->annotation, other->annotation);
engine->ancestor_used[index] = node;
++engine->stats.ancestor_nodes;
}
else
{
if (td_debug_check(engine, TD_DEBUG_ANCESTORS))
{
char guidstr[33];
td_digest_to_string(&node->guid, guidstr);
printf("no ancestor for %s with guid %s\n", node->annotation, guidstr);
}
}
}
else
{
/* We didn't load any ancestor data, just set the ancestor to NULL.
* Everything will rebuild without ancestry. */
node->ancestor_data = NULL;
}
}
static int
scan_implicit_deps(td_job_queue *queue, td_node *node)
{
int collect_stats;
double t1 = 0.0, t2 = 0.0;
td_engine *engine = queue->engine;
td_scanner *scanner = node->scanner;
int result;
if (!scanner)
return 0;
collect_stats = td_debug_check(engine, TD_DEBUG_STATS);
td_mutex_unlock_or_die(&queue->mutex);
if (collect_stats)
t1 = td_timestamp();
if (!queue->engine->settings.dry_run)
result = td_scan_includes(queue->engine, node, scanner);
else
result = 0;
td_mutex_lock_or_die(&queue->mutex);
if (collect_stats)
{
t2 = td_timestamp();
td_mutex_lock_or_die(engine->stats_lock);
engine->stats.scan_time += t2 - t1;
td_mutex_unlock_or_die(engine->stats_lock);
}
return result;
}
void
td_save_ancestors(td_engine *engine, td_node *root)
{
FILE* f;
int i, count, max_count;
int output_cursor, write_count;
td_ancestor_data *output;
unsigned char *visited;
time_t now = time(NULL);
const int dbg = td_debug_check(engine, TD_DEBUG_ANCESTORS);
if (NULL == (f = fopen(TD_ANCESTOR_FILE ".tmp", "wb")))
{
fprintf(stderr, "warning: couldn't save ancestors\n");
return;
}
max_count = engine->node_count + engine->ancestor_count;
output = (td_ancestor_data *) malloc(sizeof(td_ancestor_data) * max_count);
visited = (unsigned char *) calloc(engine->ancestor_count, 1);
output_cursor = 0;
update_ancestors(engine, root, now, &output_cursor, output, visited);
if (dbg)
printf("refreshed %d ancestors\n", output_cursor);
for (i = 0, count = engine->ancestor_count; i < count; ++i)
{
const td_ancestor_data *a = &engine->ancestors[i];
if (!visited[i] && !ancestor_timed_out(a, now))
output[output_cursor++] = *a;
}
if (dbg)
printf("%d ancestors to save in total\n", output_cursor);
qsort(output, output_cursor, sizeof(td_ancestor_data), td_compare_ancestors);
if (dbg)
{
printf("full ancestor dump on save:\n");
for (i = 0; i < output_cursor; ++i)
{
char guid[33], sig[33];
td_digest_to_string(&output[i].guid, guid);
td_digest_to_string(&output[i].input_signature, sig);
printf("%s %s %ld %d\n", guid, sig, (long) output[i].access_time, output[i].job_result);
}
}
write_count = (int) fwrite(output, sizeof(td_ancestor_data), output_cursor, f);
fclose(f);
free(visited);
free(output);
if (write_count != output_cursor)
td_croak("couldn't write %d entries; only wrote %d", output_cursor, write_count);
if (0 != td_move_file(TD_ANCESTOR_FILE ".tmp", TD_ANCESTOR_FILE))
td_croak("couldn't rename %s to %s", TD_ANCESTOR_FILE ".tmp", TD_ANCESTOR_FILE);
}
td_build_result
td_build(td_engine *engine, td_node *node, int *jobs_run)
{
int i;
int thread_count;
td_job_queue queue;
if (engine->stats.build_called)
td_croak("build() called more than once on same engine");
if (0 != td_init_exec())
td_croak("couldn't initialize command execution");
engine->stats.build_called = 1;
memset(&queue, 0, sizeof(queue));
queue.engine = engine;
pthread_mutex_init(&queue.mutex, NULL);
pthread_cond_init(&queue.work_avail, NULL);
queue.array_size = engine->node_count;
queue.array = (td_node **) calloc(engine->node_count, sizeof(td_node*));
queue.siginfo.mutex = &queue.mutex;
queue.siginfo.cond = &queue.work_avail;
thread_count = engine->settings.thread_count;
if (thread_count > TD_MAX_THREADS)
thread_count = TD_MAX_THREADS;
if (td_debug_check(engine, TD_DEBUG_QUEUE))
printf("using %d build threads\n", thread_count);
td_block_signals(1);
td_install_sighandler(&queue.siginfo);
for (i = 0; i < thread_count-1; ++i)
{
int rc;
if (td_debug_check(engine, TD_DEBUG_QUEUE))
printf("starting thread %d\n", i);
thread_arg[i].job_id = i + 2;
thread_arg[i].queue = &queue;
rc = pthread_create(&threads[i], NULL, build_worker, &thread_arg[i]);
if (0 != rc)
td_croak("couldn't start thread %d: %s", i, strerror(rc));
}
pthread_mutex_lock(&queue.mutex);
node->job.flags |= TD_JOBF_ROOT;
enqueue(&queue, node);
pthread_mutex_unlock(&queue.mutex);
pthread_cond_broadcast(&queue.work_avail);
{
thread_start_arg main_arg;
main_arg.job_id = 1;
main_arg.queue = &queue;
build_worker(&main_arg);
}
if (td_verbosity_check(engine, 2) && -1 == queue.siginfo.flag)
printf("*** aborted on signal %s\n", queue.siginfo.reason);
for (i = thread_count - 2; i >= 0; --i)
{
void *res;
int rc = pthread_join(threads[i], &res);
if (0 != rc)
td_croak("couldn't join thread %d: %s", i, strerror(rc));
}
if (0 != queue.thread_count)
td_croak("threads are still running");
/* there is a tiny race condition here if a user presses Ctrl-C just
* before the write to the static pointer occurs, but that's in nanosecond
* land */
td_remove_sighandler();
free(queue.array);
pthread_cond_destroy(&queue.work_avail);
pthread_mutex_destroy(&queue.mutex);
*jobs_run = queue.jobs_run;
if (queue.siginfo.flag < 0)
return TD_BUILD_ABORTED;
else if (queue.fail_count)
return TD_BUILD_FAILED;
else
return TD_BUILD_SUCCESS;
}
static void
advance_job(td_job_queue *queue, td_node *node, int job_id)
{
td_jobstate state;
while ((state = node->job.state) < TD_JOB_COMPLETED)
{
switch (state)
{
case TD_JOB_INITIAL:
if (node->job.block_count > 0)
{
/* enqueue any blocking jobs and transition to the blocked state */
int i, count, bc = 0;
td_node **deps = node->deps;
for (i = 0, count = node->dep_count; i < count; ++i)
{
td_node *dep = deps[i];
if (!is_completed(dep))
{
++bc;
if (!is_queued(dep) && dep->job.state < TD_JOB_BLOCKED)
enqueue(queue, dep);
}
}
assert(bc == node->job.block_count);
transition_job(queue, node, TD_JOB_BLOCKED);
pthread_cond_broadcast(&queue->work_avail);
return;
}
else
{
/* nothing is blocking this job, so scan implicit deps immediately */
transition_job(queue, node, TD_JOB_SCANNING);
}
break;
case TD_JOB_BLOCKED:
assert(0 == node->job.block_count);
if (0 == node->job.failed_deps)
transition_job(queue, node, TD_JOB_SCANNING);
else
transition_job(queue, node, TD_JOB_FAILED);
break;
case TD_JOB_SCANNING:
if (0 == scan_implicit_deps(queue, node))
{
update_input_signature(queue, node);
if (is_up_to_date(queue, node))
transition_job(queue, node, TD_JOB_UPTODATE);
else
transition_job(queue, node, TD_JOB_RUNNING);
}
else
{
/* implicit dependency scanning failed */
transition_job(queue, node, TD_JOB_FAILED);
}
break;
case TD_JOB_RUNNING:
if (0 != run_job(queue, node, job_id))
transition_job(queue, node, TD_JOB_FAILED);
else
transition_job(queue, node, TD_JOB_COMPLETED);
break;
default:
assert(0);
td_croak("can't get here");
break;
}
}
if (is_completed(node))
{
int qcount = 0;
td_job_chain *chain = node->job.pending_jobs;
if (td_debug_check(queue->engine, TD_DEBUG_QUEUE))
printf("%s completed - enqueing blocked jobs\n", node->annotation);
/* unblock all jobs that are waiting for this job and enqueue them */
while (chain)
{
td_node *n = chain->node;
if (is_failed(node))
n->job.failed_deps++;
/* nodes blocked on this node can't be completed yet */
assert(!is_completed(n));
if (0 == --n->job.block_count)
{
if (!is_queued(n))
enqueue(queue, n);
++qcount;
}
chain = chain->next;
}
if (1 < qcount)
pthread_cond_broadcast(&queue->work_avail);
else if (1 == qcount)
pthread_cond_signal(&queue->work_avail);
}
}
static int
is_up_to_date(td_job_queue *queue, td_node *node)
{
double t1 = 0.0, t2 = 0.0;
int collect_stats;
int i, count;
const td_digest *prev_signature = NULL;
td_engine *engine = queue->engine;
const td_ancestor_data *ancestor;
int up_to_date = 0;
collect_stats = td_debug_check(engine, TD_DEBUG_STATS);
if (collect_stats)
t1 = td_timestamp();
/* We can safely drop the build queue lock in here as no job state is accessed. */
td_mutex_unlock_or_die(&queue->mutex);
/* rebuild if any output files are missing */
for (i = 0, count = node->output_count; i < count; ++i)
{
td_file *file = node->outputs[i];
const td_stat *stat = td_stat_file(engine, file);
if (0 == (stat->flags & TD_STAT_EXISTS))
{
if (td_debug_check(engine, TD_DEBUG_REASON))
printf("%s: output file %s is missing\n", node->annotation, file->path);
goto leave;
}
}
if (NULL != (ancestor = node->ancestor_data))
prev_signature = &ancestor->input_signature;
/* rebuild if there is no stored signature */
if (!prev_signature)
{
if (td_debug_check(engine, TD_DEBUG_REASON))
printf("%s: no previous input signature\n", node->annotation);
goto leave;
}
/* rebuild if the job failed last time */
if (TD_JOB_FAILED == ancestor->job_result)
{
if (td_debug_check(engine, TD_DEBUG_REASON))
printf("%s: build failed last time\n", node->annotation);
goto leave;
}
/* rebuild if the input signatures have changed */
if (0 != memcmp(prev_signature->data, node->job.input_signature.data, sizeof(td_digest)))
{
if (td_debug_check(engine, TD_DEBUG_REASON))
printf("%s: input signature differs\n", node->annotation);
goto leave;
}
/* otherwise, the node is up to date */
up_to_date = 1;
leave:
if (collect_stats)
{
t2 = td_timestamp();
td_mutex_lock_or_die(engine->lock);
engine->stats.up2date_check_time += t2 - t1;
td_mutex_unlock_or_die(engine->lock);
}
td_mutex_lock_or_die(&queue->mutex);
return up_to_date;
}
static int
run_job(td_job_queue *queue, td_node *node, int job_id)
{
double t1, mkdir_time, cmd_time;
td_engine *engine = queue->engine;
int i, count, result, was_signalled = 0;
const char *command = node->action;
if (!command || '\0' == command[0])
return 0;
++queue->jobs_run;
pthread_mutex_unlock(&queue->mutex);
t1 = td_timestamp();
/* ensure directories for output files exist */
for (i = 0, count = node->output_count; i < count; ++i)
{
td_file *dir = td_parent_dir(engine, node->outputs[i]);
if (!dir)
continue;
if (0 != (result = ensure_dir_exists(engine, dir)))
goto leave;
}
mkdir_time = td_timestamp() - t1;
t1 = td_timestamp();
/* If the outputs of this node can't be overwritten; delete them now */
if (0 == (TD_NODE_OVERWRITE & node->flags))
{
delete_outputs(node);
touch_outputs(engine, node);
}
if (!engine->settings.dry_run)
{
result = td_exec(
command,
node->env_count,
node->env,
&was_signalled,
job_id,
td_verbosity_check(engine, 2),
td_verbosity_check(engine, 1) ? node->annotation : NULL);
}
else
result = 0;
cmd_time = td_timestamp() - t1;
if (0 != result)
{
td_mutex_lock_or_die(&queue->mutex);
/* Maintain a fail count so we can track why we stopped building if
* we're stopping after the first error. Otherwise it might appear as
* we succeeded. */
++queue->fail_count;
/* If the command failed or was signalled (e.g. Ctrl+C), abort the build */
if (was_signalled)
queue->siginfo.flag = -1;
else if (!engine->settings.continue_on_error)
queue->siginfo.flag = 1;
td_mutex_unlock_or_die(&queue->mutex);
}
/* If the build failed, and the node isn't set to keep all its output files
* in all possible cases (precious), then delete all output files as we
* can't assume anything about their state. */
if (0 != result && 0 == (TD_NODE_PRECIOUS & node->flags))
delete_outputs(node);
/* Mark all output files as dirty regardless of whether the build succeeded
* or not. If it succeeded, we must assume the build overwrote them.
* Otherwise, it's likely we've deleted them. In any case, touching them
* again isn't going to hurt anything.*/
touch_outputs(engine, node);
/* Update engine stats. */
if (td_debug_check(engine, TD_DEBUG_STATS))
{
td_mutex_lock_or_die(engine->stats_lock);
engine->stats.mkdir_time += mkdir_time;
engine->stats.build_time += cmd_time;
td_mutex_unlock_or_die(engine->stats_lock);
}
leave:
td_mutex_lock_or_die(&queue->mutex);
return result;
}
