void *
td_page_alloc(td_alloc *alloc, size_t size)
{
int left = alloc->page_left;
int page = alloc->page_index;
char *addr;
/* Round to 16 bytes */
size = (size + 15) & ~15;
if (left < (int) size)
{
if (page == alloc->total_page_count)
td_croak("out of string page memory");
page = alloc->page_index = page + 1;
left = alloc->page_left = alloc->page_size;
alloc->pages[page] = malloc(alloc->page_size);
if (!alloc->pages[page])
td_croak("out of memory allocating string page");
}
addr = alloc->pages[page] + alloc->page_size - left;
alloc->page_left -= (int) size;
#ifndef NDEBUG
memset(addr, 0xcc, size);
#endif
return addr;
}
void
td_load_ancestors(td_engine *engine)
{
FILE* f;
int i, count;
long file_size;
size_t read_count;
if (NULL == (f = fopen(TD_ANCESTOR_FILE, "rb")))
{
if (td_debug_check(engine, TD_DEBUG_ANCESTORS))
printf("couldn't open %s; no ancestor information present\n", TD_ANCESTOR_FILE);
return;
}
fseek(f, 0, SEEK_END);
file_size = ftell(f);
rewind(f);
if (file_size % sizeof(td_ancestor_data) != 0)
td_croak("illegal ancestor file: %d not a multiple of %d bytes",
(int) file_size, sizeof(td_ancestor_data));
engine->ancestor_count = count = (int) (file_size / sizeof(td_ancestor_data));
engine->ancestors = malloc(file_size);
engine->ancestor_used = calloc(sizeof(td_node *), count);
read_count = fread(engine->ancestors, sizeof(td_ancestor_data), count, f);
if (td_debug_check(engine, TD_DEBUG_ANCESTORS))
printf("read %d ancestors\n", count);
if (read_count != (size_t) count)
td_croak("only read %d items, wanted %d", read_count, count);
for (i = 1; i < count; ++i)
{
int cmp = td_compare_ancestors(&engine->ancestors[i-1], &engine->ancestors[i]);
if (cmp == 0)
td_croak("bad ancestor file; duplicate item (%d/%d)", i, count);
if (cmp > 0)
td_croak("bad ancestor file; bad sort order on item (%d/%d)", i, count);
}
if (td_debug_check(engine, TD_DEBUG_ANCESTORS))
{
printf("full ancestor dump on load:\n");
for (i = 0; i < count; ++i)
{
char guid[33], sig[33];
td_digest_to_string(&engine->ancestors[i].guid, guid);
td_digest_to_string(&engine->ancestors[i].input_signature, sig);
printf("%s %s %ld %d\n", guid, sig, (long) engine->ancestors[i].access_time, engine->ancestors[i].job_result);
}
}
fclose(f);
}
static int
tokenize_path(char* scratch, strseg segments[], int maxseg)
{
int segcount = 0;
char *last = scratch;
for (;;)
{
char ch = *scratch;
if ('\\' == ch || '/' == ch || '\0' == ch)
{
int len = (int) (scratch - last);
int is_dotdot = 2 == len && 0 == memcmp("..", last, 2);
int is_dot = 1 == len && '.' == last[0];
if (segcount == maxseg)
td_croak("too many segments in path; limit is %d", maxseg);
segments[segcount].ptr = last;
segments[segcount].len = len;
segments[segcount].dotdot = is_dotdot;
segments[segcount].drop = is_dot;
last = scratch + 1;
++segcount;
if ('\0' == ch)
break;
}
++scratch;
}
return segcount;
}
void
td_build_path(char *buffer, int buffer_size, const td_file *base, const char *subpath, int subpath_len, td_build_path_mode mode)
{
int offset = 0;
int path_len = 0;
switch (mode)
{
case TD_BUILD_CONCAT:
path_len = base->path_len;
break;
case TD_BUILD_REPLACE_NAME:
path_len = (int) (base->name - base->path);
break;
default:
assert(0);
break;
}
if (path_len + subpath_len + 2 > buffer_size)
td_croak("combined path too long: %s -> include %s (limit: %d)", base->path, subpath, buffer_size);
memcpy(buffer, base->path, path_len);
offset = path_len;
if (path_len > 0 && buffer[offset-1] != '/' && buffer[offset-1] != '\\')
buffer[offset++] = TD_PATHSEP;
memcpy(buffer + offset, subpath, subpath_len);
offset += subpath_len;
buffer[offset] = '\0';
}
static int push_include(include_set *set, td_file *f)
{
int i, count;
for (i = 0, count = set->count; i < count; ++i)
{
if (f == set->files[i])
return 0;
}
if (TD_MAX_INCLUDES == set->count)
td_croak("too many includes");
set->files[set->count++] = f;
return 1;
}
static int
td_load_embedded_file(lua_State *L)
{
int i, count;
const char *module_name = luaL_checkstring(L, 1);
for (i = 0, count = td_lua_file_count; i < count; ++i)
{
if (0 == strcmp(td_lua_files[i].module_name, module_name))
{
if (0 != luaL_loadbuffer(L, td_lua_files[i].data, td_lua_files[i].size, module_name))
td_croak("couldn't load embedded chunk for %s", module_name);
return 1;
}
}
return 0;
}
void
td_setup_ancestor_data(td_engine *engine, td_node *node)
{
compute_node_guid(engine, node);
++engine->stats.ancestor_checks;
if (engine->ancestors)
{
td_ancestor_data key;
key.guid = node->guid; /* only key field is relevant */
node->ancestor_data = (td_ancestor_data *)
bsearch(&key, engine->ancestors, engine->ancestor_count, sizeof(td_ancestor_data), td_compare_ancestors);
if (node->ancestor_data)
{
int index = (int) (node->ancestor_data - engine->ancestors);
td_node *other;
if (NULL != (other = engine->ancestor_used[index]))
td_croak("node error: nodes \"%s\" and \"%s\" share the same ancestor", node->annotation, other->annotation);
engine->ancestor_used[index] = node;
++engine->stats.ancestor_nodes;
}
else
{
if (td_debug_check(engine, TD_DEBUG_ANCESTORS))
{
char guidstr[33];
td_digest_to_string(&node->guid, guidstr);
printf("no ancestor for %s with guid %s\n", node->annotation, guidstr);
}
}
}
else
{
/* We didn't load any ancestor data, just set the ancestor to NULL.
* Everything will rebuild without ancestry. */
node->ancestor_data = NULL;
}
}
void
td_save_ancestors(td_engine *engine, td_node *root)
{
FILE* f;
int i, count, max_count;
int output_cursor, write_count;
td_ancestor_data *output;
unsigned char *visited;
time_t now = time(NULL);
const int dbg = td_debug_check(engine, TD_DEBUG_ANCESTORS);
if (NULL == (f = fopen(TD_ANCESTOR_FILE ".tmp", "wb")))
{
fprintf(stderr, "warning: couldn't save ancestors\n");
return;
}
max_count = engine->node_count + engine->ancestor_count;
output = (td_ancestor_data *) malloc(sizeof(td_ancestor_data) * max_count);
visited = (unsigned char *) calloc(engine->ancestor_count, 1);
output_cursor = 0;
update_ancestors(engine, root, now, &output_cursor, output, visited);
if (dbg)
printf("refreshed %d ancestors\n", output_cursor);
for (i = 0, count = engine->ancestor_count; i < count; ++i)
{
const td_ancestor_data *a = &engine->ancestors[i];
if (!visited[i] && !ancestor_timed_out(a, now))
output[output_cursor++] = *a;
}
if (dbg)
printf("%d ancestors to save in total\n", output_cursor);
qsort(output, output_cursor, sizeof(td_ancestor_data), td_compare_ancestors);
if (dbg)
{
printf("full ancestor dump on save:\n");
for (i = 0; i < output_cursor; ++i)
{
char guid[33], sig[33];
td_digest_to_string(&output[i].guid, guid);
td_digest_to_string(&output[i].input_signature, sig);
printf("%s %s %ld %d\n", guid, sig, (long) output[i].access_time, output[i].job_result);
}
}
write_count = (int) fwrite(output, sizeof(td_ancestor_data), output_cursor, f);
fclose(f);
free(visited);
free(output);
if (write_count != output_cursor)
td_croak("couldn't write %d entries; only wrote %d", output_cursor, write_count);
if (0 != td_move_file(TD_ANCESTOR_FILE ".tmp", TD_ANCESTOR_FILE))
td_croak("couldn't rename %s to %s", TD_ANCESTOR_FILE ".tmp", TD_ANCESTOR_FILE);
}
static int
scan_file_data(td_alloc *scratch, td_file *file, td_include_data *out, int max_count, td_scanner *scanner)
{
FILE *f;
int file_count = 0;
int at_start_of_file = 1;
int at_end_of_file = 0;
char line_buffer[1024];
char *buffer_start = line_buffer;
int buffer_size = sizeof(line_buffer);
static const unsigned char utf8_mark[] = { 0xef, 0xbb, 0xbf };
td_scan_fn *line_scanner = scanner->scan_fn;
if (NULL == (f = fopen(file->path, "r")))
return 0;
for (; !at_end_of_file && buffer_size != 0 ;)
{
char *p, *line;
int count, remain;
count = (int) fread(buffer_start, 1, (int) buffer_size, f);
if (0 == count)
{
/* add an implicit newline at the end of the file, which is C++11 conformant and good practical behaviour */
buffer_start[0] = '\n';
count = 1;
at_end_of_file = 1;
}
/* skip past any UTF-8 bytemark, or isspace() and related functions trigger asserts in MSVC debug builds! */
if (at_start_of_file && count >= sizeof(utf8_mark) && 0 == memcmp(buffer_start, utf8_mark, sizeof(utf8_mark)))
{
memmove(buffer_start, buffer_start + sizeof(utf8_mark), count - sizeof(utf8_mark));
count -= sizeof(utf8_mark);
}
at_start_of_file = 0;
buffer_start += count;
line = line_buffer;
for (p = line_buffer; p < buffer_start; ++p)
{
if ('\n' == *p)
{
*p = 0;
if (file_count == max_count)
td_croak("%s: too many includes", file->path);
file_count += (*line_scanner)(scratch, line, &out[file_count], scanner);
line = p+1;
}
}
if (line > buffer_start)
line = buffer_start;
remain = (int) (buffer_start - line);
memmove(line_buffer, line, remain);
buffer_start = line_buffer + remain;
buffer_size = sizeof(line_buffer) - remain;
}
fclose(f);
return file_count;
}
td_build_result
td_build(td_engine *engine, td_node *node, int *jobs_run)
{
int i;
int thread_count;
td_job_queue queue;
if (engine->stats.build_called)
td_croak("build() called more than once on same engine");
if (0 != td_init_exec())
td_croak("couldn't initialize command execution");
engine->stats.build_called = 1;
memset(&queue, 0, sizeof(queue));
queue.engine = engine;
pthread_mutex_init(&queue.mutex, NULL);
pthread_cond_init(&queue.work_avail, NULL);
queue.array_size = engine->node_count;
queue.array = (td_node **) calloc(engine->node_count, sizeof(td_node*));
queue.siginfo.mutex = &queue.mutex;
queue.siginfo.cond = &queue.work_avail;
thread_count = engine->settings.thread_count;
if (thread_count > TD_MAX_THREADS)
thread_count = TD_MAX_THREADS;
if (td_debug_check(engine, TD_DEBUG_QUEUE))
printf("using %d build threads\n", thread_count);
td_block_signals(1);
td_install_sighandler(&queue.siginfo);
for (i = 0; i < thread_count-1; ++i)
{
int rc;
if (td_debug_check(engine, TD_DEBUG_QUEUE))
printf("starting thread %d\n", i);
thread_arg[i].job_id = i + 2;
thread_arg[i].queue = &queue;
rc = pthread_create(&threads[i], NULL, build_worker, &thread_arg[i]);
if (0 != rc)
td_croak("couldn't start thread %d: %s", i, strerror(rc));
}
pthread_mutex_lock(&queue.mutex);
node->job.flags |= TD_JOBF_ROOT;
enqueue(&queue, node);
pthread_mutex_unlock(&queue.mutex);
pthread_cond_broadcast(&queue.work_avail);
{
thread_start_arg main_arg;
main_arg.job_id = 1;
main_arg.queue = &queue;
build_worker(&main_arg);
}
if (td_verbosity_check(engine, 2) && -1 == queue.siginfo.flag)
printf("*** aborted on signal %s\n", queue.siginfo.reason);
for (i = thread_count - 2; i >= 0; --i)
{
void *res;
int rc = pthread_join(threads[i], &res);
if (0 != rc)
td_croak("couldn't join thread %d: %s", i, strerror(rc));
}
if (0 != queue.thread_count)
td_croak("threads are still running");
/* there is a tiny race condition here if a user presses Ctrl-C just
* before the write to the static pointer occurs, but that's in nanosecond
* land */
td_remove_sighandler();
free(queue.array);
pthread_cond_destroy(&queue.work_avail);
pthread_mutex_destroy(&queue.mutex);
*jobs_run = queue.jobs_run;
if (queue.siginfo.flag < 0)
return TD_BUILD_ABORTED;
else if (queue.fail_count)
return TD_BUILD_FAILED;
else
return TD_BUILD_SUCCESS;
}
static void
advance_job(td_job_queue *queue, td_node *node, int job_id)
{
td_jobstate state;
while ((state = node->job.state) < TD_JOB_COMPLETED)
{
switch (state)
{
case TD_JOB_INITIAL:
if (node->job.block_count > 0)
{
/* enqueue any blocking jobs and transition to the blocked state */
int i, count, bc = 0;
td_node **deps = node->deps;
for (i = 0, count = node->dep_count; i < count; ++i)
{
td_node *dep = deps[i];
if (!is_completed(dep))
{
++bc;
if (!is_queued(dep) && dep->job.state < TD_JOB_BLOCKED)
enqueue(queue, dep);
}
}
assert(bc == node->job.block_count);
transition_job(queue, node, TD_JOB_BLOCKED);
pthread_cond_broadcast(&queue->work_avail);
return;
}
else
{
/* nothing is blocking this job, so scan implicit deps immediately */
transition_job(queue, node, TD_JOB_SCANNING);
}
break;
case TD_JOB_BLOCKED:
assert(0 == node->job.block_count);
if (0 == node->job.failed_deps)
transition_job(queue, node, TD_JOB_SCANNING);
else
transition_job(queue, node, TD_JOB_FAILED);
break;
case TD_JOB_SCANNING:
if (0 == scan_implicit_deps(queue, node))
{
update_input_signature(queue, node);
if (is_up_to_date(queue, node))
transition_job(queue, node, TD_JOB_UPTODATE);
else
transition_job(queue, node, TD_JOB_RUNNING);
}
else
{
/* implicit dependency scanning failed */
transition_job(queue, node, TD_JOB_FAILED);
}
break;
case TD_JOB_RUNNING:
if (0 != run_job(queue, node, job_id))
transition_job(queue, node, TD_JOB_FAILED);
else
transition_job(queue, node, TD_JOB_COMPLETED);
break;
default:
assert(0);
td_croak("can't get here");
break;
}
}
if (is_completed(node))
{
int qcount = 0;
td_job_chain *chain = node->job.pending_jobs;
if (td_debug_check(queue->engine, TD_DEBUG_QUEUE))
printf("%s completed - enqueing blocked jobs\n", node->annotation);
/* unblock all jobs that are waiting for this job and enqueue them */
while (chain)
{
td_node *n = chain->node;
if (is_failed(node))
n->job.failed_deps++;
/* nodes blocked on this node can't be completed yet */
assert(!is_completed(n));
if (0 == --n->job.block_count)
{
if (!is_queued(n))
enqueue(queue, n);
++qcount;
}
chain = chain->next;
}
if (1 < qcount)
pthread_cond_broadcast(&queue->work_avail);
else if (1 == qcount)
pthread_cond_signal(&queue->work_avail);
}
}
int main(int argc, char** argv)
{
td_bin_allocator bin_alloc;
const char *homedir;
int res, rc, i;
lua_State* L;
td_init_portable();
#if !defined(TD_STANDALONE)
if (NULL == (homedir = td_init_homedir()))
return 1;
#else
homedir = "";
#endif
td_bin_allocator_init(&bin_alloc);
L = lua_newstate(td_lua_alloc, &bin_alloc);
if (!L)
exit(1);
lua_atpanic(L, on_lua_panic);
luaL_openlibs(L);
tundra_open(L);
#if defined(TD_STANDALONE)
/* this is equivalent to table.insert(package.loaders, 1, td_load_embedded_file) */
/* get the function */
lua_getglobal(L, "table");
lua_getfield(L, -1, "insert");
lua_remove(L, -2);
assert(!lua_isnil(L, -1));
/* arg1: the package.loaders table */
lua_getglobal(L, "package");
lua_getfield(L, -1, "loaders");
lua_remove(L, -2);
assert(!lua_isnil(L, -1));
lua_pushinteger(L, 1); /* arg 2 */
lua_pushcfunction(L, td_load_embedded_file); /* arg 3 */
lua_call(L, 3, 0);
#endif
/* setup package.path */
{
char ppath[1024];
snprintf(ppath, sizeof(ppath),
"%s" TD_PATHSEP_STR "scripts" TD_PATHSEP_STR "?.lua;"
"%s" TD_PATHSEP_STR "lua" TD_PATHSEP_STR "etc" TD_PATHSEP_STR "?.lua", homedir, homedir);
lua_getglobal(L, "package");
assert(LUA_TTABLE == lua_type(L, -1));
lua_pushstring(L, ppath);
lua_setfield(L, -2, "path");
}
/* push our error handler on the stack now (before the chunk to run) */
lua_pushcclosure(L, get_traceback, 0);
switch (luaL_loadbuffer(L, boot_snippet, sizeof(boot_snippet)-1, "boot_snippet"))
{
case LUA_ERRMEM:
td_croak("out of memory");
return 1;
case LUA_ERRSYNTAX:
td_croak("syntax error\n%s\n", lua_tostring(L, -1));
return 1;
}
lua_newtable(L);
lua_pushstring(L, homedir);
lua_rawseti(L, -2, 1);
for (i=1; i<argc; ++i)
{
lua_pushstring(L, argv[i]);
lua_rawseti(L, -2, i+1);
}
{
double t2;
script_call_t1 = td_timestamp();
res = lua_pcall(L, /*narg:*/1, /*nres:*/0, /*errorfunc:*/ -3);
t2 = td_timestamp();
if (global_tundra_stats)
printf("total time spent in tundra: %.4fs\n", t2 - script_call_t1);
}
if (res == 0)
{
rc = global_tundra_exit_code;
}
else
{
fprintf(stderr, "%s\n", lua_tostring(L, -1));
rc = 1;
}
lua_close(L);
td_bin_allocator_cleanup(&bin_alloc);
return rc;
}
td_file *
td_engine_get_file(td_engine *engine, const char *input_path, td_get_file_mode mode)
{
unsigned int hash;
int slot;
td_file *chain;
td_file *f;
int path_len;
char path[1024];
const char *out_path;
size_t input_path_len = strlen(input_path);
if (input_path_len >= sizeof(path))
td_croak("path too long: %s", input_path);
if (TD_COPY_STRING == mode)
{
if (isabspath(input_path))
{
strcpy(path, input_path);
}
else
{
snprintf(path, sizeof path, "%s%s", cwd, input_path);
}
sanitize_path(path, sizeof(path), input_path_len);
hash = (unsigned int) djb2_hash(path);
out_path = path;
}
else
{
hash = (unsigned int) djb2_hash(input_path);
out_path = input_path;
}
td_mutex_lock_or_die(engine->lock);
slot = (int) (hash % engine->file_hash_size);
chain = engine->file_hash[slot];
while (chain)
{
if (chain->hash == hash && 0 == strcmp(out_path, chain->path))
{
td_mutex_unlock_or_die(engine->lock);
return chain;
}
chain = chain->bucket_next;
}
++engine->stats.file_count;
f = td_page_alloc(&engine->alloc, sizeof(td_file));
memset(f, 0, sizeof(td_file));
f->path_len = path_len = (int) strlen(out_path);
if (TD_COPY_STRING == mode)
f->path = td_page_strdup(&engine->alloc, out_path, path_len);
else
f->path = out_path;
f->hash = hash;
f->name = find_basename(f->path, path_len);
f->bucket_next = engine->file_hash[slot];
f->signer = engine->default_signer;
f->stat_dirty = 1;
f->signature_dirty = 1;
f->frozen_relstring_index = ~(0u);
engine->file_hash[slot] = f;
td_mutex_unlock_or_die(engine->lock);
return f;
}
