int lexer_read_command(struct lexer *lx)
{
struct list *tokens = lexer_read_command_aux(lx);
struct token *t;
if(list_size(tokens) < 2) {
/* If the only token in the list is a NEWLINE, then this is an empty line. */
while((t = list_pop_head(tokens)))
lexer_free_token(t);
list_delete(tokens);
return 1;
}
/* Add command start marker.*/
lexer_push_token(lx, lexer_pack_token(lx, TOKEN_COMMAND));
/* Merge command tokens into main queue. */
/* First merge command modifiers, if any. */
list_first_item(tokens);
while((t = list_peek_head(tokens))) {
if(t->type == TOKEN_LITERAL &&
((strcmp(t->lexeme, "LOCAL") == 0) ||
(strcmp(t->lexeme, "MAKEFLOW") == 0) )) {
t = list_pop_head(tokens);
lexer_push_token(lx, t);
} else if(t->type == TOKEN_SPACE) {
//Discard spaces between modifiers.
t = list_pop_head(tokens);
lexer_free_token(t);
} else {
break;
}
}
/* Mark end of modifiers. */
lexer_push_token(lx, lexer_pack_token(lx, TOKEN_COMMAND_MOD_END));
/* Now merge tha actual command tokens */
/* Gives the number of actual command tokens, not taking into account command modifiers. */
int count = 0;
while((t = list_pop_head(tokens))) {
count++;
lexer_push_token(lx, t);
}
list_delete(tokens);
if(count < 1)
lexer_report_error(lx, "Command is empty.\n");
return count;
}
static int chirp_fs_confuga_init (const char url[CHIRP_PATH_MAX])
{
int rc;
int i;
CATCH_CONFUGA(confuga_connect(&C, url, list_peek_head(catalog_host_list)));
for (i = 0; i < CHIRP_FILESYSTEM_MAXFD; i++)
open_files[i].type = CHIRP_FS_CONFUGA_CLOSED;
PROLOGUE
}
struct token *lexer_peek_next_token(struct lexer *lx)
{
if(list_size(lx->token_queue) == 0) {
if(lx->eof)
return NULL;
lexer_read_line(lx);
return lexer_peek_next_token(lx);
}
struct token *head = list_peek_head(lx->token_queue);
return head;
}
void aspace_destroy(aspace_t *a)
{
area_t *area;
/* tear down all attached areas */
while((area = (area_t*) list_peek_head(&a->areas))){
area_destroy(a, area->rsrc.id);
}
rsrc_release(&a->rsrc);
/* release page directory and table(s) */
kfreepage(a->pdir);
kfreepage(a->ptab);
kfree(aspace_t, a);
}
/* Does a depth first search marking all reachable nodes (->visited = 1) */
void graph_explore_dfs(graph_t *g, graph_vertex_t *s, graph_vertex_t *e,
visit_func_t pre, visit_func_t post) {
/* clear graph search assets unless we're exploring the whole graph */
if (g) graph_clear_pathtrace(g);
/* just a simple stack for depth first */
list_t *stack = list_init(NULL, NULL);
list_push(stack, s);
while ((s = list_peek_head(stack))) {
int clock = 0;
list_node_t *it;
/* break if searching for a specific destination */
if (e && e == s) break;
/* back at the parent: call post-operations */
if (s->path.visited) {
s = list_pop(stack);
if (post) post(s);
s->path.post_clock = clock++;
continue;
}
s->path.visited = 1;
s->path.pre_clock = clock++;
if (pre) pre(s);
for (it = s->edges->last; it != NULL; it = it->prev) {
graph_vertex_t *v = ((graph_edge_t *)it->data)->connection;
/* avoid back edges */
if (!v->path.visited) {
v->path.prev = s;
v->path.distance = s->path.distance + 1;
list_push(stack, v);
}
}
}
list_destroy(stack);
}
struct cluster *nearest_neighbor_clustering(struct list *initial_clusters, double (*cmp)(struct cluster *, struct cluster *))
{
struct cluster *top, *closest, *subtop;
struct list *stack;
struct itable *active_clusters;
double dclosest, dsubtop;
int merge = 0;
list_first_item(initial_clusters);
top = list_next_item(initial_clusters);
/* Return immediately if top is NULL, or there is a unique
* initial cluster */
if(list_size(initial_clusters) < 2)
return top;
stack = list_create(0);
list_push_head(stack, top);
/* Add all of the initial clusters as active clusters. */
active_clusters = itable_create(0);
while( (top = list_next_item(initial_clusters)) )
itable_insert(active_clusters, (uintptr_t) top, (void *) top);
do
{
/* closest might be NULL if all of the clusters are in
* the stack now. subtop might be NULL if top was the
* only cluster in the stack */
top = list_pop_head( stack );
closest = cluster_nearest_neighbor(active_clusters, top, cmp);
subtop = list_peek_head( stack );
dclosest = -1;
dsubtop = -1;
if(closest)
dclosest = cluster_ward_distance(top, closest);
if(subtop)
dsubtop = cluster_ward_distance(top, subtop);
/* The nearest neighbor of top is either one of the
* remaining active clusters, or the second topmost
* cluster in the stack */
if( closest && subtop )
{
/* Use pointer address to systematically break ties. */
if(dclosest < dsubtop || ((dclosest == dsubtop) && (uintptr_t)closest < (uintptr_t)subtop))
merge = 0;
else
merge = 1;
}
else if( subtop )
merge = 1;
else if( closest )
merge = 0;
else
fatal("Zero clusters?\n"); //We should never reach here.
if(merge)
{
/* If the two topmost clusters in the stack are
* mutual nearest neighbors, merge them into a single
* cluster */
subtop = list_pop_head( stack );
list_push_head(stack, cluster_merge(top, subtop));
}
else
{
/* Otherwise, push the nearest neighbor of top to the
* stack */
itable_remove(active_clusters, (uintptr_t) closest);
list_push_head(stack, top);
list_push_head(stack, closest);
}
debug(D_DEBUG, "stack: %d active: %d closest: %lf subtop: %lf\n",
list_size(stack), itable_size(active_clusters), dclosest, dsubtop);
/* If there are no more active_clusters, but there is not
* a single cluster in the stack, we try again,
* converting the clusters in the stack into new active
* clusters. */
if(itable_size(active_clusters) == 0 && list_size(stack) > 3)
{
itable_delete(active_clusters);
return nearest_neighbor_clustering(stack, cmp);
}
}while( !(itable_size(active_clusters) == 0 && list_size(stack) == 1) );
/* top is now the root of a cluster hierarchy, of
* cluster->right, cluster->left. */
top = list_pop_head(stack);
list_delete(stack);
itable_delete(active_clusters);
return top;
}
int parser_eval(const expr_t *e, long double *r, hashtbl_t *vars) {
if (!e || !r) {
fprintf(stderr, "eval error: null expression or result var\n");
return 1;
}
/* load known functions */
static hashtbl_t *functions = NULL;
if (unlikely(functions == NULL)) {
functions = hashtbl_init(NULL, NULL);
register_functions(functions);
}
/* stash constants into whatever symtab we get */
if (unlikely(vars && !hashtbl_get(vars, "_stashed"))) {
register_constants(vars);
}
const list_t *l = (const list_t*)e;
const list_node_t *n = list_last(l);
list_t *args = list_init(free, NULL);
const symbol_t *s;
while (n && (s = (const symbol_t*)list_data(n))) {
long double *d = NULL, *v = NULL;
long double (*f)(list_t*, size_t);
switch (s->type) {
case stNumber:
d = (long double*)zmalloc(sizeof(long double));
*d = s->number;
list_push(args, d);
break;
case stVariable:
if (!vars) {
fprintf(stderr, "eval error: no symbol table\n");
list_destroy(args);
return 1;
}
if (!(v = (long double*)hashtbl_get(vars, s->variable))) {
fprintf(stderr, "eval error: uninitialized variable [%s]\n", s->variable);
list_destroy(args);
return 1;
}
d = (long double*)zmalloc(sizeof(long double));
*d = *v;
list_push(args, d);
break;
case stBinOperator:
/* rhs operand */
if (!(v = (long double*)list_pop(args))) {
fprintf(stderr, "eval error: missing rhs operand\n");
list_destroy(args);
return 1;
}
case stUniOperator:
/* lhs operand, don't pop it... use it to store the result too */
if (!(d = (long double*)list_peek_head(args))) {
fprintf(stderr, "eval error: missing lhs operand\n");
list_destroy(args);
return 1;
}
*d = semanter_operator(s->operator, *d, s->type == stBinOperator ? *v : 0.0);
free(v);
break;
case stFunction:
if (!(f = (long double(*)(list_t*, size_t))hashtbl_get(functions, s->func.name))) {
fprintf(stderr, "eval error: unknown function [%s]\n", s->func.name);
list_destroy(args);
return 1;
}
d = (long double*)zmalloc(sizeof(long double));
*d = f(args, s->func.nargs);
list_push(args, d);
break;
}
n = list_prev(n);
}
if (list_size(args) != 1) {
fprintf(stderr, "eval error: corrupt args stack\n");
list_destroy(args);
return 1;
}
long double *d = (long double*)list_peek_head(args);
*r = *d;
list_destroy(args);
return 0;
}
int master_main(const char *host, int port, const char *addr) {
time_t idle_stoptime;
struct link *master = NULL;
int num_workers, i;
struct mpi_queue_job **workers;
struct itable *active_jobs = itable_create(0);
struct itable *waiting_jobs = itable_create(0);
struct list *complete_jobs = list_create();
MPI_Comm_size(MPI_COMM_WORLD, &num_workers);
workers = malloc(num_workers * sizeof(*workers));
memset(workers, 0, num_workers * sizeof(*workers));
idle_stoptime = time(0) + idle_timeout;
while(!abort_flag) {
char line[MPI_QUEUE_LINE_MAX];
if(time(0) > idle_stoptime) {
if(master) {
printf("mpi master: gave up after waiting %ds to receive a task.\n", idle_timeout);
} else {
printf("mpi master: gave up after waiting %ds to connect to %s port %d.\n", idle_timeout, host, port);
}
break;
}
if(!master) {
char working_dir[MPI_QUEUE_LINE_MAX];
master = link_connect(addr, port, idle_stoptime);
if(!master) {
sleep(5);
continue;
}
link_tune(master, LINK_TUNE_INTERACTIVE);
link_readline(master, line, sizeof(line), time(0) + active_timeout);
memset(working_dir, 0, MPI_QUEUE_LINE_MAX);
if(sscanf(line, "workdir %s", working_dir) == 1) {
MPI_Bcast(working_dir, MPI_QUEUE_LINE_MAX, MPI_CHAR, 0, MPI_COMM_WORLD);
} else {
link_close(master);
master = NULL;
continue;
}
}
if(link_readline(master, line, sizeof(line), time(0) + short_timeout)) {
struct mpi_queue_operation *op;
int jobid, mode;
INT64_T length;
char path[MPI_QUEUE_LINE_MAX];
op = NULL;
debug(D_MPI, "received: %s\n", line);
if(!strcmp(line, "get results")) {
struct mpi_queue_job *job;
debug(D_MPI, "results requested: %d available\n", list_size(complete_jobs));
link_putfstring(master, "num results %d\n", time(0) + active_timeout, list_size(complete_jobs));
while(list_size(complete_jobs)) {
job = list_pop_head(complete_jobs);
link_putfstring(master, "result %d %d %d %lld\n", time(0) + active_timeout, job->jobid, job->status, job->result, job->output_length);
if(job->output_length) {
link_write(master, job->output, job->output_length, time(0)+active_timeout);
}
mpi_queue_job_delete(job);
}
} else if(sscanf(line, "work %d %lld", &jobid, &length)) {
op = malloc(sizeof(*op));
memset(op, 0, sizeof(*op));
op->type = MPI_QUEUE_OP_WORK;
op->buffer_length = length+1;
op->buffer = malloc(length+1);
op->buffer[op->buffer_length] = 0;
link_read(master, op->buffer, length, time(0) + active_timeout);
op->result = -1;
} else if(sscanf(line, "stat %d %s", &jobid, path) == 2) {
op = malloc(sizeof(*op));
memset(op, 0, sizeof(*op));
op->type = MPI_QUEUE_OP_STAT;
sprintf(op->args, "%s", path);
op->result = -1;
} else if(sscanf(line, "unlink %d %s", &jobid, path) == 2) {
op = malloc(sizeof(*op));
memset(op, 0, sizeof(*op));
op->type = MPI_QUEUE_OP_UNLINK;
sprintf(op->args, "%s", path);
op->result = -1;
} else if(sscanf(line, "mkdir %d %s %o", &jobid, path, &mode) == 3) {
op = malloc(sizeof(*op));
memset(op, 0, sizeof(*op));
op->type = MPI_QUEUE_OP_MKDIR;
sprintf(op->args, "%s %o", path, mode);
op->result = -1;
} else if(sscanf(line, "close %d", &jobid) == 1) {
op = malloc(sizeof(*op));
memset(op, 0, sizeof(*op));
op->type = MPI_QUEUE_OP_CLOSE;
op->result = -1;
// } else if(sscanf(line, "symlink %d %s %s", &jobid, path, filename) == 3) {
// } else if(sscanf(line, "put %d %s %lld %o", &jobid, filename, &length, &mode) == 4) {
// } else if(sscanf(line, "rget %d %s", &jobid, filename) == 2) {
// } else if(sscanf(line, "get %d %s", &jobid, filename) == 2) {
// } else if(sscanf(line, "thirdget %d %d %s %[^\n]", &jobid, &mode, filename, path) == 4) {
// } else if(sscanf(line, "thirdput %d %d %s %[^\n]", &jobid, &mode, filename, path) == 4) {
} else if(!strcmp(line, "exit")) {
break;
} else {
abort_flag = 1;
continue;
}
if(op) {
struct mpi_queue_job *job;
job = itable_lookup(active_jobs, jobid);
if(!job) {
job = itable_lookup(waiting_jobs, jobid);
}
if(!job) {
job = malloc(sizeof(*job));
memset(job, 0, sizeof(*job));
job->jobid = jobid;
job->operations = list_create();
job->status = MPI_QUEUE_JOB_WAITING;
job->worker_rank = -1;
itable_insert(waiting_jobs, jobid, job);
}
list_push_tail(job->operations, op);
}
idle_stoptime = time(0) + idle_timeout;
} else {
link_close(master);
master = 0;
sleep(5);
}
int num_waiting_jobs = itable_size(waiting_jobs);
int num_unvisited_jobs = itable_size(active_jobs);
for(i = 1; i < num_workers && (num_unvisited_jobs > 0 || num_waiting_jobs > 0); i++) {
struct mpi_queue_job *job;
struct mpi_queue_operation *op;
int flag = 0;
UINT64_T jobid;
if(!workers[i]) {
if(num_waiting_jobs) {
itable_firstkey(waiting_jobs);
itable_nextkey(waiting_jobs, &jobid, (void **)&job);
itable_remove(waiting_jobs, jobid);
itable_insert(active_jobs, jobid, job);
workers[i] = job;
num_waiting_jobs--;
job->worker_rank = i;
job->status = MPI_QUEUE_JOB_READY;
} else {
continue;
}
} else {
num_unvisited_jobs--;
if(workers[i]->status == MPI_QUEUE_JOB_BUSY) {
MPI_Test(&workers[i]->request, &flag, &workers[i]->mpi_status);
if(flag) {
op = list_pop_head(workers[i]->operations);
if(op->output_length) {
op->output_buffer = malloc(op->output_length);
MPI_Recv(op->output_buffer, op->output_length, MPI_BYTE, workers[i]->worker_rank, 0, MPI_COMM_WORLD, &workers[i]->mpi_status);
}
workers[i]->status = MPI_QUEUE_JOB_READY;
if(op->type == MPI_QUEUE_OP_WORK || op->result < 0) {
if(workers[i]->output)
free(workers[i]->output);
workers[i]->output = op->output_buffer;
op->output_buffer = NULL;
workers[i]->output_length = op->output_length;
workers[i]->result = op->result;
if(op->result < 0) {
workers[i]->status = MPI_QUEUE_JOB_FAILED | op->type;
op->type = MPI_QUEUE_OP_CLOSE;
list_push_head(workers[i]->operations, op);
op = NULL;
}
}
if(op) {
if(op->buffer)
free(op->buffer);
if(op->output_buffer)
free(op->output_buffer);
free(op);
}
}
}
}
if( workers[i]->status != MPI_QUEUE_JOB_BUSY && list_size(workers[i]->operations)) {
op = list_peek_head(workers[i]->operations);
if(op->type == MPI_QUEUE_OP_CLOSE) {
itable_remove(active_jobs, workers[i]->jobid);
list_push_tail(complete_jobs, workers[i]);
if(!(workers[i]->status & MPI_QUEUE_JOB_FAILED))
workers[i]->status = MPI_QUEUE_JOB_COMPLETE;
workers[i] = NULL;
i--;
continue;
}
MPI_Send(op, sizeof(*op), MPI_BYTE, workers[i]->worker_rank, 0, MPI_COMM_WORLD);
if(op->buffer_length) {
MPI_Send(op->buffer, op->buffer_length, MPI_BYTE, workers[i]->worker_rank, 0, MPI_COMM_WORLD);
free(op->buffer);
op->buffer_length = 0;
op->buffer = NULL;
}
MPI_Irecv(op, sizeof(*op), MPI_BYTE, workers[i]->worker_rank, 0, MPI_COMM_WORLD, &workers[i]->request);
workers[i]->status = MPI_QUEUE_JOB_BUSY;
}
}
}
/** Clean up waiting & complete jobs, send Exit commands to each worker */
if(!master) {
// If the master link hasn't been set up yet
// the workers will be waiting for the working directory
char line[MPI_QUEUE_LINE_MAX];
memset(line, 0, MPI_QUEUE_LINE_MAX);
MPI_Bcast(line, MPI_QUEUE_LINE_MAX, MPI_CHAR, 0, MPI_COMM_WORLD);
} else {
link_close(master);
}
for(i = 1; i < num_workers; i++) {
struct mpi_queue_operation *op, close;
memset(&close, 0, sizeof(close));
close.type = MPI_QUEUE_OP_EXIT;
if(workers[i]) {
if(workers[i]->status == MPI_QUEUE_JOB_BUSY) {
MPI_Wait(&workers[i]->request, &workers[i]->mpi_status);
op = list_peek_head(workers[i]->operations);
if(op->output_length) {
op->output_buffer = malloc(op->output_length);
MPI_Recv(op->output_buffer, op->output_length, MPI_BYTE, workers[i]->worker_rank, 0, MPI_COMM_WORLD, &workers[i]->mpi_status);
}
}
itable_remove(active_jobs, workers[i]->jobid);
list_push_tail(complete_jobs, workers[i]);
}
MPI_Send(&close, sizeof(close), MPI_BYTE, i, 0, MPI_COMM_WORLD);
}
itable_firstkey(waiting_jobs);
while(itable_size(waiting_jobs)) {
struct mpi_queue_job *job;
UINT64_T jobid;
itable_nextkey(waiting_jobs, &jobid, (void **)&job);
itable_remove(waiting_jobs, jobid);
list_push_tail(complete_jobs, job);
}
while(list_size(complete_jobs)) {
mpi_queue_job_delete(list_pop_head(complete_jobs));
}
MPI_Finalize();
return abort_flag;
}
