int makeflow_catalog_summary(struct dag* d, char* name, batch_queue_type_t type, timestamp_t start){
struct dag_node *n;
dag_node_state_t state;
int tasks_completed = 0;
int tasks_aborted = 0;
int tasks_waiting = 0;
int tasks_running = 0;
int tasks_failed = 0;
for (n = d->nodes; n; n = n->next) {
state = n->state;
if (state == DAG_NODE_STATE_FAILED)
tasks_failed++;
else if (state == DAG_NODE_STATE_ABORTED)
tasks_aborted++;
else if (state == DAG_NODE_STATE_COMPLETE)
tasks_completed++;
else if(state == DAG_NODE_STATE_RUNNING)
tasks_running++;
else if(state == DAG_NODE_STATE_WAITING)
tasks_waiting++;
}
//transmit report here
char* host = CATALOG_HOST;
char username[USERNAME_MAX];
username_get(username);
const char* batch_type = batch_queue_type_to_string(type);
struct jx *j = jx_object(0);
jx_insert_string(j,"type","makeflow");
jx_insert_integer(j,"total",itable_size(d->node_table));
jx_insert_integer(j,"running",tasks_running);
jx_insert_integer(j,"waiting",tasks_waiting);
jx_insert_integer(j,"aborted",tasks_aborted);
jx_insert_integer(j,"completed",tasks_completed);
jx_insert_integer(j,"failed",tasks_failed);
jx_insert_string(j,"project",name);
jx_insert_string(j,"owner",username);
char* timestring = string_format("%" PRIu64 "", start);
jx_insert_string(j,"time_started",timestring);
jx_insert_string(j,"batch_type",batch_type);
//creates memory
char* text = jx_print_string(j);
int resp = catalog_query_send_update(host, text);
free(text);
free(timestring);
jx_delete(j);
return resp;//all good
}
struct jx *jx_copy( struct jx *j )
{
if(!j) return 0;
switch(j->type) {
case JX_NULL:
return jx_null();
case JX_DOUBLE:
return jx_double(j->u.double_value);
case JX_BOOLEAN:
return jx_boolean(j->u.boolean_value);
case JX_INTEGER:
return jx_integer(j->u.integer_value);
case JX_SYMBOL:
return jx_symbol(j->u.symbol_name);
case JX_STRING:
return jx_string(j->u.string_value);
case JX_ARRAY:
return jx_array(jx_item_copy(j->u.items));
case JX_OBJECT:
return jx_object(jx_pair_copy(j->u.pairs));
case JX_OPERATOR:
return jx_operator(j->u.oper.type,jx_copy(j->u.oper.left),jx_copy(j->u.oper.right));
case JX_FUNCTION:
return jx_function(j->u.func.function, jx_copy(j->u.func.arguments));
case JX_ERROR:
return jx_error(jx_copy(j->u.err));
}
/* not reachable, but some compilers complain. */
return 0;
}
struct jx * nvpair_to_jx( struct nvpair *nv )
{
struct jx *object = jx_object(0);
char *key;
char *value;
struct jx *jvalue;
long long integer_value;
double double_value;
nvpair_first_item(nv);
while(nvpair_next_item(nv,&key,&value)) {
if(!strcmp(value,"true")) {
jvalue = jx_boolean(1);
} else if(!strcmp(value,"false")) {
jvalue = jx_boolean(0);
} else if(!strcmp(value,"null")) {
jvalue = jx_null();
} else if(string_is_integer(value,&integer_value)) {
jvalue = jx_integer(integer_value);
} else if(string_is_float(value,&double_value)) {
jvalue = jx_double(double_value);
} else if(value[0]=='[' || value[0]=='{') {
jvalue = jx_parse_string(value);
if(!jvalue) jvalue = jx_string(value);
} else {
jvalue = jx_string(value);
}
jx_insert(object,jx_string(key),jvalue);
}
return object;
}
struct jx * jx_eval( struct jx *j, struct jx *context )
{
if(!j) return 0;
switch(j->type) {
case JX_SYMBOL:
if(context) {
struct jx *result = jx_lookup(context,j->u.symbol_name);
if(result) return jx_copy(result);
}
return jx_null();
case JX_DOUBLE:
case JX_BOOLEAN:
case JX_INTEGER:
case JX_STRING:
case JX_NULL:
return jx_copy(j);
case JX_ARRAY:
return jx_array(jx_eval_item(j->u.items,context));
case JX_OBJECT:
return jx_object(jx_eval_pair(j->u.pairs,context));
case JX_OPERATOR:
return jx_eval_operator(&j->u.oper,context);
}
/* not reachable, but some compilers complain. */
return 0;
}
struct jx *jx_merge(struct jx *j, ...) {
va_list ap;
va_start (ap, j);
struct jx *result = jx_object(NULL);
for (struct jx *next = j; jx_istype(next, JX_OBJECT); next = va_arg(ap, struct jx *)) {
for (struct jx_pair *p = next->u.pairs; p; p = p->next) {
jx_delete(jx_remove(result, p->key));
jx_insert(result, jx_copy(p->key), jx_copy(p->value));
}
}
va_end(ap);
return result;
}
struct jx * jx_parse( struct jx_parser *s )
{
jx_token_t t = jx_scan(s);
switch(t) {
case JX_TOKEN_EOF:
return 0;
case JX_TOKEN_LBRACE:
return jx_object(jx_parse_pair_list(s));
case JX_TOKEN_LBRACKET:
return jx_array(jx_parse_item_list(s));
case JX_TOKEN_STRING:
return jx_string(s->token);
case JX_TOKEN_INTEGER:
return jx_integer(s->integer_value);
case JX_TOKEN_DOUBLE:
return jx_double(s->double_value);
case JX_TOKEN_TRUE:
return jx_boolean(1);
case JX_TOKEN_FALSE:
return jx_boolean(0);
case JX_TOKEN_NULL:
return jx_null();
case JX_TOKEN_SYMBOL:
if(s->strict_mode) {
jx_parse_error(s,"symbols are not allowed in strict parsing mode");
return 0;
} else {
return jx_symbol(s->token);
}
case JX_TOKEN_RBRACE:
case JX_TOKEN_RBRACKET:
case JX_TOKEN_COMMA:
case JX_TOKEN_COLON:
case JX_TOKEN_ERROR:
jx_parse_error(s,"unexpected token");
return 0;
}
/*
We shouldn't get here, since all the token types
should be handled above. But just in case...
*/
jx_parse_error(s,"parse error");
return 0;
}
static void update_all_catalogs()
{
struct jx *j = jx_object(0);
jx_insert_string(j,"type","catalog");
jx_insert(j, jx_string("version"), jx_format("%d.%d.%d", CCTOOLS_VERSION_MAJOR, CCTOOLS_VERSION_MINOR, CCTOOLS_VERSION_MICRO));
jx_insert_string(j,"owner",owner);
jx_insert_integer(j,"starttime",starttime);
jx_insert_integer(j,"port",port);
jx_insert(j,
jx_string("url"),
jx_format("http://%s:%d",preferred_hostname,port)
);
char *text = jx_print_string(j);
jx_delete(j);
list_iterate(outgoing_host_list, (list_op_t) catalog_query_send_update, text);
free(text);
}
struct jx * dag_node_env_create( struct dag *d, struct dag_node *n )
{
struct dag_variable_lookup_set s = { d, n->category, n, NULL };
char *key;
struct jx *object = jx_object(0);
char *num_cores = dag_variable_lookup_string(RESOURCES_CORES, &s);
char *num_omp_threads = dag_variable_lookup_string("OMP_NUM_THREADS", &s);
if (num_cores && !num_omp_threads) {
// if number of cores is set, number of omp threads is not set,
// then we set number of omp threads to number of cores
jx_insert(object, jx_string("OMP_NUM_THREADS"), jx_string(num_cores));
} else if (num_omp_threads) {
// if number of omp threads is set, then we set number of cores
// to the number of omp threads
jx_insert(object, jx_string(RESOURCES_CORES), jx_string(num_omp_threads));
} else {
// if both number of cores and omp threads are not set, we
// set them to 1
jx_insert(object, jx_string("OMP_NUM_THREADS"), jx_string("1"));
jx_insert(object, jx_string(RESOURCES_CORES), jx_string("1"));
}
set_first_element(d->export_vars);
while((key = set_next_element(d->export_vars))) {
char *value = dag_variable_lookup_string(key, &s);
if(value) {
jx_insert(object,jx_string(key),jx_string(value));
debug(D_MAKEFLOW_RUN, "export %s=%s", key, value);
}
}
free(num_cores);
free(num_omp_threads);
return object;
}
int main(int argc, char *argv[]) {
char *host = CATALOG_HOST;
int port = CATALOG_PORT;
static const struct option long_options[] = {
{"catalog", required_argument, 0, 'c'},
{0,0,0,0}
};
signed int c;
while ((c = getopt_long(argc, argv, "c:", long_options, NULL)) > -1) {
switch (c) {
case 'c':
host = optarg;
break;
default:
show_help(argv[0]);
return EXIT_FAILURE;
}
}
struct datagram *d;
d = datagram_create(DATAGRAM_PORT_ANY);
if (!d) {
fatal("could not create datagram port!");
}
struct utsname name;
int cpus;
int uptime;
double load[3];
UINT64_T memory_total, memory_avail;
char owner[USERNAME_MAX];
uname(&name);
string_tolower(name.sysname);
string_tolower(name.machine);
string_tolower(name.release);
load_average_get(load);
cpus = load_average_get_cpus();
host_memory_info_get(&memory_avail, &memory_total);
uptime = uptime_get();
username_get(owner);
struct jx *j = jx_object(0);
jx_insert_string(j,"type","node");
jx_insert(j,jx_string("version"),jx_format("%d.%d.%d",CCTOOLS_VERSION_MAJOR, CCTOOLS_VERSION_MINOR, CCTOOLS_VERSION_MICRO));
jx_insert_string(j,"cpu",name.machine);
jx_insert_string(j,"opsys",name.sysname);
jx_insert_string(j,"opsysversion",name.release);
jx_insert_double(j,"load1",load[0]);
jx_insert_double(j,"load5",load[1]);
jx_insert_double(j,"load15",load[2]);
jx_insert_integer(j,"memory_total",memory_total);
jx_insert_integer(j,"memory_avail",memory_avail);
jx_insert_integer(j,"cpus",cpus);
jx_insert_integer(j,"uptime,",uptime);
jx_insert_string(j,"owner",owner);
int i;
for (i = optind; i < argc; i++) {
char *name;
char *value;
name = argv[i];
value = strchr(name, '=');
if (!value) {
fatal("invalid name/value pair = %s", name);
} else {
*value++ = 0;
}
jx_insert_string(j,name,value);
}
char *text = jx_print_string(j);
char address[DATAGRAM_ADDRESS_MAX];
if (domain_name_cache_lookup(host, address)) {
datagram_send(d, text, strlen(text), address, port);
} else {
fatal("unable to lookup address of host: %s", host);
}
jx_delete(j);
datagram_delete(d);
return EXIT_SUCCESS;
}
static int makeflow_module_sandbox_register(struct makeflow_hook *h, struct list *hooks, struct jx **args)
{
*args = jx_object(NULL);
return MAKEFLOW_HOOK_SUCCESS;
}
struct jx *rmsummary_to_json(struct rmsummary *s, int only_resources) {
struct jx *output = jx_object(NULL);
struct jx *array;
if(s->disk > -1) {
array = jx_arrayv(jx_integer(s->disk), jx_string("MB"), NULL);
jx_insert(output, jx_string("disk"), array);
}
if(s->total_files > -1)
jx_insert_integer(output, "total_files", s->total_files);
if(s->bytes_written > -1) {
array = jx_arrayv(jx_integer(s->bytes_written), jx_string("MB"), NULL);
jx_insert(output, jx_string("bytes_written"), array);
}
if(s->bytes_read > -1) {
array = jx_arrayv(jx_integer(s->bytes_read), jx_string("MB"), NULL);
jx_insert(output, jx_string("bytes_read"), array);
}
if(s->swap_memory > -1) {
array = jx_arrayv(jx_integer(s->swap_memory), jx_string("MB"), NULL);
jx_insert(output, jx_string("swap_memory"), array);
}
if(s->memory > -1) {
array = jx_arrayv(jx_integer(s->memory), jx_string("MB"), NULL);
jx_insert(output, jx_string("memory"), array);
}
if(s->virtual_memory > -1) {
array = jx_arrayv(jx_integer(s->virtual_memory), jx_string("MB"), NULL);
jx_insert(output, jx_string("virtual_memory"), array);
}
if(s->total_processes > -1)
jx_insert_integer(output, "total_processes", s->total_processes);
if(s->max_concurrent_processes > -1)
jx_insert_integer(output, "max_concurrent_processes", s->max_concurrent_processes);
if(s->cores > -1)
jx_insert_integer(output, "cores", s->cores);
if(s->cpu_time > -1) {
array = jx_arrayv(jx_integer(s->cpu_time), jx_string("us"), NULL);
jx_insert(output, jx_string("cpu_time"), array);
}
if(s->wall_time > -1) {
array = jx_arrayv(jx_integer(s->wall_time), jx_string("us"), NULL);
jx_insert(output, jx_string("wall_time"), array);
}
if(s->end > -1) {
array = jx_arrayv(jx_integer(s->end), jx_string("us"), NULL);
jx_insert(output, jx_string("end"), array);
}
if(s->start > -1) {
array = jx_arrayv(jx_integer(s->start), jx_string("us"), NULL);
jx_insert(output, jx_string("start"), array);
}
if(!only_resources) {
if(s->exit_type)
{
if( strcmp(s->exit_type, "signal") == 0 ) {
jx_insert_integer(output, "signal", s->signal);
} else if( strcmp(s->exit_type, "limits") == 0 ) {
if(s->limits_exceeded) {
struct jx *lim = rmsummary_to_json(s->limits_exceeded, 1);
jx_insert(output, jx_string("limits_exceeded"), lim);
}
jx_insert_string(output, "exit_type", "limits");
}
}
if(s->last_error)
jx_insert_integer(output, "last_error", s->last_error);
jx_insert_integer(output, "exit_status", s->exit_status);
if(s->command)
jx_insert_string(output, "command", s->command);
if(s->category)
jx_insert_string(output, "category", s->category);
}
return output;
}
/* Write the task and run info to the task directory
* These files are hardcoded to task_info and run_info */
static int makeflow_archive_write_task_info(struct archive_instance *a, struct dag_node *n, struct batch_task *t, char *archive_path) {
struct batch_file *f;
/* task_info :
* COMMAND: Tasks command that was run
* SRC_COMMAND: Origin node's command for reference
* SRC_LINE: Line of origin node in SRC_MAKEFLOW
* SRC_MAKEFLOW: ID of file for the original Makeflow stored in archive
* INPUT_FILES: Alphabetic list of input files checksum IDs
* OUTPUT_FILES: Alphabetic list of output file inner_names
*/
struct jx *task_jx = jx_object(NULL);
jx_insert(task_jx, jx_string("COMMAND"), jx_string(t->command));
jx_insert(task_jx, jx_string("SRC_COMMAND"), jx_string(n->command));
jx_insert(task_jx, jx_string("SRC_LINE"), jx_integer(n->linenum));
jx_insert(task_jx, jx_string("SRC_MAKEFLOW"), jx_string(a->source_makeflow));
struct jx * input_files = jx_object(NULL);
struct list_cursor *cur = list_cursor_create(t->input_files);
for(list_seek(cur, 0); list_get(cur, (void**)&f); list_next(cur)) {
/* Generate the file archive id (content based) if does not exist. */
char * id;
if(path_is_dir(f->inner_name) == 1){
f->hash = batch_file_generate_id_dir(f->inner_name);
id = xxstrdup(f->hash);
}
else{
id = batch_file_generate_id(f);
}
jx_insert(input_files, jx_string(f->inner_name), jx_string(id));
free(id);
}
list_cursor_destroy(cur);
jx_insert(task_jx, jx_string("INPUT_FILES"), input_files);
struct jx * output_files = jx_object(NULL);
cur = list_cursor_create(t->output_files);
for(list_seek(cur, 0); list_get(cur, (void**)&f); list_next(cur)) {
/* Generate the file archive id (content based) if does not exist. */
char * id;
if(path_is_dir(f->inner_name) == 1){
f->hash = batch_file_generate_id_dir(f->inner_name);
id = xxstrdup(f->hash);
}
else{
id = batch_file_generate_id(f);
}
jx_insert(output_files, jx_string(f->inner_name), jx_string(id));
free(id);
}
list_cursor_destroy(cur);
jx_insert(task_jx, jx_string("OUTPUT_FILES"), output_files);
char *task_info = string_format("%s/task_info", archive_path);
FILE *fp = fopen(task_info, "w");
if (fp == NULL) {
free(task_info);
debug(D_ERROR|D_MAKEFLOW_HOOK, "could not create task_info for node %d archive", n->nodeid);
return 0;
} else {
jx_pretty_print_stream(task_jx, fp);
}
fclose(fp);
free(task_info);
jx_delete(task_jx);
/* run_info :
* SUBMITTED : Time task was submitted
* STARTED : Time task was started
* FINISHED : Time task was completed
* EXIT_NORMALLY : 0 if abnormal exit, 1 is normal
* EXIT_CODE : Task's exit code
* EXIT_SIGNAL : Int value of signal if occurred
*/
struct jx * run_jx = jx_object(NULL);
jx_insert(run_jx, jx_string("SUBMITTED"), jx_integer(t->info->submitted));
jx_insert(run_jx, jx_string("STARTED"), jx_integer(t->info->started));
jx_insert(run_jx, jx_string("FINISHED"), jx_integer(t->info->finished));
jx_insert(run_jx, jx_string("EXIT_NORMAL"), jx_integer(t->info->exited_normally));
jx_insert(run_jx, jx_string("EXIT_CODE"), jx_integer(t->info->exit_code));
jx_insert(run_jx, jx_string("EXIT_SIGNAL"), jx_integer(t->info->exit_signal));
task_info = string_format("%s/run_info", archive_path);
fp = fopen(task_info, "w");
if (fp == NULL) {
free(task_info);
debug(D_ERROR|D_MAKEFLOW_HOOK, "could not create run_info for node %d archive", n->nodeid);
return 0;
} else {
jx_pretty_print_stream(run_jx, fp);
}
fclose(fp);
free(task_info);
jx_delete(run_jx);
return 1;
}
