struct jx * nvpair_to_jx( struct nvpair *nv )
{
struct jx *object = jx_object(0);
char *key;
char *value;
struct jx *jvalue;
long long integer_value;
double double_value;
nvpair_first_item(nv);
while(nvpair_next_item(nv,&key,&value)) {
if(!strcmp(value,"true")) {
jvalue = jx_boolean(1);
} else if(!strcmp(value,"false")) {
jvalue = jx_boolean(0);
} else if(!strcmp(value,"null")) {
jvalue = jx_null();
} else if(string_is_integer(value,&integer_value)) {
jvalue = jx_integer(integer_value);
} else if(string_is_float(value,&double_value)) {
jvalue = jx_double(double_value);
} else if(value[0]=='[' || value[0]=='{') {
jvalue = jx_parse_string(value);
if(!jvalue) jvalue = jx_string(value);
} else {
jvalue = jx_string(value);
}
jx_insert(object,jx_string(key),jvalue);
}
return object;
}
static void update_all_catalogs()
{
struct jx *j = jx_object(0);
jx_insert_string(j,"type","catalog");
jx_insert(j, jx_string("version"), jx_format("%d.%d.%d", CCTOOLS_VERSION_MAJOR, CCTOOLS_VERSION_MINOR, CCTOOLS_VERSION_MICRO));
jx_insert_string(j,"owner",owner);
jx_insert_integer(j,"starttime",starttime);
jx_insert_integer(j,"port",port);
jx_insert(j,
jx_string("url"),
jx_format("http://%s:%d",preferred_hostname,port)
);
char *text = jx_print_string(j);
jx_delete(j);
list_iterate(outgoing_host_list, (list_op_t) catalog_query_send_update, text);
free(text);
}
int jx_insert_unless_empty( struct jx *object, struct jx *key, struct jx *value ) {
switch(value->type) {
case JX_OBJECT:
case JX_ARRAY:
/* C99 says union members have the same start address, so
* just pick one, they're both pointers. */
if(value->u.pairs == NULL) {
jx_delete(key);
jx_delete(value);
return -1;
} else {
return jx_insert(object, key, value);
}
break;
default:
return jx_insert(object, key, value);
break;
}
}
struct jx * dag_node_env_create( struct dag *d, struct dag_node *n )
{
struct dag_variable_lookup_set s = { d, n->category, n, NULL };
char *key;
struct jx *object = jx_object(0);
char *num_cores = dag_variable_lookup_string(RESOURCES_CORES, &s);
char *num_omp_threads = dag_variable_lookup_string("OMP_NUM_THREADS", &s);
if (num_cores && !num_omp_threads) {
// if number of cores is set, number of omp threads is not set,
// then we set number of omp threads to number of cores
jx_insert(object, jx_string("OMP_NUM_THREADS"), jx_string(num_cores));
} else if (num_omp_threads) {
// if number of omp threads is set, then we set number of cores
// to the number of omp threads
jx_insert(object, jx_string(RESOURCES_CORES), jx_string(num_omp_threads));
} else {
// if both number of cores and omp threads are not set, we
// set them to 1
jx_insert(object, jx_string("OMP_NUM_THREADS"), jx_string("1"));
jx_insert(object, jx_string(RESOURCES_CORES), jx_string("1"));
}
set_first_element(d->export_vars);
while((key = set_next_element(d->export_vars))) {
char *value = dag_variable_lookup_string(key, &s);
if(value) {
jx_insert(object,jx_string(key),jx_string(value));
debug(D_MAKEFLOW_RUN, "export %s=%s", key, value);
}
}
free(num_cores);
free(num_omp_threads);
return object;
}
struct jx *jx_merge(struct jx *j, ...) {
va_list ap;
va_start (ap, j);
struct jx *result = jx_object(NULL);
for (struct jx *next = j; jx_istype(next, JX_OBJECT); next = va_arg(ap, struct jx *)) {
for (struct jx_pair *p = next->u.pairs; p; p = p->next) {
jx_delete(jx_remove(result, p->key));
jx_insert(result, jx_copy(p->key), jx_copy(p->value));
}
}
va_end(ap);
return result;
}
int deltadb_update_event( struct deltadb *db, const char *key, const char *name, struct jx *jvalue )
{
struct jx * jobject = hash_table_lookup(db->table,key);
if(!jobject) return 1;
struct jx *jname = jx_string(name);
jx_delete(jx_remove(jobject,jname));
jx_insert(jobject,jname,jvalue);
if(display_mode==MODE_STREAM) {
display_deferred_time(db);
char *str = jx_print_string(jvalue);
printf("U %s %s %s\n",key,name,str);
free(str);
}
return 1;
}
void rmsummary_print(FILE *stream, struct rmsummary *s, struct jx *verbatim_fields)
{
struct jx *jsum = rmsummary_to_json(s, 0);
if(verbatim_fields) {
if(!jx_istype(verbatim_fields, JX_OBJECT)) {
fatal("Vebatim fields are not a json object.");
}
struct jx_pair *head = verbatim_fields->u.pairs;
while(head) {
jx_insert(jsum, jx_copy(head->key), jx_copy(head->value));
head = head->next;
}
}
jx_pretty_print_stream(jsum, stream);
jx_delete(jsum);
}
static int log_replay( struct jx_database *db, const char *filename, time_t snapshot)
{
char line[LOG_LINE_MAX];
char value[LOG_LINE_MAX];
char name[LOG_LINE_MAX];
char key[LOG_LINE_MAX];
int n;
struct jx *jvalue, *jobject;
long long current = 0;
FILE *file = fopen(filename,"r");
if(!file) return 0;
while(fgets(line,sizeof(line),file)) {
if(line[0]=='C') {
n = sscanf(line,"C %s %[^\n]",key,value);
if(n==1) {
struct nvpair *nv = nvpair_create();
nvpair_parse_stream(nv,file);
jvalue = nvpair_to_jx(nv);
hash_table_insert(db->table,key,jvalue);
} else if(n==2) {
jvalue = jx_parse_string(value);
if(jvalue) {
hash_table_insert(db->table,key,jvalue);
} else {
corrupt_data(filename,line);
}
} else {
corrupt_data(filename,line);
continue;
}
} else if(line[0]=='D') {
n = sscanf(line,"D %s\n",key);
if(n!=1) {
corrupt_data(filename,line);
continue;
}
jx_delete(hash_table_remove(db->table,key));
} else if(line[0]=='U') {
n=sscanf(line,"U %s %s %[^\n],",key,name,value);
if(n!=3) {
corrupt_data(filename,line);
continue;
}
jobject = hash_table_lookup(db->table,key);
if(!jobject) {
corrupt_data(filename,line);
continue;
}
jvalue = jx_parse_string(value);
if(!jvalue) jvalue = jx_string(value);
struct jx *jname = jx_string(name);
jx_delete(jx_remove(jobject,jname));
jx_insert(jobject,jname,jvalue);
} else if(line[0]=='R') {
n=sscanf(line,"R %s %s",key,name);
if(n!=2) {
corrupt_data(filename,line);
continue;
}
jobject = hash_table_lookup(db->table,key);
if(!jobject) {
corrupt_data(filename,line);
continue;
}
struct jx *jname = jx_string(name);
jx_delete(jx_remove(jobject,jname));
jx_delete(jname);
} else if(line[0]=='T') {
n = sscanf(line,"T %lld",¤t);
if(n!=1) {
corrupt_data(filename,line);
continue;
}
if(current>snapshot) break;
} else if(line[0]=='\n') {
continue;
} else {
corrupt_data(filename,line);
}
}
fclose(file);
return 1;
}
int main(int argc, char *argv[]) {
char *host = CATALOG_HOST;
int port = CATALOG_PORT;
static const struct option long_options[] = {
{"catalog", required_argument, 0, 'c'},
{0,0,0,0}
};
signed int c;
while ((c = getopt_long(argc, argv, "c:", long_options, NULL)) > -1) {
switch (c) {
case 'c':
host = optarg;
break;
default:
show_help(argv[0]);
return EXIT_FAILURE;
}
}
struct datagram *d;
d = datagram_create(DATAGRAM_PORT_ANY);
if (!d) {
fatal("could not create datagram port!");
}
struct utsname name;
int cpus;
int uptime;
double load[3];
UINT64_T memory_total, memory_avail;
char owner[USERNAME_MAX];
uname(&name);
string_tolower(name.sysname);
string_tolower(name.machine);
string_tolower(name.release);
load_average_get(load);
cpus = load_average_get_cpus();
host_memory_info_get(&memory_avail, &memory_total);
uptime = uptime_get();
username_get(owner);
struct jx *j = jx_object(0);
jx_insert_string(j,"type","node");
jx_insert(j,jx_string("version"),jx_format("%d.%d.%d",CCTOOLS_VERSION_MAJOR, CCTOOLS_VERSION_MINOR, CCTOOLS_VERSION_MICRO));
jx_insert_string(j,"cpu",name.machine);
jx_insert_string(j,"opsys",name.sysname);
jx_insert_string(j,"opsysversion",name.release);
jx_insert_double(j,"load1",load[0]);
jx_insert_double(j,"load5",load[1]);
jx_insert_double(j,"load15",load[2]);
jx_insert_integer(j,"memory_total",memory_total);
jx_insert_integer(j,"memory_avail",memory_avail);
jx_insert_integer(j,"cpus",cpus);
jx_insert_integer(j,"uptime,",uptime);
jx_insert_string(j,"owner",owner);
int i;
for (i = optind; i < argc; i++) {
char *name;
char *value;
name = argv[i];
value = strchr(name, '=');
if (!value) {
fatal("invalid name/value pair = %s", name);
} else {
*value++ = 0;
}
jx_insert_string(j,name,value);
}
char *text = jx_print_string(j);
char address[DATAGRAM_ADDRESS_MAX];
if (domain_name_cache_lookup(host, address)) {
datagram_send(d, text, strlen(text), address, port);
} else {
fatal("unable to lookup address of host: %s", host);
}
jx_delete(j);
datagram_delete(d);
return EXIT_SUCCESS;
}
void jx_insert_string( struct jx *j, const char *key, const char *value )
{
jx_insert(j,jx_string(key),jx_string(value));
}
void jx_insert_double( struct jx *j, const char *key, double value )
{
jx_insert(j,jx_string(key),jx_double(value));
}
void jx_insert_integer( struct jx *j, const char *key, jx_int_t value )
{
jx_insert(j,jx_string(key),jx_integer(value));
}
struct jx *rmsummary_to_json(struct rmsummary *s, int only_resources) {
struct jx *output = jx_object(NULL);
struct jx *array;
if(s->disk > -1) {
array = jx_arrayv(jx_integer(s->disk), jx_string("MB"), NULL);
jx_insert(output, jx_string("disk"), array);
}
if(s->total_files > -1)
jx_insert_integer(output, "total_files", s->total_files);
if(s->bytes_written > -1) {
array = jx_arrayv(jx_integer(s->bytes_written), jx_string("MB"), NULL);
jx_insert(output, jx_string("bytes_written"), array);
}
if(s->bytes_read > -1) {
array = jx_arrayv(jx_integer(s->bytes_read), jx_string("MB"), NULL);
jx_insert(output, jx_string("bytes_read"), array);
}
if(s->swap_memory > -1) {
array = jx_arrayv(jx_integer(s->swap_memory), jx_string("MB"), NULL);
jx_insert(output, jx_string("swap_memory"), array);
}
if(s->memory > -1) {
array = jx_arrayv(jx_integer(s->memory), jx_string("MB"), NULL);
jx_insert(output, jx_string("memory"), array);
}
if(s->virtual_memory > -1) {
array = jx_arrayv(jx_integer(s->virtual_memory), jx_string("MB"), NULL);
jx_insert(output, jx_string("virtual_memory"), array);
}
if(s->total_processes > -1)
jx_insert_integer(output, "total_processes", s->total_processes);
if(s->max_concurrent_processes > -1)
jx_insert_integer(output, "max_concurrent_processes", s->max_concurrent_processes);
if(s->cores > -1)
jx_insert_integer(output, "cores", s->cores);
if(s->cpu_time > -1) {
array = jx_arrayv(jx_integer(s->cpu_time), jx_string("us"), NULL);
jx_insert(output, jx_string("cpu_time"), array);
}
if(s->wall_time > -1) {
array = jx_arrayv(jx_integer(s->wall_time), jx_string("us"), NULL);
jx_insert(output, jx_string("wall_time"), array);
}
if(s->end > -1) {
array = jx_arrayv(jx_integer(s->end), jx_string("us"), NULL);
jx_insert(output, jx_string("end"), array);
}
if(s->start > -1) {
array = jx_arrayv(jx_integer(s->start), jx_string("us"), NULL);
jx_insert(output, jx_string("start"), array);
}
if(!only_resources) {
if(s->exit_type)
{
if( strcmp(s->exit_type, "signal") == 0 ) {
jx_insert_integer(output, "signal", s->signal);
} else if( strcmp(s->exit_type, "limits") == 0 ) {
if(s->limits_exceeded) {
struct jx *lim = rmsummary_to_json(s->limits_exceeded, 1);
jx_insert(output, jx_string("limits_exceeded"), lim);
}
jx_insert_string(output, "exit_type", "limits");
}
}
if(s->last_error)
jx_insert_integer(output, "last_error", s->last_error);
jx_insert_integer(output, "exit_status", s->exit_status);
if(s->command)
jx_insert_string(output, "command", s->command);
if(s->category)
jx_insert_string(output, "category", s->category);
}
return output;
}
/* Write the task and run info to the task directory
* These files are hardcoded to task_info and run_info */
static int makeflow_archive_write_task_info(struct archive_instance *a, struct dag_node *n, struct batch_task *t, char *archive_path) {
struct batch_file *f;
/* task_info :
* COMMAND: Tasks command that was run
* SRC_COMMAND: Origin node's command for reference
* SRC_LINE: Line of origin node in SRC_MAKEFLOW
* SRC_MAKEFLOW: ID of file for the original Makeflow stored in archive
* INPUT_FILES: Alphabetic list of input files checksum IDs
* OUTPUT_FILES: Alphabetic list of output file inner_names
*/
struct jx *task_jx = jx_object(NULL);
jx_insert(task_jx, jx_string("COMMAND"), jx_string(t->command));
jx_insert(task_jx, jx_string("SRC_COMMAND"), jx_string(n->command));
jx_insert(task_jx, jx_string("SRC_LINE"), jx_integer(n->linenum));
jx_insert(task_jx, jx_string("SRC_MAKEFLOW"), jx_string(a->source_makeflow));
struct jx * input_files = jx_object(NULL);
struct list_cursor *cur = list_cursor_create(t->input_files);
for(list_seek(cur, 0); list_get(cur, (void**)&f); list_next(cur)) {
/* Generate the file archive id (content based) if does not exist. */
char * id;
if(path_is_dir(f->inner_name) == 1){
f->hash = batch_file_generate_id_dir(f->inner_name);
id = xxstrdup(f->hash);
}
else{
id = batch_file_generate_id(f);
}
jx_insert(input_files, jx_string(f->inner_name), jx_string(id));
free(id);
}
list_cursor_destroy(cur);
jx_insert(task_jx, jx_string("INPUT_FILES"), input_files);
struct jx * output_files = jx_object(NULL);
cur = list_cursor_create(t->output_files);
for(list_seek(cur, 0); list_get(cur, (void**)&f); list_next(cur)) {
/* Generate the file archive id (content based) if does not exist. */
char * id;
if(path_is_dir(f->inner_name) == 1){
f->hash = batch_file_generate_id_dir(f->inner_name);
id = xxstrdup(f->hash);
}
else{
id = batch_file_generate_id(f);
}
jx_insert(output_files, jx_string(f->inner_name), jx_string(id));
free(id);
}
list_cursor_destroy(cur);
jx_insert(task_jx, jx_string("OUTPUT_FILES"), output_files);
char *task_info = string_format("%s/task_info", archive_path);
FILE *fp = fopen(task_info, "w");
if (fp == NULL) {
free(task_info);
debug(D_ERROR|D_MAKEFLOW_HOOK, "could not create task_info for node %d archive", n->nodeid);
return 0;
} else {
jx_pretty_print_stream(task_jx, fp);
}
fclose(fp);
free(task_info);
jx_delete(task_jx);
/* run_info :
* SUBMITTED : Time task was submitted
* STARTED : Time task was started
* FINISHED : Time task was completed
* EXIT_NORMALLY : 0 if abnormal exit, 1 is normal
* EXIT_CODE : Task's exit code
* EXIT_SIGNAL : Int value of signal if occurred
*/
struct jx * run_jx = jx_object(NULL);
jx_insert(run_jx, jx_string("SUBMITTED"), jx_integer(t->info->submitted));
jx_insert(run_jx, jx_string("STARTED"), jx_integer(t->info->started));
jx_insert(run_jx, jx_string("FINISHED"), jx_integer(t->info->finished));
jx_insert(run_jx, jx_string("EXIT_NORMAL"), jx_integer(t->info->exited_normally));
jx_insert(run_jx, jx_string("EXIT_CODE"), jx_integer(t->info->exit_code));
jx_insert(run_jx, jx_string("EXIT_SIGNAL"), jx_integer(t->info->exit_signal));
task_info = string_format("%s/run_info", archive_path);
fp = fopen(task_info, "w");
if (fp == NULL) {
free(task_info);
debug(D_ERROR|D_MAKEFLOW_HOOK, "could not create run_info for node %d archive", n->nodeid);
return 0;
} else {
jx_pretty_print_stream(run_jx, fp);
}
fclose(fp);
free(task_info);
jx_delete(run_jx);
return 1;
}
