struct jx * nvpair_to_jx( struct nvpair *nv ) { struct jx *object = jx_object(0); char *key; char *value; struct jx *jvalue; long long integer_value; double double_value; nvpair_first_item(nv); while(nvpair_next_item(nv,&key,&value)) { if(!strcmp(value,"true")) { jvalue = jx_boolean(1); } else if(!strcmp(value,"false")) { jvalue = jx_boolean(0); } else if(!strcmp(value,"null")) { jvalue = jx_null(); } else if(string_is_integer(value,&integer_value)) { jvalue = jx_integer(integer_value); } else if(string_is_float(value,&double_value)) { jvalue = jx_double(double_value); } else if(value[0]=='[' || value[0]=='{') { jvalue = jx_parse_string(value); if(!jvalue) jvalue = jx_string(value); } else { jvalue = jx_string(value); } jx_insert(object,jx_string(key),jvalue); } return object; }
struct jx *jx_copy( struct jx *j ) { if(!j) return 0; switch(j->type) { case JX_NULL: return jx_null(); case JX_DOUBLE: return jx_double(j->u.double_value); case JX_BOOLEAN: return jx_boolean(j->u.boolean_value); case JX_INTEGER: return jx_integer(j->u.integer_value); case JX_SYMBOL: return jx_symbol(j->u.symbol_name); case JX_STRING: return jx_string(j->u.string_value); case JX_ARRAY: return jx_array(jx_item_copy(j->u.items)); case JX_OBJECT: return jx_object(jx_pair_copy(j->u.pairs)); case JX_OPERATOR: return jx_operator(j->u.oper.type,jx_copy(j->u.oper.left),jx_copy(j->u.oper.right)); case JX_FUNCTION: return jx_function(j->u.func.function, jx_copy(j->u.func.arguments)); case JX_ERROR: return jx_error(jx_copy(j->u.err)); } /* not reachable, but some compilers complain. */ return 0; }
static void update_all_catalogs() { struct jx *j = jx_object(0); jx_insert_string(j,"type","catalog"); jx_insert(j, jx_string("version"), jx_format("%d.%d.%d", CCTOOLS_VERSION_MAJOR, CCTOOLS_VERSION_MINOR, CCTOOLS_VERSION_MICRO)); jx_insert_string(j,"owner",owner); jx_insert_integer(j,"starttime",starttime); jx_insert_integer(j,"port",port); jx_insert(j, jx_string("url"), jx_format("http://%s:%d",preferred_hostname,port) ); char *text = jx_print_string(j); jx_delete(j); list_iterate(outgoing_host_list, (list_op_t) catalog_query_send_update, text); free(text); }
struct jx * dag_node_env_create( struct dag *d, struct dag_node *n ) { struct dag_variable_lookup_set s = { d, n->category, n, NULL }; char *key; struct jx *object = jx_object(0); char *num_cores = dag_variable_lookup_string(RESOURCES_CORES, &s); char *num_omp_threads = dag_variable_lookup_string("OMP_NUM_THREADS", &s); if (num_cores && !num_omp_threads) { // if number of cores is set, number of omp threads is not set, // then we set number of omp threads to number of cores jx_insert(object, jx_string("OMP_NUM_THREADS"), jx_string(num_cores)); } else if (num_omp_threads) { // if number of omp threads is set, then we set number of cores // to the number of omp threads jx_insert(object, jx_string(RESOURCES_CORES), jx_string(num_omp_threads)); } else { // if both number of cores and omp threads are not set, we // set them to 1 jx_insert(object, jx_string("OMP_NUM_THREADS"), jx_string("1")); jx_insert(object, jx_string(RESOURCES_CORES), jx_string("1")); } set_first_element(d->export_vars); while((key = set_next_element(d->export_vars))) { char *value = dag_variable_lookup_string(key, &s); if(value) { jx_insert(object,jx_string(key),jx_string(value)); debug(D_MAKEFLOW_RUN, "export %s=%s", key, value); } } free(num_cores); free(num_omp_threads); return object; }
struct jx * jx_parse( struct jx_parser *s ) { jx_token_t t = jx_scan(s); switch(t) { case JX_TOKEN_EOF: return 0; case JX_TOKEN_LBRACE: return jx_object(jx_parse_pair_list(s)); case JX_TOKEN_LBRACKET: return jx_array(jx_parse_item_list(s)); case JX_TOKEN_STRING: return jx_string(s->token); case JX_TOKEN_INTEGER: return jx_integer(s->integer_value); case JX_TOKEN_DOUBLE: return jx_double(s->double_value); case JX_TOKEN_TRUE: return jx_boolean(1); case JX_TOKEN_FALSE: return jx_boolean(0); case JX_TOKEN_NULL: return jx_null(); case JX_TOKEN_SYMBOL: if(s->strict_mode) { jx_parse_error(s,"symbols are not allowed in strict parsing mode"); return 0; } else { return jx_symbol(s->token); } case JX_TOKEN_RBRACE: case JX_TOKEN_RBRACKET: case JX_TOKEN_COMMA: case JX_TOKEN_COLON: case JX_TOKEN_ERROR: jx_parse_error(s,"unexpected token"); return 0; } /* We shouldn't get here, since all the token types should be handled above. But just in case... */ jx_parse_error(s,"parse error"); return 0; }
int deltadb_remove_event( struct deltadb *db, const char *key, const char *name ) { struct jx *jobject = hash_table_lookup(db->table,key); if(!jobject) return 1; struct jx *jname = jx_string(name); jx_delete(jx_remove(jobject,jname)); jx_delete(jname); if(display_mode==MODE_STREAM) { display_deferred_time(db); printf("R %s %s\n",key,name); return 1; } return 1; }
int deltadb_update_event( struct deltadb *db, const char *key, const char *name, struct jx *jvalue ) { struct jx * jobject = hash_table_lookup(db->table,key); if(!jobject) return 1; struct jx *jname = jx_string(name); jx_delete(jx_remove(jobject,jname)); jx_insert(jobject,jname,jvalue); if(display_mode==MODE_STREAM) { display_deferred_time(db); char *str = jx_print_string(jvalue); printf("U %s %s %s\n",key,name,str); free(str); } return 1; }
static int log_replay( struct jx_database *db, const char *filename, time_t snapshot) { char line[LOG_LINE_MAX]; char value[LOG_LINE_MAX]; char name[LOG_LINE_MAX]; char key[LOG_LINE_MAX]; int n; struct jx *jvalue, *jobject; long long current = 0; FILE *file = fopen(filename,"r"); if(!file) return 0; while(fgets(line,sizeof(line),file)) { if(line[0]=='C') { n = sscanf(line,"C %s %[^\n]",key,value); if(n==1) { struct nvpair *nv = nvpair_create(); nvpair_parse_stream(nv,file); jvalue = nvpair_to_jx(nv); hash_table_insert(db->table,key,jvalue); } else if(n==2) { jvalue = jx_parse_string(value); if(jvalue) { hash_table_insert(db->table,key,jvalue); } else { corrupt_data(filename,line); } } else { corrupt_data(filename,line); continue; } } else if(line[0]=='D') { n = sscanf(line,"D %s\n",key); if(n!=1) { corrupt_data(filename,line); continue; } jx_delete(hash_table_remove(db->table,key)); } else if(line[0]=='U') { n=sscanf(line,"U %s %s %[^\n],",key,name,value); if(n!=3) { corrupt_data(filename,line); continue; } jobject = hash_table_lookup(db->table,key); if(!jobject) { corrupt_data(filename,line); continue; } jvalue = jx_parse_string(value); if(!jvalue) jvalue = jx_string(value); struct jx *jname = jx_string(name); jx_delete(jx_remove(jobject,jname)); jx_insert(jobject,jname,jvalue); } else if(line[0]=='R') { n=sscanf(line,"R %s %s",key,name); if(n!=2) { corrupt_data(filename,line); continue; } jobject = hash_table_lookup(db->table,key); if(!jobject) { corrupt_data(filename,line); continue; } struct jx *jname = jx_string(name); jx_delete(jx_remove(jobject,jname)); jx_delete(jname); } else if(line[0]=='T') { n = sscanf(line,"T %lld",¤t); if(n!=1) { corrupt_data(filename,line); continue; } if(current>snapshot) break; } else if(line[0]=='\n') { continue; } else { corrupt_data(filename,line); } } fclose(file); return 1; }
int main(int argc, char *argv[]) { char *host = CATALOG_HOST; int port = CATALOG_PORT; static const struct option long_options[] = { {"catalog", required_argument, 0, 'c'}, {0,0,0,0} }; signed int c; while ((c = getopt_long(argc, argv, "c:", long_options, NULL)) > -1) { switch (c) { case 'c': host = optarg; break; default: show_help(argv[0]); return EXIT_FAILURE; } } struct datagram *d; d = datagram_create(DATAGRAM_PORT_ANY); if (!d) { fatal("could not create datagram port!"); } struct utsname name; int cpus; int uptime; double load[3]; UINT64_T memory_total, memory_avail; char owner[USERNAME_MAX]; uname(&name); string_tolower(name.sysname); string_tolower(name.machine); string_tolower(name.release); load_average_get(load); cpus = load_average_get_cpus(); host_memory_info_get(&memory_avail, &memory_total); uptime = uptime_get(); username_get(owner); struct jx *j = jx_object(0); jx_insert_string(j,"type","node"); jx_insert(j,jx_string("version"),jx_format("%d.%d.%d",CCTOOLS_VERSION_MAJOR, CCTOOLS_VERSION_MINOR, CCTOOLS_VERSION_MICRO)); jx_insert_string(j,"cpu",name.machine); jx_insert_string(j,"opsys",name.sysname); jx_insert_string(j,"opsysversion",name.release); jx_insert_double(j,"load1",load[0]); jx_insert_double(j,"load5",load[1]); jx_insert_double(j,"load15",load[2]); jx_insert_integer(j,"memory_total",memory_total); jx_insert_integer(j,"memory_avail",memory_avail); jx_insert_integer(j,"cpus",cpus); jx_insert_integer(j,"uptime,",uptime); jx_insert_string(j,"owner",owner); int i; for (i = optind; i < argc; i++) { char *name; char *value; name = argv[i]; value = strchr(name, '='); if (!value) { fatal("invalid name/value pair = %s", name); } else { *value++ = 0; } jx_insert_string(j,name,value); } char *text = jx_print_string(j); char address[DATAGRAM_ADDRESS_MAX]; if (domain_name_cache_lookup(host, address)) { datagram_send(d, text, strlen(text), address, port); } else { fatal("unable to lookup address of host: %s", host); } jx_delete(j); datagram_delete(d); return EXIT_SUCCESS; }
static void handle_update( const char *addr, int port, const char *raw_data, int raw_data_length, const char *protocol ) { char key[LINE_MAX]; unsigned long data_length; struct jx *j; // If the packet starts with Control-Z (0x1A), it is compressed, // so uncompress it to data[]. Otherwise just copy to data[];. if(raw_data[0]==0x1A) { data_length = sizeof(data); int success = uncompress((Bytef*)data,&data_length,(const Bytef*)&raw_data[1],raw_data_length-1); if(success!=Z_OK) { debug(D_DEBUG,"warning: %s:%d sent invalid compressed data (ignoring it)\n",addr,port); return; } } else { memcpy(data,raw_data,raw_data_length); data_length = raw_data_length; } // Make sure the string data is null terminated. data[data_length] = 0; // Once uncompressed, if it starts with a bracket, // then it is JX/JSON, otherwise it is the legacy nvpair format. if(data[0]=='{') { j = jx_parse_string(data); if(!j) { debug(D_DEBUG,"warning: %s:%d sent invalid JSON data (ignoring it)\n%s\n",addr,port,data); return; } if(!jx_is_constant(j)) { debug(D_DEBUG,"warning: %s:%d sent non-constant JX data (ignoring it)\n%s\n",addr,port,data); jx_delete(j); return; } } else { struct nvpair *nv = nvpair_create(); if(!nv) return; nvpair_parse(nv, data); j = nvpair_to_jx(nv); nvpair_delete(nv); } jx_insert_string(j, "address", addr); jx_insert_integer(j, "lastheardfrom", time(0)); /* If the server reports unbelievable numbers, simply reset them */ if(max_server_size > 0) { INT64_T total = jx_lookup_integer(j, "total"); INT64_T avail = jx_lookup_integer(j, "avail"); if(total > max_server_size || avail > max_server_size) { jx_insert_integer(j, "total", max_server_size); jx_insert_integer(j, "avail", max_server_size); } } /* Do not believe the server's reported name, just resolve it backwards. */ char name[DOMAIN_NAME_MAX]; if(domain_name_cache_lookup_reverse(addr, name)) { /* Special case: Prior bug resulted in multiple name entries in logged data. When removing the name property, keep looking until all items are removed. */ struct jx *jname = jx_string("name"); struct jx *n; while((n=jx_remove(j,jname))) { jx_delete(n); } jx_delete(jname); jx_insert_string(j,"name",name); } else if (jx_lookup_string(j, "name") == NULL) { /* If rDNS is unsuccessful, then we use the name reported if given. * This allows for hostnames that are only valid in the subnet of * the reporting server. Here we set the "name" field to the IP * Address, addr, because it was not set by the reporting server. */ jx_insert_string(j, "name", addr); } make_hash_key(j, key); if(logfile) { if(!jx_database_lookup(table,key)) { jx_print_stream(j,logfile); fprintf(logfile,"\n"); fflush(logfile); } } jx_database_insert(table, key, j); debug(D_DEBUG, "received %s update from %s",protocol,key); }
/* * Obtains information from the Catalog, format it, and make return it to user. */ int main(int argc, char** argv) { static const struct option long_options[] = { {"help", no_argument, 0, 'h'}, {"project", required_argument, 0, 'N'}, {"server", required_argument, 0, 's'}, {"timeout", required_argument, 0, 't'}, {"username", required_argument, 0, 'u'}, {0, 0, 0, 0} }; struct catalog_query *q; struct jx *j; int c; unsigned int i; time_t timeout = 60; char* catalog_host = NULL; char* username = NULL; char* project = NULL; char* server = NULL; while ((c = getopt_long(argc, argv, "N:t:u:w:s:h", long_options, NULL)) > -1) { switch (c) { case 'N': project = xxstrdup(optarg); break; case 't': timeout = string_time_parse(optarg); break; case 'u': username = xxstrdup(optarg); break; case 's': server = xxstrdup(optarg); break; case 'h': default: show_help(argv[0]); return 1; } } //setup address if(server==NULL){ catalog_host = CATALOG_HOST; } //make query struct jx *jexpr = jx_operator( JX_OP_EQ, jx_symbol("type"), jx_string("makeflow") ); if (project) { jexpr = jx_operator( JX_OP_AND, jexpr, jx_operator( JX_OP_EQ, jx_symbol("project"), jx_string(project) ) ); } if (username) { jexpr = jx_operator( JX_OP_AND, jexpr, jx_operator( JX_OP_EQ, jx_symbol("username"), jx_string(username) ) ); } time_t stoptime = time(0) + timeout; unsigned int count = 0; //create catalog_query from jx q = catalog_query_create(catalog_host, jexpr, stoptime); if (!q) { fprintf(stderr, "couldn't query catalog: %s\n", strerror(errno)); return 1; } while ((j = catalog_query_read(q, stoptime))) { table[count++] = j; } catalog_query_delete(q);//all done with connection //sort qsort(table, count, sizeof(*table), (int (*)(const void *, const void *)) compare_entries); //print them out printf("%-10s %-18s %6s %6s %6s %6s %6s %6s %6s\n", "OWNER", "PROJECT", "JOBS", "WAIT", "RUN", "COMP", "ABRT", "FAIL", "TYPE"); for(i=0; i<count; i++){ printf("%-10s %-18s %6" PRId64 " %6" PRId64 " %6" PRId64 " %6" PRId64 " %6" PRId64 " %6" PRId64 " %6s\n", jx_lookup_string(table[i], "owner"), jx_lookup_string(table[i], "project"), jx_lookup_integer(table[i], "total"), jx_lookup_integer(table[i], "waiting"), jx_lookup_integer(table[i], "running"), jx_lookup_integer(table[i], "completed"), jx_lookup_integer(table[i], "aborted"), jx_lookup_integer(table[i], "failed"), jx_lookup_string(table[i], "batch_type") ); } printf("\n");//be polite //cleanup for(i=0;i<count;i++) { jx_delete(table[i]); } //done return (EXIT_SUCCESS); }
void jx_insert_string( struct jx *j, const char *key, const char *value ) { jx_insert(j,jx_string(key),jx_string(value)); }
void jx_insert_double( struct jx *j, const char *key, double value ) { jx_insert(j,jx_string(key),jx_double(value)); }
void jx_insert_integer( struct jx *j, const char *key, jx_int_t value ) { jx_insert(j,jx_string(key),jx_integer(value)); }
struct jx *rmsummary_to_json(struct rmsummary *s, int only_resources) { struct jx *output = jx_object(NULL); struct jx *array; if(s->disk > -1) { array = jx_arrayv(jx_integer(s->disk), jx_string("MB"), NULL); jx_insert(output, jx_string("disk"), array); } if(s->total_files > -1) jx_insert_integer(output, "total_files", s->total_files); if(s->bytes_written > -1) { array = jx_arrayv(jx_integer(s->bytes_written), jx_string("MB"), NULL); jx_insert(output, jx_string("bytes_written"), array); } if(s->bytes_read > -1) { array = jx_arrayv(jx_integer(s->bytes_read), jx_string("MB"), NULL); jx_insert(output, jx_string("bytes_read"), array); } if(s->swap_memory > -1) { array = jx_arrayv(jx_integer(s->swap_memory), jx_string("MB"), NULL); jx_insert(output, jx_string("swap_memory"), array); } if(s->memory > -1) { array = jx_arrayv(jx_integer(s->memory), jx_string("MB"), NULL); jx_insert(output, jx_string("memory"), array); } if(s->virtual_memory > -1) { array = jx_arrayv(jx_integer(s->virtual_memory), jx_string("MB"), NULL); jx_insert(output, jx_string("virtual_memory"), array); } if(s->total_processes > -1) jx_insert_integer(output, "total_processes", s->total_processes); if(s->max_concurrent_processes > -1) jx_insert_integer(output, "max_concurrent_processes", s->max_concurrent_processes); if(s->cores > -1) jx_insert_integer(output, "cores", s->cores); if(s->cpu_time > -1) { array = jx_arrayv(jx_integer(s->cpu_time), jx_string("us"), NULL); jx_insert(output, jx_string("cpu_time"), array); } if(s->wall_time > -1) { array = jx_arrayv(jx_integer(s->wall_time), jx_string("us"), NULL); jx_insert(output, jx_string("wall_time"), array); } if(s->end > -1) { array = jx_arrayv(jx_integer(s->end), jx_string("us"), NULL); jx_insert(output, jx_string("end"), array); } if(s->start > -1) { array = jx_arrayv(jx_integer(s->start), jx_string("us"), NULL); jx_insert(output, jx_string("start"), array); } if(!only_resources) { if(s->exit_type) { if( strcmp(s->exit_type, "signal") == 0 ) { jx_insert_integer(output, "signal", s->signal); } else if( strcmp(s->exit_type, "limits") == 0 ) { if(s->limits_exceeded) { struct jx *lim = rmsummary_to_json(s->limits_exceeded, 1); jx_insert(output, jx_string("limits_exceeded"), lim); } jx_insert_string(output, "exit_type", "limits"); } } if(s->last_error) jx_insert_integer(output, "last_error", s->last_error); jx_insert_integer(output, "exit_status", s->exit_status); if(s->command) jx_insert_string(output, "command", s->command); if(s->category) jx_insert_string(output, "category", s->category); } return output; }
/* Write the task and run info to the task directory * These files are hardcoded to task_info and run_info */ static int makeflow_archive_write_task_info(struct archive_instance *a, struct dag_node *n, struct batch_task *t, char *archive_path) { struct batch_file *f; /* task_info : * COMMAND: Tasks command that was run * SRC_COMMAND: Origin node's command for reference * SRC_LINE: Line of origin node in SRC_MAKEFLOW * SRC_MAKEFLOW: ID of file for the original Makeflow stored in archive * INPUT_FILES: Alphabetic list of input files checksum IDs * OUTPUT_FILES: Alphabetic list of output file inner_names */ struct jx *task_jx = jx_object(NULL); jx_insert(task_jx, jx_string("COMMAND"), jx_string(t->command)); jx_insert(task_jx, jx_string("SRC_COMMAND"), jx_string(n->command)); jx_insert(task_jx, jx_string("SRC_LINE"), jx_integer(n->linenum)); jx_insert(task_jx, jx_string("SRC_MAKEFLOW"), jx_string(a->source_makeflow)); struct jx * input_files = jx_object(NULL); struct list_cursor *cur = list_cursor_create(t->input_files); for(list_seek(cur, 0); list_get(cur, (void**)&f); list_next(cur)) { /* Generate the file archive id (content based) if does not exist. */ char * id; if(path_is_dir(f->inner_name) == 1){ f->hash = batch_file_generate_id_dir(f->inner_name); id = xxstrdup(f->hash); } else{ id = batch_file_generate_id(f); } jx_insert(input_files, jx_string(f->inner_name), jx_string(id)); free(id); } list_cursor_destroy(cur); jx_insert(task_jx, jx_string("INPUT_FILES"), input_files); struct jx * output_files = jx_object(NULL); cur = list_cursor_create(t->output_files); for(list_seek(cur, 0); list_get(cur, (void**)&f); list_next(cur)) { /* Generate the file archive id (content based) if does not exist. */ char * id; if(path_is_dir(f->inner_name) == 1){ f->hash = batch_file_generate_id_dir(f->inner_name); id = xxstrdup(f->hash); } else{ id = batch_file_generate_id(f); } jx_insert(output_files, jx_string(f->inner_name), jx_string(id)); free(id); } list_cursor_destroy(cur); jx_insert(task_jx, jx_string("OUTPUT_FILES"), output_files); char *task_info = string_format("%s/task_info", archive_path); FILE *fp = fopen(task_info, "w"); if (fp == NULL) { free(task_info); debug(D_ERROR|D_MAKEFLOW_HOOK, "could not create task_info for node %d archive", n->nodeid); return 0; } else { jx_pretty_print_stream(task_jx, fp); } fclose(fp); free(task_info); jx_delete(task_jx); /* run_info : * SUBMITTED : Time task was submitted * STARTED : Time task was started * FINISHED : Time task was completed * EXIT_NORMALLY : 0 if abnormal exit, 1 is normal * EXIT_CODE : Task's exit code * EXIT_SIGNAL : Int value of signal if occurred */ struct jx * run_jx = jx_object(NULL); jx_insert(run_jx, jx_string("SUBMITTED"), jx_integer(t->info->submitted)); jx_insert(run_jx, jx_string("STARTED"), jx_integer(t->info->started)); jx_insert(run_jx, jx_string("FINISHED"), jx_integer(t->info->finished)); jx_insert(run_jx, jx_string("EXIT_NORMAL"), jx_integer(t->info->exited_normally)); jx_insert(run_jx, jx_string("EXIT_CODE"), jx_integer(t->info->exit_code)); jx_insert(run_jx, jx_string("EXIT_SIGNAL"), jx_integer(t->info->exit_signal)); task_info = string_format("%s/run_info", archive_path); fp = fopen(task_info, "w"); if (fp == NULL) { free(task_info); debug(D_ERROR|D_MAKEFLOW_HOOK, "could not create run_info for node %d archive", n->nodeid); return 0; } else { jx_pretty_print_stream(run_jx, fp); } fclose(fp); free(task_info); jx_delete(run_jx); return 1; }