static void log_updates( struct jx_database *db, const char *key, struct jx *a, struct jx *b )
{
// For each item in the old object:
// If the new one is different, log an update event.
// If the new one is missing, log a remove event.
struct jx_pair *p;
for(p=a->u.pairs;p;p=p->next) {
const char *name = p->key->u.string_value;
struct jx *avalue = p->value;
// Do not log these special cases, because they do not carry new information:
if(!strcmp(name,"lastheardfrom")) continue;
if(!strcmp(name,"uptime")) continue;
struct jx *bvalue = jx_lookup(b,name);
if(bvalue) {
if(jx_equals(avalue,bvalue)) {
// items match, do nothing.
} else {
// item changed, print it.
char *str = jx_print_string(bvalue);
log_message(db,"U %s %s %s\n",key,name,str);
free(str);
}
} else {
// item was removed.
log_message(db,"R %s %s\n",key,name);
}
}
// For each item in the new object:
// If it doesn't exist in the old one, log an update event.
for(p=b->u.pairs;p;p=p->next) {
const char *name = p->key->u.string_value;
struct jx *bvalue = p->value;
struct jx *avalue = jx_lookup(a,name);
if(!avalue) {
// item changed, print it.
char *str = jx_print_string(bvalue);
log_message(db,"U %s %s %s\n",key,name,str);
free(str);
}
}
}
int makeflow_catalog_summary(struct dag* d, char* name, batch_queue_type_t type, timestamp_t start){
struct dag_node *n;
dag_node_state_t state;
int tasks_completed = 0;
int tasks_aborted = 0;
int tasks_waiting = 0;
int tasks_running = 0;
int tasks_failed = 0;
for (n = d->nodes; n; n = n->next) {
state = n->state;
if (state == DAG_NODE_STATE_FAILED)
tasks_failed++;
else if (state == DAG_NODE_STATE_ABORTED)
tasks_aborted++;
else if (state == DAG_NODE_STATE_COMPLETE)
tasks_completed++;
else if(state == DAG_NODE_STATE_RUNNING)
tasks_running++;
else if(state == DAG_NODE_STATE_WAITING)
tasks_waiting++;
}
//transmit report here
char* host = CATALOG_HOST;
char username[USERNAME_MAX];
username_get(username);
const char* batch_type = batch_queue_type_to_string(type);
struct jx *j = jx_object(0);
jx_insert_string(j,"type","makeflow");
jx_insert_integer(j,"total",itable_size(d->node_table));
jx_insert_integer(j,"running",tasks_running);
jx_insert_integer(j,"waiting",tasks_waiting);
jx_insert_integer(j,"aborted",tasks_aborted);
jx_insert_integer(j,"completed",tasks_completed);
jx_insert_integer(j,"failed",tasks_failed);
jx_insert_string(j,"project",name);
jx_insert_string(j,"owner",username);
char* timestring = string_format("%" PRIu64 "", start);
jx_insert_string(j,"time_started",timestring);
jx_insert_string(j,"batch_type",batch_type);
//creates memory
char* text = jx_print_string(j);
int resp = catalog_query_send_update(host, text);
free(text);
free(timestring);
jx_delete(j);
return resp;//all good
}
struct nvpair * jx_to_nvpair( struct jx *object )
{
struct nvpair *nv = nvpair_create();
struct jx_pair *p;
for(p=object->pairs;p;p=p->next) {
if(p->value->type==JX_STRING) {
nvpair_insert_string(nv,p->key->string_value,p->value->string_value);
} else {
char *s = jx_print_string(p->value);
nvpair_insert_string(nv,p->key->string_value,s);
free(s);
}
}
return nv;
}
int deltadb_update_event( struct deltadb *db, const char *key, const char *name, struct jx *jvalue )
{
struct jx * jobject = hash_table_lookup(db->table,key);
if(!jobject) return 1;
struct jx *jname = jx_string(name);
jx_delete(jx_remove(jobject,jname));
jx_insert(jobject,jname,jvalue);
if(display_mode==MODE_STREAM) {
display_deferred_time(db);
char *str = jx_print_string(jvalue);
printf("U %s %s %s\n",key,name,str);
free(str);
}
return 1;
}
static void update_all_catalogs()
{
struct jx *j = jx_object(0);
jx_insert_string(j,"type","catalog");
jx_insert(j, jx_string("version"), jx_format("%d.%d.%d", CCTOOLS_VERSION_MAJOR, CCTOOLS_VERSION_MINOR, CCTOOLS_VERSION_MICRO));
jx_insert_string(j,"owner",owner);
jx_insert_integer(j,"starttime",starttime);
jx_insert_integer(j,"port",port);
jx_insert(j,
jx_string("url"),
jx_format("http://%s:%d",preferred_hostname,port)
);
char *text = jx_print_string(j);
jx_delete(j);
list_iterate(outgoing_host_list, (list_op_t) catalog_query_send_update, text);
free(text);
}
static void log_create( struct jx_database *db, const char *key, struct jx *j )
{
char *str = jx_print_string(j);
log_message(db,"C %s %s\n",key,str);
free(str);
}
int main( int argc, char *argv[] )
{
const char *dbdir=0;
const char *dbfile=0;
struct jx *where_expr = 0;
struct jx *filter_expr = 0;
struct list *output_exprs = list_create();
struct list *reduce_exprs = list_create();
time_t start_time = 0;
time_t stop_time = 0;
int display_every = 0;
int epoch_mode = 0;
char reduce_name[1024];
char reduce_attr[1024];
time_t current = time(0);
int c;
while((c=getopt_long(argc,argv,"D:L:o:w:f:F:T:e:tvh",long_options,0))!=-1) {
switch(c) {
case 'D':
dbdir = optarg;
break;
case 'L':
dbfile = optarg;
break;
case 'o':
if(2==sscanf(optarg,"%[^(](%[^)])",reduce_name,reduce_attr)) {
struct jx *reduce_expr = jx_parse_string(reduce_attr);
if(!reduce_expr) {
fprintf(stderr,"deltadb_query: invalid expression: %s\n",reduce_attr);
return 1;
}
struct deltadb_reduction *r = deltadb_reduction_create(reduce_name,reduce_expr);
if(!r) {
fprintf(stderr,"deltadb_query: invalid reduction: %s\n",reduce_name);
return 1;
}
list_push_tail(reduce_exprs,r);
} else {
struct jx *j = jx_parse_string(optarg);
if(!j) {
fprintf(stderr,"invalid expression: %s\n",optarg);
return 1;
}
list_push_tail(output_exprs,j);
}
break;
case 'w':
if(where_expr) {
fprintf(stderr,"Only one --where expression is allowed. Try joining the expressions with the && (and) operator.");
return 1;
}
where_expr = jx_parse_string(optarg);
if(!where_expr) {
fprintf(stderr,"invalid expression: %s\n",optarg);
return 1;
}
break;
case 'f':
if(filter_expr) {
fprintf(stderr,"Only one --filter expression is allowed. Try joining the expressions with the && (and) operator.");
return 1;
}
filter_expr = jx_parse_string(optarg);
if(!filter_expr) {
fprintf(stderr,"invalid expression: %s\n",optarg);
return 1;
}
break;
case 'F':
start_time = parse_time(optarg,current);
break;
case 'T':
stop_time = parse_time(optarg,current);
break;
case 'e':
display_every = string_time_parse(optarg);
break;
case 't':
epoch_mode = 1;
break;
case 'v':
cctools_version_print(stdout,"deltadb_query");
break;
case 'h':
show_help();
break;
}
}
if(!dbdir && !dbfile) {
fprintf(stderr,"deltadb_query: either --db or --file argument is required\n");
return 1;
}
if(start_time==0) {
fprintf(stderr,"deltadb_query: invalid --from time (must be \"YY-MM-DD\" or \"YY-MM-DD HH:MM:SS\")\n");
return 1;
}
if(stop_time==0) {
stop_time = time(0);
}
struct deltadb *db = deltadb_create(dbdir);
db->where_expr = where_expr;
db->filter_expr = filter_expr;
db->epoch_mode = epoch_mode;
db->output_exprs = output_exprs;
db->reduce_exprs = reduce_exprs;
db->display_every = display_every;
if(list_size(db->reduce_exprs) && list_size(db->output_exprs) ) {
struct deltadb_reduction *r = db->reduce_exprs->head->data;
const char *name = jx_print_string(db->output_exprs->head->data);
fprintf(stderr,"deltadb_query: cannot mix reductions like 'MAX(%s)' with plain outputs like '%s'\n",jx_print_string(r->expr),name);
return 1;
}
if(list_size(db->reduce_exprs)) {
display_mode = MODE_REDUCE;
} else if(list_size(db->output_exprs)) {
display_mode = MODE_OBJECT;
} else {
display_mode = MODE_STREAM;
}
if(dbfile) {
FILE *file = fopen(dbfile,"r");
if(!file) {
fprintf(stderr,"deltadb_query: couldn't open %s: %s\n",dbfile,strerror(errno));
return 1;
}
deltadb_process_stream(db,file,start_time,stop_time);
fclose(file);
} else {
log_play_time(db,start_time,stop_time);
}
return 0;
}
int main(int argc, char *argv[]) {
char *host = CATALOG_HOST;
int port = CATALOG_PORT;
static const struct option long_options[] = {
{"catalog", required_argument, 0, 'c'},
{0,0,0,0}
};
signed int c;
while ((c = getopt_long(argc, argv, "c:", long_options, NULL)) > -1) {
switch (c) {
case 'c':
host = optarg;
break;
default:
show_help(argv[0]);
return EXIT_FAILURE;
}
}
struct datagram *d;
d = datagram_create(DATAGRAM_PORT_ANY);
if (!d) {
fatal("could not create datagram port!");
}
struct utsname name;
int cpus;
int uptime;
double load[3];
UINT64_T memory_total, memory_avail;
char owner[USERNAME_MAX];
uname(&name);
string_tolower(name.sysname);
string_tolower(name.machine);
string_tolower(name.release);
load_average_get(load);
cpus = load_average_get_cpus();
host_memory_info_get(&memory_avail, &memory_total);
uptime = uptime_get();
username_get(owner);
struct jx *j = jx_object(0);
jx_insert_string(j,"type","node");
jx_insert(j,jx_string("version"),jx_format("%d.%d.%d",CCTOOLS_VERSION_MAJOR, CCTOOLS_VERSION_MINOR, CCTOOLS_VERSION_MICRO));
jx_insert_string(j,"cpu",name.machine);
jx_insert_string(j,"opsys",name.sysname);
jx_insert_string(j,"opsysversion",name.release);
jx_insert_double(j,"load1",load[0]);
jx_insert_double(j,"load5",load[1]);
jx_insert_double(j,"load15",load[2]);
jx_insert_integer(j,"memory_total",memory_total);
jx_insert_integer(j,"memory_avail",memory_avail);
jx_insert_integer(j,"cpus",cpus);
jx_insert_integer(j,"uptime,",uptime);
jx_insert_string(j,"owner",owner);
int i;
for (i = optind; i < argc; i++) {
char *name;
char *value;
name = argv[i];
value = strchr(name, '=');
if (!value) {
fatal("invalid name/value pair = %s", name);
} else {
*value++ = 0;
}
jx_insert_string(j,name,value);
}
char *text = jx_print_string(j);
char address[DATAGRAM_ADDRESS_MAX];
if (domain_name_cache_lookup(host, address)) {
datagram_send(d, text, strlen(text), address, port);
} else {
fatal("unable to lookup address of host: %s", host);
}
jx_delete(j);
datagram_delete(d);
return EXIT_SUCCESS;
}
int category_update_first_allocation(struct category *c, const struct rmsummary *max_worker) {
/* buffer used only for debug output. */
static buffer_t *b = NULL;
if(!b) {
b = malloc(sizeof(buffer_t));
buffer_init(b);
}
if(c->allocation_mode == CATEGORY_ALLOCATION_MODE_FIXED)
return 0;
if(c->total_tasks < 1)
return 0;
struct rmsummary *top = rmsummary_create(-1);
rmsummary_merge_override(top, max_worker);
rmsummary_merge_override(top, c->max_resources_seen);
rmsummary_merge_override(top, c->max_allocation);
if(!c->first_allocation) {
c->first_allocation = rmsummary_create(-1);
}
update_first_allocation_field(c, top, 1, cpu_time);
update_first_allocation_field(c, top, 1, wall_time);
update_first_allocation_field(c, top, c->time_peak_independece, cores);
update_first_allocation_field(c, top, c->time_peak_independece, virtual_memory);
update_first_allocation_field(c, top, c->time_peak_independece, memory);
update_first_allocation_field(c, top, c->time_peak_independece, swap_memory);
update_first_allocation_field(c, top, c->time_peak_independece, bytes_read);
update_first_allocation_field(c, top, c->time_peak_independece, bytes_written);
update_first_allocation_field(c, top, c->time_peak_independece, bytes_received);
update_first_allocation_field(c, top, c->time_peak_independece, bytes_sent);
update_first_allocation_field(c, top, c->time_peak_independece, bandwidth);
update_first_allocation_field(c, top, c->time_peak_independece, total_files);
update_first_allocation_field(c, top, c->time_peak_independece, disk);
update_first_allocation_field(c, top, c->time_peak_independece, max_concurrent_processes);
update_first_allocation_field(c, top, c->time_peak_independece, total_processes);
/* From here on we only print debugging info. */
struct jx *jsum = rmsummary_to_json(c->first_allocation, 1);
if(jsum) {
char *str = jx_print_string(jsum);
debug(D_DEBUG, "Updating first allocation '%s':", c->name);
debug(D_DEBUG, "%s", str);
jx_delete(jsum);
free(str);
}
jsum = rmsummary_to_json(top, 1);
if(jsum) {
char *str = jx_print_string(jsum);
debug(D_DEBUG, "From max resources '%s':", c->name);
debug(D_DEBUG, "%s", str);
jx_delete(jsum);
free(str);
}
rmsummary_delete(top);
return 1;
}
