int makeflow_catalog_summary(struct dag* d, char* name, batch_queue_type_t type, timestamp_t start){
struct dag_node *n;
dag_node_state_t state;
int tasks_completed = 0;
int tasks_aborted = 0;
int tasks_waiting = 0;
int tasks_running = 0;
int tasks_failed = 0;
for (n = d->nodes; n; n = n->next) {
state = n->state;
if (state == DAG_NODE_STATE_FAILED)
tasks_failed++;
else if (state == DAG_NODE_STATE_ABORTED)
tasks_aborted++;
else if (state == DAG_NODE_STATE_COMPLETE)
tasks_completed++;
else if(state == DAG_NODE_STATE_RUNNING)
tasks_running++;
else if(state == DAG_NODE_STATE_WAITING)
tasks_waiting++;
}
//transmit report here
char* host = CATALOG_HOST;
char username[USERNAME_MAX];
username_get(username);
const char* batch_type = batch_queue_type_to_string(type);
struct jx *j = jx_object(0);
jx_insert_string(j,"type","makeflow");
jx_insert_integer(j,"total",itable_size(d->node_table));
jx_insert_integer(j,"running",tasks_running);
jx_insert_integer(j,"waiting",tasks_waiting);
jx_insert_integer(j,"aborted",tasks_aborted);
jx_insert_integer(j,"completed",tasks_completed);
jx_insert_integer(j,"failed",tasks_failed);
jx_insert_string(j,"project",name);
jx_insert_string(j,"owner",username);
char* timestring = string_format("%" PRIu64 "", start);
jx_insert_string(j,"time_started",timestring);
jx_insert_string(j,"batch_type",batch_type);
//creates memory
char* text = jx_print_string(j);
int resp = catalog_query_send_update(host, text);
free(text);
free(timestring);
jx_delete(j);
return resp;//all good
}
static void update_all_catalogs()
{
struct jx *j = jx_object(0);
jx_insert_string(j,"type","catalog");
jx_insert(j, jx_string("version"), jx_format("%d.%d.%d", CCTOOLS_VERSION_MAJOR, CCTOOLS_VERSION_MINOR, CCTOOLS_VERSION_MICRO));
jx_insert_string(j,"owner",owner);
jx_insert_integer(j,"starttime",starttime);
jx_insert_integer(j,"port",port);
jx_insert(j,
jx_string("url"),
jx_format("http://%s:%d",preferred_hostname,port)
);
char *text = jx_print_string(j);
jx_delete(j);
list_iterate(outgoing_host_list, (list_op_t) catalog_query_send_update, text);
free(text);
}
void print_stats(struct list *masters, struct list *foremen, int submitted, int needed, int requested, int connected)
{
struct timeval tv;
struct tm *tm;
gettimeofday(&tv, 0);
tm = localtime(&tv.tv_sec);
int to_connect = submitted - connected;
needed = needed > 0 ? needed : 0;
requested = requested > 0 ? requested : 0;
to_connect = to_connect > 0 ? to_connect : 0;
fprintf(stdout, "%04d/%02d/%02d %02d:%02d:%02d: "
"|submitted: %d |needed: %d |waiting connection: %d |requested: %d \n",
tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec,
submitted, needed, to_connect, requested);
int master_count = 0;
master_count += masters ? list_size(masters) : 0;
master_count += foremen ? list_size(foremen) : 0;
if(master_count < 1)
{
fprintf(stdout, "No change this cycle.\n\n");
return;
}
int columns = 80;
char *column_str = getenv("COLUMNS");
if(column_str) {
columns = atoi(column_str);
columns = columns < 1 ? 80 : columns;
}
jx_table_print_header(queue_headers,stdout,columns);
struct jx *j;
if(masters && list_size(masters) > 0)
{
list_first_item(masters);
while((j = list_next_item(masters)))
{
if(!using_catalog) {
jx_insert_string(j, "name", master_host);
}
jx_table_print(queue_headers, j, stdout, columns);
}
}
if(foremen && list_size(foremen) > 0)
{
fprintf(stdout, "foremen:\n");
list_first_item(foremen);
while((j = list_next_item(foremen)))
{
jx_table_print(queue_headers, j, stdout, columns);
}
}
fprintf(stdout, "\n");
fflush(stdout);
}
int main(int argc, char *argv[]) {
char *host = CATALOG_HOST;
int port = CATALOG_PORT;
static const struct option long_options[] = {
{"catalog", required_argument, 0, 'c'},
{0,0,0,0}
};
signed int c;
while ((c = getopt_long(argc, argv, "c:", long_options, NULL)) > -1) {
switch (c) {
case 'c':
host = optarg;
break;
default:
show_help(argv[0]);
return EXIT_FAILURE;
}
}
struct datagram *d;
d = datagram_create(DATAGRAM_PORT_ANY);
if (!d) {
fatal("could not create datagram port!");
}
struct utsname name;
int cpus;
int uptime;
double load[3];
UINT64_T memory_total, memory_avail;
char owner[USERNAME_MAX];
uname(&name);
string_tolower(name.sysname);
string_tolower(name.machine);
string_tolower(name.release);
load_average_get(load);
cpus = load_average_get_cpus();
host_memory_info_get(&memory_avail, &memory_total);
uptime = uptime_get();
username_get(owner);
struct jx *j = jx_object(0);
jx_insert_string(j,"type","node");
jx_insert(j,jx_string("version"),jx_format("%d.%d.%d",CCTOOLS_VERSION_MAJOR, CCTOOLS_VERSION_MINOR, CCTOOLS_VERSION_MICRO));
jx_insert_string(j,"cpu",name.machine);
jx_insert_string(j,"opsys",name.sysname);
jx_insert_string(j,"opsysversion",name.release);
jx_insert_double(j,"load1",load[0]);
jx_insert_double(j,"load5",load[1]);
jx_insert_double(j,"load15",load[2]);
jx_insert_integer(j,"memory_total",memory_total);
jx_insert_integer(j,"memory_avail",memory_avail);
jx_insert_integer(j,"cpus",cpus);
jx_insert_integer(j,"uptime,",uptime);
jx_insert_string(j,"owner",owner);
int i;
for (i = optind; i < argc; i++) {
char *name;
char *value;
name = argv[i];
value = strchr(name, '=');
if (!value) {
fatal("invalid name/value pair = %s", name);
} else {
*value++ = 0;
}
jx_insert_string(j,name,value);
}
char *text = jx_print_string(j);
char address[DATAGRAM_ADDRESS_MAX];
if (domain_name_cache_lookup(host, address)) {
datagram_send(d, text, strlen(text), address, port);
} else {
fatal("unable to lookup address of host: %s", host);
}
jx_delete(j);
datagram_delete(d);
return EXIT_SUCCESS;
}
static void handle_update( const char *addr, int port, const char *raw_data, int raw_data_length, const char *protocol )
{
char key[LINE_MAX];
unsigned long data_length;
struct jx *j;
// If the packet starts with Control-Z (0x1A), it is compressed,
// so uncompress it to data[]. Otherwise just copy to data[];.
if(raw_data[0]==0x1A) {
data_length = sizeof(data);
int success = uncompress((Bytef*)data,&data_length,(const Bytef*)&raw_data[1],raw_data_length-1);
if(success!=Z_OK) {
debug(D_DEBUG,"warning: %s:%d sent invalid compressed data (ignoring it)\n",addr,port);
return;
}
} else {
memcpy(data,raw_data,raw_data_length);
data_length = raw_data_length;
}
// Make sure the string data is null terminated.
data[data_length] = 0;
// Once uncompressed, if it starts with a bracket,
// then it is JX/JSON, otherwise it is the legacy nvpair format.
if(data[0]=='{') {
j = jx_parse_string(data);
if(!j) {
debug(D_DEBUG,"warning: %s:%d sent invalid JSON data (ignoring it)\n%s\n",addr,port,data);
return;
}
if(!jx_is_constant(j)) {
debug(D_DEBUG,"warning: %s:%d sent non-constant JX data (ignoring it)\n%s\n",addr,port,data);
jx_delete(j);
return;
}
} else {
struct nvpair *nv = nvpair_create();
if(!nv) return;
nvpair_parse(nv, data);
j = nvpair_to_jx(nv);
nvpair_delete(nv);
}
jx_insert_string(j, "address", addr);
jx_insert_integer(j, "lastheardfrom", time(0));
/* If the server reports unbelievable numbers, simply reset them */
if(max_server_size > 0) {
INT64_T total = jx_lookup_integer(j, "total");
INT64_T avail = jx_lookup_integer(j, "avail");
if(total > max_server_size || avail > max_server_size) {
jx_insert_integer(j, "total", max_server_size);
jx_insert_integer(j, "avail", max_server_size);
}
}
/* Do not believe the server's reported name, just resolve it backwards. */
char name[DOMAIN_NAME_MAX];
if(domain_name_cache_lookup_reverse(addr, name)) {
/*
Special case: Prior bug resulted in multiple name
entries in logged data. When removing the name property,
keep looking until all items are removed.
*/
struct jx *jname = jx_string("name");
struct jx *n;
while((n=jx_remove(j,jname))) {
jx_delete(n);
}
jx_delete(jname);
jx_insert_string(j,"name",name);
} else if (jx_lookup_string(j, "name") == NULL) {
/* If rDNS is unsuccessful, then we use the name reported if given.
* This allows for hostnames that are only valid in the subnet of
* the reporting server. Here we set the "name" field to the IP
* Address, addr, because it was not set by the reporting server.
*/
jx_insert_string(j, "name", addr);
}
make_hash_key(j, key);
if(logfile) {
if(!jx_database_lookup(table,key)) {
jx_print_stream(j,logfile);
fprintf(logfile,"\n");
fflush(logfile);
}
}
jx_database_insert(table, key, j);
debug(D_DEBUG, "received %s update from %s",protocol,key);
}
struct jx *rmsummary_to_json(struct rmsummary *s, int only_resources) {
struct jx *output = jx_object(NULL);
struct jx *array;
if(s->disk > -1) {
array = jx_arrayv(jx_integer(s->disk), jx_string("MB"), NULL);
jx_insert(output, jx_string("disk"), array);
}
if(s->total_files > -1)
jx_insert_integer(output, "total_files", s->total_files);
if(s->bytes_written > -1) {
array = jx_arrayv(jx_integer(s->bytes_written), jx_string("MB"), NULL);
jx_insert(output, jx_string("bytes_written"), array);
}
if(s->bytes_read > -1) {
array = jx_arrayv(jx_integer(s->bytes_read), jx_string("MB"), NULL);
jx_insert(output, jx_string("bytes_read"), array);
}
if(s->swap_memory > -1) {
array = jx_arrayv(jx_integer(s->swap_memory), jx_string("MB"), NULL);
jx_insert(output, jx_string("swap_memory"), array);
}
if(s->memory > -1) {
array = jx_arrayv(jx_integer(s->memory), jx_string("MB"), NULL);
jx_insert(output, jx_string("memory"), array);
}
if(s->virtual_memory > -1) {
array = jx_arrayv(jx_integer(s->virtual_memory), jx_string("MB"), NULL);
jx_insert(output, jx_string("virtual_memory"), array);
}
if(s->total_processes > -1)
jx_insert_integer(output, "total_processes", s->total_processes);
if(s->max_concurrent_processes > -1)
jx_insert_integer(output, "max_concurrent_processes", s->max_concurrent_processes);
if(s->cores > -1)
jx_insert_integer(output, "cores", s->cores);
if(s->cpu_time > -1) {
array = jx_arrayv(jx_integer(s->cpu_time), jx_string("us"), NULL);
jx_insert(output, jx_string("cpu_time"), array);
}
if(s->wall_time > -1) {
array = jx_arrayv(jx_integer(s->wall_time), jx_string("us"), NULL);
jx_insert(output, jx_string("wall_time"), array);
}
if(s->end > -1) {
array = jx_arrayv(jx_integer(s->end), jx_string("us"), NULL);
jx_insert(output, jx_string("end"), array);
}
if(s->start > -1) {
array = jx_arrayv(jx_integer(s->start), jx_string("us"), NULL);
jx_insert(output, jx_string("start"), array);
}
if(!only_resources) {
if(s->exit_type)
{
if( strcmp(s->exit_type, "signal") == 0 ) {
jx_insert_integer(output, "signal", s->signal);
} else if( strcmp(s->exit_type, "limits") == 0 ) {
if(s->limits_exceeded) {
struct jx *lim = rmsummary_to_json(s->limits_exceeded, 1);
jx_insert(output, jx_string("limits_exceeded"), lim);
}
jx_insert_string(output, "exit_type", "limits");
}
}
if(s->last_error)
jx_insert_integer(output, "last_error", s->last_error);
jx_insert_integer(output, "exit_status", s->exit_status);
if(s->command)
jx_insert_string(output, "command", s->command);
if(s->category)
jx_insert_string(output, "category", s->category);
}
return output;
}
