int compare_entries(struct nvpair **a, struct nvpair **b)
{
int result;
const char *x, *y;
x = nvpair_lookup_string(*a, "type");
if(!x)
x = "unknown";
y = nvpair_lookup_string(*b, "type");
if(!y)
y = "unknown";
result = strcasecmp(x, y);
if(result != 0)
return result;
x = nvpair_lookup_string(*a, "name");
if(!x)
x = "unknown";
y = nvpair_lookup_string(*b, "name");
if(!y)
y = "unknown";
return strcasecmp(x, y);
}
int my_foreman(int *space, const char *host, int port, time_t stoptime)
{
int i = 0; //global_catalog iterator
char full_address[1024];
if(!domain_name_cache_lookup(host, full_address) || !full_address)
{
debug(D_WQ,"Could not resolve %s into an ip address\n",host);
return 0;
}
sprintf(full_address, "%s:%d", full_address, port);
while(global_catalog[i] != NULL){
const char *temp_my_master = nvpair_lookup_string(global_catalog[i], "my_master");
if(temp_my_master && !strcmp(temp_my_master, full_address)){
char modified_proj[50];
memset(modified_proj,0,sizeof(modified_proj));
space_relations(*space, modified_proj); //append '->' for proper recursive depth
sprintf(modified_proj,"%s%s",modified_proj,nvpair_lookup_string(global_catalog[i], "project")); //prepare modified project name with proper depth
nvpair_remove(global_catalog[i], nvpair_lookup_string(global_catalog[i], "project")); //remove old project look
nvpair_insert_string(global_catalog[i], "project", modified_proj); //replace with the modified relation version
if(resource_mode){
nvpair_print_table(global_catalog[i], stdout, master_resource_headers);
}else if(format_mode == FORMAT_TABLE){
nvpair_print_table(global_catalog[i], stdout, queue_headers);
}
int new_space = *space + 1; //so that spaces are preserved in recursive calls
my_foreman(&new_space,nvpair_lookup_string(global_catalog[i], "name"),atoi(nvpair_lookup_string(global_catalog[i], "port")),stoptime);
}
i++;
}
return 1;
}
int do_catalog_query( time_t stoptime )
{
int i = 0; //global_catalog iterator
if(resource_mode == 0 && format_mode == FORMAT_TABLE) nvpair_print_table_header(stdout, queue_headers);
else if(resource_mode) nvpair_print_table_header(stdout, master_resource_headers);
while(global_catalog[i] != NULL){
if(!(resource_mode || format_mode == FORMAT_TABLE)){ //long options
nvpair_print_text(global_catalog[i], stdout);
}else{
const char *temp_my_master = nvpair_lookup_string(global_catalog[i], "my_master");
if( !temp_my_master || !strcmp(temp_my_master, "127.0.0.1:-1") ) { //this master has no master
if(resource_mode) {
debug(D_WQ,"%s resources -- cores:%s memory:%s disk:%s\n",nvpair_lookup_string(global_catalog[i],"project"),nvpair_lookup_string(global_catalog[i],"cores_total"),nvpair_lookup_string(global_catalog[i],"memory_total"),nvpair_lookup_string(global_catalog[i],"disk_total"));
nvpair_print_table(global_catalog[i], stdout, master_resource_headers);
}else if(format_mode == FORMAT_TABLE){
nvpair_print_table(global_catalog[i], stdout, queue_headers);
}
int space = 1;
my_foreman(&space, nvpair_lookup_string(global_catalog[i], "name"), atoi(nvpair_lookup_string(global_catalog[i], "port")), stoptime);
}
}
i++;
}
if(format_mode == FORMAT_TABLE){
nvpair_print_table_footer(stdout, queue_headers);
}else if(resource_mode){
nvpair_print_table_footer(stdout, master_resource_headers);
}
global_catalog_cleanup();
return EXIT_SUCCESS;
}
struct work_queue_pool *parse_work_queue_pool_nvpair(struct nvpair *nv)
{
struct work_queue_pool *p;
p = xxmalloc(sizeof(*p));
strncpy(p->addr, nvpair_lookup_string(nv, "address"), LINK_ADDRESS_MAX);
strncpy(p->name, nvpair_lookup_string(nv, "pool_name"), WORK_QUEUE_POOL_NAME_MAX);
p->decision = xxstrdup(nvpair_lookup_string(nv, "decision"));
strncpy(p->owner, nvpair_lookup_string(nv, "owner"), USERNAME_MAX);
return p;
}
struct work_queue_master *parse_work_queue_master_nvpair(struct nvpair *nv)
{
struct work_queue_master *m;
m = xxmalloc(sizeof(struct work_queue_master));
strncpy(m->addr, nvpair_lookup_string(nv, "address"), LINK_ADDRESS_MAX);
const char *project;
project = nvpair_lookup_string(nv, "project");
if(project) {
strncpy(m->proj, project, WORK_QUEUE_NAME_MAX);
} else {
strncpy(m->proj, "unknown", WORK_QUEUE_NAME_MAX);
}
m->port = nvpair_lookup_integer(nv, "port");
m->start_time = nvpair_lookup_integer(nv, "starttime");
m->priority = nvpair_lookup_integer(nv, "priority");
if(m->priority < 0) m->priority = 0;
m->capacity = nvpair_lookup_integer(nv, "capacity");
m->tasks_waiting = nvpair_lookup_integer(nv, "tasks_waiting");
m->tasks_running = nvpair_lookup_integer(nv, "tasks_running");
m->tasks_complete = nvpair_lookup_integer(nv, "tasks_complete");
m->total_tasks_dispatched = nvpair_lookup_integer(nv, "total_tasks_dispatched");
m->workers_init = nvpair_lookup_integer(nv, "workers_init");
m->workers_ready = nvpair_lookup_integer(nv, "workers_ready");
m->workers_busy = nvpair_lookup_integer(nv, "workers_busy");
m->workers_full = nvpair_lookup_integer(nv, "workers_full");
m->workers = nvpair_lookup_integer(nv, "workers");
const char *workers_by_pool;
workers_by_pool = nvpair_lookup_string(nv, "workers_by_pool");
if(workers_by_pool) {
m->workers_by_pool = xxstrdup(workers_by_pool);
} else {
m->workers_by_pool = xxstrdup("unknown");
}
const char *owner;
owner = nvpair_lookup_string(nv, "owner");
if(owner) {
strncpy(m->owner, nvpair_lookup_string(nv, "owner"), USERNAME_MAX);
} else {
strncpy(m->owner, "unknown", USERNAME_MAX);
}
return m;
}
int compare_nvpair(const void *a, const void *b)
{
struct nvpair **pa = (struct nvpair **) a;
struct nvpair **pb = (struct nvpair **) b;
const char *sa = nvpair_lookup_string(*pa, "name");
const char *sb = nvpair_lookup_string(*pb, "name");
if(!sa)
sa = "unknown";
if(!sb)
sb = "unknown";
return strcasecmp(sa, sb);
}
int get_masters(time_t stoptime)
{
struct catalog_query *cq;
struct nvpair *nv;
int i = 0; //nvpair pointer array iterator
if(!catalog_host) {
catalog_host = strdup(CATALOG_HOST);
catalog_port = CATALOG_PORT;
}
cq = catalog_query_create(catalog_host, catalog_port, stoptime );
if(!cq) {
fprintf(stderr, "failed to query catalog server %s:%d: %s \n",catalog_host,catalog_port,strerror(errno));
exit(EXIT_FAILURE);
return 0;
}
while((nv = catalog_query_read(cq,stoptime))) {
resize_catalog(i);
if(strcmp(nvpair_lookup_string(nv, "type"), CATALOG_TYPE_WORK_QUEUE_MASTER) == 0) {
global_catalog[i] = nv; //make the global catalog point to this memory that nv references
i++; //only increment i when a master nvpair is found
}else{
nvpair_delete(nv); //free the memory so something valid can take its place
}
}
resize_catalog(i);
global_catalog[i] = NULL;
catalog_query_delete(cq);
return 1;
}
static int compat_checkpoint_read( struct deltadb *db, const char *filename )
{
FILE * file = fopen(filename,"r");
if(!file) return 0;
while(1) {
struct nvpair *nv = nvpair_create();
if(nvpair_parse_stream(nv,file)) {
const char *key = nvpair_lookup_string(nv,"key");
if(key) {
nvpair_delete(hash_table_remove(db->table,key));
struct jx *j = nvpair_to_jx(nv);
/* skip objects that don't match the filter */
if(deltadb_boolean_expr(db->filter_expr,j)) {
hash_table_insert(db->table,key,j);
} else {
jx_delete(j);
}
}
nvpair_delete(nv);
} else {
nvpair_delete(nv);
break;
}
}
fclose(file);
return 1;
}
void nvpair_print_html_with_link(struct nvpair *n, FILE * s, struct nvpair_header *h, const char *linkname, const char *linktext)
{
fprintf(s, "<tr bgcolor=%s>\n", color_counter % 2 ? COLOR_ONE : COLOR_TWO);
color_counter++;
while(h->name) {
const char *text = nvpair_lookup_string(n, h->name);
if(!text)
text = "???";
fprintf(s, "<td align=%s>", align_string(h));
if(h->mode == NVPAIR_MODE_URL) {
fprintf(s, "<a href=%s>%s</a>\n", text, text);
} else if(h->mode == NVPAIR_MODE_METRIC) {
char line[1024];
string_metric(atof(text), -1, line);
fprintf(s, "%sB\n", line);
} else {
if(linkname && !strcmp(linkname, h->name)) {
fprintf(s, "<a href=%s>%s</a>\n", linktext, text);
} else {
fprintf(s, "%s\n", text);
}
}
h++;
}
}
static int compat_checkpoint_read( struct jx_database *db, const char *filename )
{
FILE * file = fopen(filename,"r");
if(!file) return 0;
while(1) {
struct nvpair *nv = nvpair_create();
if(nvpair_parse_stream(nv,file)) {
const char *key = nvpair_lookup_string(nv,"key");
if(key) {
nvpair_delete(hash_table_remove(db->table,key));
struct jx *j = nvpair_to_jx(nv);
hash_table_insert(db->table,key,j);
}
nvpair_delete(nv);
} else {
nvpair_delete(nv);
break;
}
}
fclose(file);
return 1;
}
static int checkpoint_read( struct deltadb *db )
{
FILE * file = stdin;
if(!file) return 0;
char firstline[NVPAIR_LINE_MAX];
fgets(firstline, sizeof(firstline), file);
printf("%s",firstline);
while(1) {
int keep = 0;
struct nvpair *nv = nvpair_create();
int num_pairs = nvpair_parse_stream(nv,file);
if(num_pairs>0) {
const char *key = nvpair_lookup_string(nv,"key");
if(key) {
keep = 1;
struct argument *arg = db->args;
while (arg!=NULL){
const char *var = nvpair_lookup_string(nv,arg->param);
if ( var!=NULL && keep_object(arg,var)==0 ){
keep = 0;
break;
}
arg = arg->next;
}
nvpair_delete(hash_table_remove(db->table,key));
if (keep==1){
nvpair_print_text(nv,stdout);
hash_table_insert(db->table,key,nv);
}
} else debug(D_NOTICE,"no key in object create.");
} else if (num_pairs == -1) {
return 1;
} else {
break;
}
if (!keep)
nvpair_delete(nv);
}
return 1;
}
static int count_workers_needed( struct list *masters_list, int only_waiting )
{
int needed_workers=0;
int masters=0;
struct nvpair *nv;
if(!masters_list) {
return needed_workers;
}
list_first_item(masters_list);
while((nv=list_next_item(masters_list))) {
const char *project = nvpair_lookup_string(nv,"project");
const char *host = nvpair_lookup_string(nv,"name");
const int port = nvpair_lookup_integer(nv,"port");
const char *owner = nvpair_lookup_string(nv,"owner");
const int tr = nvpair_lookup_integer(nv,"tasks_running");
const int tw = nvpair_lookup_integer(nv,"tasks_waiting");
const int tl = nvpair_lookup_integer(nv,"tasks_left");
const int capacity = nvpair_lookup_integer(nv,"capacity");
int tasks = tr+tw+tl;
int need;
if(only_waiting) {
need = tw;
} else {
need = tasks;
}
if(consider_capacity && capacity>0) {
need = MIN(capacity,tasks);
}
debug(D_WQ,"%s %s:%d %s %d %d %d",project,host,port,owner,tasks,capacity,need);
needed_workers += need;
masters++;
}
needed_workers = (int) ceil(needed_workers / tasks_per_worker);
return needed_workers;
}
static void make_hash_key(struct nvpair *nv, char *key)
{
const char *name, *addr;
int port;
addr = nvpair_lookup_string(nv, "address");
if(!addr)
addr = "unknown";
port = nvpair_lookup_integer(nv, "port");
name = nvpair_lookup_string(nv, "name");
if(!name)
name = "unknown";
sprintf(key, "%s:%d:%s", addr, port, name);
}
struct list *get_masters_from_catalog(const char *catalog_host, int catalog_port, struct list *regex_list)
{
struct catalog_query *q;
struct nvpair *nv;
struct list *ml;
struct work_queue_master *m;
char *regex;
time_t timeout = 60, stoptime;
stoptime = time(0) + timeout;
q = catalog_query_create(catalog_host, catalog_port, stoptime);
if(!q) {
fprintf(stderr, "Failed to query catalog server at %s:%d\n", catalog_host, catalog_port);
return NULL;
}
ml = list_create();
if(!ml)
return NULL;
while((nv = catalog_query_read(q, stoptime))) {
if(strcmp(nvpair_lookup_string(nv, "type"), CATALOG_TYPE_WORK_QUEUE_MASTER) == 0) {
m = parse_work_queue_master_nvpair(nv);
if(m) {
if(regex_list) {
// Matched preferred masters
int match_found = 0;
list_first_item(regex_list);
while((regex = (char *)list_next_item(regex_list))) {
if(whole_string_match_regex(m->proj, regex)) {
debug(D_WQ, "Master matched: %s -> %s\n", regex, m->proj);
list_push_head(ml, m);
match_found = 1;
break;
}
}
if(match_found == 0) {
free_work_queue_master(m);
}
} else {
list_push_head(ml, m);
}
} else {
fprintf(stderr, "Failed to parse a work queue master record!\n");
}
}
nvpair_delete(nv);
}
// Must delete the query otherwise it would occupy 1 tcp connection forever!
catalog_query_delete(q);
return ml;
}
int nvpair_parse_stream(struct nvpair *n, FILE * stream)
{
int num_pairs = 0;
char line[NVPAIR_LINE_MAX];
char name[NVPAIR_LINE_MAX];
char value[NVPAIR_LINE_MAX];
char key[NVPAIR_LINE_MAX];
key[0] = '\0';
while(fgets(line, sizeof(line), stream)) {
if(line[0] == '\n') {
if (strlen(key)==0){
sprintf(key,"%s:%s:%s",nvpair_lookup_string(n,"address"),nvpair_lookup_string(n,"port"),nvpair_lookup_string(n,"name"));
nvpair_insert_string(n, "key", key);
}
if (num_pairs){
return num_pairs;
} else {
continue;
}
}
if(sscanf(line, "%s %[^\r\n]", name, value) == 2) {
if (strcmp(name,"key")==0)
strcpy(key,value);
//printf("-----%s,%s\n",name,value);
nvpair_insert_string(n, name, value);
num_pairs += 1;
} else {
debug(D_DEBUG,"corrupt log data: %s",line);
//return 0;
}
}
return 0;
}
void emit_all_deltadb_reductions( struct deltadb *db, time_t current, int first_output )
{
int i;
struct nvpair *nv;
char *key;
const char *value;
/* Reset all deltadb_reduction state. */
for(i=0;i<db->ndeltadb_reductions;i++) {
deltadb_reduction_reset(db->deltadb_reductions[i]);
}
/* After each event, iterate over all objects... */
hash_table_firstkey(db->table);
while(hash_table_nextkey(db->table,&key,(void**)&nv)) {
/* Update all deltadb_reductions for that object. */
for(i=0;i<db->ndeltadb_reductions;i++) {
struct deltadb_reduction *r = db->deltadb_reductions[i];
value = nvpair_lookup_string(nv,r->attr);
if(value) deltadb_reduction_update(r,value);
}
}
if(first_output) {
/* The first time we do this, make it a checkpoint record. */
printf("T %ld\n",(long)current);
printf("C 0\n");
for(i=0;i<db->ndeltadb_reductions;i++) {
struct deltadb_reduction *r = db->deltadb_reductions[i];
deltadb_reduction_print(r);
}
printf("\n");
//printf(".Checkpoint End.\n");
first_output = 0;
} else {
/* After that, make it an update record. */
printf("T %ld\n",(long)current);
for(i=0;i<db->ndeltadb_reductions;i++) {
struct deltadb_reduction *r = db->deltadb_reductions[i];
printf("U 0 ");
deltadb_reduction_print(r);
}
}
}
void emit_table_values( struct deltadb *db, time_t current)
{
char *key;
struct nvpair *nv;
int i;
hash_table_firstkey(db->table);
while(hash_table_nextkey(db->table, &key, (void **) &nv)) {
printf("%ld\t",current);
for(i=0;i<db->nfields;i++) {
const char *value = nvpair_lookup_string(nv,db->fields[i]);
if(!value) value = "null";
printf("%s\t",value);
}
printf("\n");
}
}
int get_pool_decisions_from_catalog(const char *catalog_host, int catalog_port, const char *proj, struct list *decisions) {
struct catalog_query *q;
struct nvpair *nv;
time_t timeout = 60, stoptime;
stoptime = time(0) + timeout;
if(!decisions) {
fprintf(stderr, "No list to store pool decisions.\n");
return 0;
}
q = catalog_query_create(catalog_host, catalog_port, stoptime);
if(!q) {
fprintf(stderr, "Failed to query catalog server at %s:%d\n", catalog_host, catalog_port);
return 0;
}
// multiple pools
while((nv = catalog_query_read(q, stoptime))) {
if(strcmp(nvpair_lookup_string(nv, "type"), CATALOG_TYPE_WORK_QUEUE_POOL) == 0) {
struct work_queue_pool *p;
p = parse_work_queue_pool_nvpair(nv);
debug(D_WQ, "pool %s's decision: %s\n", p->name, p->decision);
int x = workers_by_item(p->decision, proj);
if(x >= 0) {
struct pool_info *pi;
pi = (struct pool_info *)xxmalloc(sizeof(*pi));
strncpy(pi->name, p->name, WORK_QUEUE_POOL_NAME_MAX);
pi->count = x;
list_push_tail(decisions, pi);
}
free(p->decision);
free(p);
}
nvpair_delete(nv);
}
// Must delete the query otherwise it would occupy 1 tcp connection forever!
catalog_query_delete(q);
return 1;
}
void nvpair_print_table(struct nvpair *n, FILE * s, struct nvpair_header *h)
{
while(h->name) {
const char *text = nvpair_lookup_string(n, h->name);
char *aligned = xxmalloc(h->width + 1);
char *line;
if(!text) {
line = xxstrdup("???");
} else if(h->mode == NVPAIR_MODE_METRIC) {
line = xxmalloc(10);
string_metric(atof(text), -1, line);
strcat(line, "B");
} else if(h->mode == NVPAIR_MODE_TIMESTAMP || h->mode == NVPAIR_MODE_TIME) {
line = xxmalloc(h->width);
timestamp_t ts;
int ret = 0;
if(sscanf(text, "%" SCNu64, &ts) == 1) {
if(h->mode == NVPAIR_MODE_TIME) {
ts *= 1000000;
}
ret = timestamp_fmt(line, h->width, "%R %b %d, %Y", ts);
}
if(ret == 0) {
strcpy(line, "???");
}
} else {
line = xxmalloc(strlen(text) + 1);
strcpy(line, text);
}
fill_string(line, aligned, h->width, h->align);
printf("%s ", aligned);
free(line);
free(aligned);
h++;
}
printf("\n");
}
int main(int argc, char *argv[])
{
struct catalog_query *cq;
struct nvpair *nv;
work_queue_status_parse_command_line_arguments(argc, argv);
if(optind > argc) {
work_queue_status_show_help("work_queue_status");
exit(EXIT_FAILURE);
}
cq = catalog_query_create(CATALOG_HOST, CATALOG_PORT, time(0) + Work_Queue_Status_Timeout);
if(!cq) {
fprintf(stderr, "couldn't query catalog %s:%d: %s\n", CATALOG_HOST, CATALOG_PORT, strerror(errno));
return 1;
}
if(Work_Queue_Status_Mode == MODE_TABLE)
nvpair_print_table_header(stdout, headers);
while((nv = catalog_query_read(cq, time(0) + Work_Queue_Status_Timeout))) {
if(strcmp(nvpair_lookup_string(nv, "type"), CATALOG_TYPE_WORK_QUEUE_MASTER) == 0) {
if(Work_Queue_Status_Mode == MODE_TABLE)
nvpair_print_table(nv, stdout, headers);
else
nvpair_print_text(nv, stdout);
}
nvpair_delete(nv);
}
if(Work_Queue_Status_Mode == MODE_TABLE)
nvpair_print_table_footer(stdout, headers);
return EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
struct catalog_query *q;
struct nvpair *n;
time_t timeout = 60, stoptime;
const char *catalog_host = 0;
signed char c;
int i;
int count = 0;
int mode = MODE_TABLE;
INT64_T total = 0, avail = 0;
const char *filter_name = 0;
const char *filter_value = 0;
debug_config(argv[0]);
static struct option long_options[] = {
{"catalog", required_argument, 0, 'c'},
{"debug", required_argument, 0, 'd'},
{"debug-file", required_argument, 0, 'o'},
{"debug-rotate-max", required_argument, 0, 'O'},
{"server-space", required_argument, 0, 'A'},
{"all", no_argument, 0, 'a'},
{"timeout", required_argument, 0, 't'},
{"brief", no_argument, 0, 's'},
{"verbose", no_argument, 0, 'l'},
{"totals", no_argument, 0, 'T'},
{"version", no_argument, 0, 'v'},
{"help", no_argument, 0, 'h'},
{0, 0, 0, 0}
};
while((c = getopt_long(argc, argv, "aA:c:d:t:o:O:sTlvh", long_options, NULL)) > -1) {
switch (c) {
case 'a':
show_all_types = 1;
break;
case 'c':
catalog_host = optarg;
break;
case 'd':
debug_flags_set(optarg);
break;
case 't':
timeout = string_time_parse(optarg);
break;
case 'A':
minavail = string_metric_parse(optarg);
break;
case 'o':
debug_config_file(optarg);
break;
case 'O':
debug_config_file_size(string_metric_parse(optarg));
break;
case 'v':
cctools_version_print(stdout, argv[0]);
return 1;
case 's':
mode = MODE_SHORT;
break;
case 'l':
mode = MODE_LONG;
break;
case 'T':
mode = MODE_TOTAL;
break;
case 'h':
default:
show_help(argv[0]);
return 1;
}
}
cctools_version_debug(D_DEBUG, argv[0]);
if(argc - optind == 0) {
// fine, keep going
} else if((argc - optind) == 1) {
filter_name = "name";
filter_value = argv[optind];
} else if((argc - optind) == 2) {
filter_name = argv[optind];
filter_value = argv[optind + 1];
} else {
show_help(argv[0]);
return 1;
}
stoptime = time(0) + timeout;
q = catalog_query_create(catalog_host, 0, stoptime);
if(!q) {
fprintf(stderr, "couldn't query catalog: %s\n", strerror(errno));
return 1;
}
if(mode == MODE_TABLE) {
nvpair_print_table_header(stdout, headers);
}
while((n = catalog_query_read(q, stoptime))) {
table[count++] = n;
}
qsort(table, count, sizeof(*table), (int (*)(const void *, const void *)) compare_entries);
for(i = 0; i < count; i++) {
const char *etype = nvpair_lookup_string(table[i], "type");
if(!show_all_types) {
if(etype) {
if(!strcmp(etype, "chirp") || !strcmp(etype, "catalog")) {
/* ok, keep going */
} else {
continue;
}
} else {
continue;
}
}
if(minavail != 0) {
if(minavail > nvpair_lookup_integer(table[i], "avail")) {
continue;
}
}
if(filter_name) {
const char *v = nvpair_lookup_string(table[i], filter_name);
if(!v || strcmp(filter_value, v))
continue;
}
if(mode == MODE_SHORT) {
const char *t = nvpair_lookup_string(table[i], "type");
if(t && !strcmp(t, "chirp")) {
printf("%s:%d\n", nvpair_lookup_string(table[i], "name"), (int) nvpair_lookup_integer(table[i], "port"));
}
} else if(mode == MODE_LONG) {
nvpair_print_text(table[i], stdout);
} else if(mode == MODE_TABLE) {
nvpair_print_table(table[i], stdout, headers);
} else if(mode == MODE_TOTAL) {
avail += nvpair_lookup_integer(table[i], "avail");
total += nvpair_lookup_integer(table[i], "total");
}
}
if(mode == MODE_TOTAL) {
printf("NODES: %4d\n", count);
printf("TOTAL: %6sB\n", string_metric(total, -1, 0));
printf("AVAIL: %6sB\n", string_metric(avail, -1, 0));
printf("INUSE: %6sB\n", string_metric(total - avail, -1, 0));
}
if(mode == MODE_TABLE) {
nvpair_print_table_footer(stdout, headers);
}
return 0;
}
CONFUGA_IAPI int confugaS_catalog_sync (confuga *C)
{
static const char SQL[] =
"SELECT COUNT(*) FROM Confuga.StorageNode WHERE strftime('%s', 'now', '-2 minutes') <= lastheardfrom;"
"BEGIN IMMEDIATE TRANSACTION;"
"UPDATE Confuga.StorageNode"
" SET address = ?, avail = ?, backend = ?, bytes_read = ?, bytes_written = ?, cpu = ?, cpus = ?, lastheardfrom = ?, load1 = ?, load5 = ?, load15 = ?, memory_avail = ?, memory_total = ?, minfree = ?, name = ?, opsys = ?, opsysversion = ?, owner = ?, port = ?, starttime = ?, total = ?, total_ops = ?, url = ?, version = ?"
" WHERE hostport = ? || ':' || ? OR"
" hostport = ? || ':' || ? OR"
" 'chirp://' || hostport = ?;"
"END TRANSACTION;";
int rc;
sqlite3 *db = C->db;
sqlite3_stmt *stmt = NULL;
const char *current = SQL;
time_t stoptime = time(NULL)+15;
struct catalog_query *Q = NULL;
struct nvpair *nv = NULL;
sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t));
sqlcatchcode(sqlite3_step(stmt), SQLITE_ROW);
if (sqlite3_column_int(stmt, 0) > 0) {
rc = 0;
goto out;
}
sqlcatch(sqlite3_finalize(stmt); stmt = NULL);
debug(D_DEBUG|D_CONFUGA, "syncing with catalog");
Q = catalog_query_create(C->catalog_host, C->catalog_port, stoptime);
CATCH(Q == NULL ? errno : 0);
/* FIXME sqlcatch is silent about EAGAIN, what should we do? */
sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t));
sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE);
sqlcatch(sqlite3_finalize(stmt); stmt = NULL);
sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t));
while ((nv = catalog_query_read(Q, stoptime))) {
const char *type = nvpair_lookup_string(nv, "type");
if (type && strcmp(type, "chirp") == 0) {
int n = 1;
/* UPDATE */
sqlcatch(sqlite3_bind_text(stmt, n++, nvpair_lookup_string(nv, "address"), -1, SQLITE_TRANSIENT));
sqlcatch(sqlite3_bind_text(stmt, n++, nvpair_lookup_string(nv, "avail"), -1, SQLITE_TRANSIENT));
sqlcatch(sqlite3_bind_text(stmt, n++, nvpair_lookup_string(nv, "backend"), -1, SQLITE_TRANSIENT));
sqlcatch(sqlite3_bind_text(stmt, n++, nvpair_lookup_string(nv, "bytes_read"), -1, SQLITE_TRANSIENT));
sqlcatch(sqlite3_bind_text(stmt, n++, nvpair_lookup_string(nv, "bytes_written"), -1, SQLITE_TRANSIENT));
sqlcatch(sqlite3_bind_text(stmt, n++, nvpair_lookup_string(nv, "cpu"), -1, SQLITE_TRANSIENT));
sqlcatch(sqlite3_bind_text(stmt, n++, nvpair_lookup_string(nv, "cpus"), -1, SQLITE_TRANSIENT));
sqlcatch(sqlite3_bind_text(stmt, n++, nvpair_lookup_string(nv, "lastheardfrom"), -1, SQLITE_TRANSIENT));
sqlcatch(sqlite3_bind_text(stmt, n++, nvpair_lookup_string(nv, "load1"), -1, SQLITE_TRANSIENT));
sqlcatch(sqlite3_bind_text(stmt, n++, nvpair_lookup_string(nv, "load5"), -1, SQLITE_TRANSIENT));
sqlcatch(sqlite3_bind_text(stmt, n++, nvpair_lookup_string(nv, "load15"), -1, SQLITE_TRANSIENT));
sqlcatch(sqlite3_bind_text(stmt, n++, nvpair_lookup_string(nv, "memory_avail"), -1, SQLITE_TRANSIENT));
sqlcatch(sqlite3_bind_text(stmt, n++, nvpair_lookup_string(nv, "memory_total"), -1, SQLITE_TRANSIENT));
sqlcatch(sqlite3_bind_text(stmt, n++, nvpair_lookup_string(nv, "minfree"), -1, SQLITE_TRANSIENT));
sqlcatch(sqlite3_bind_text(stmt, n++, nvpair_lookup_string(nv, "name"), -1, SQLITE_TRANSIENT));
sqlcatch(sqlite3_bind_text(stmt, n++, nvpair_lookup_string(nv, "opsys"), -1, SQLITE_TRANSIENT));
sqlcatch(sqlite3_bind_text(stmt, n++, nvpair_lookup_string(nv, "opsysversion"), -1, SQLITE_TRANSIENT));
sqlcatch(sqlite3_bind_text(stmt, n++, nvpair_lookup_string(nv, "owner"), -1, SQLITE_TRANSIENT));
sqlcatch(sqlite3_bind_text(stmt, n++, nvpair_lookup_string(nv, "port"), -1, SQLITE_TRANSIENT));
sqlcatch(sqlite3_bind_text(stmt, n++, nvpair_lookup_string(nv, "starttime"), -1, SQLITE_TRANSIENT));
sqlcatch(sqlite3_bind_text(stmt, n++, nvpair_lookup_string(nv, "total"), -1, SQLITE_TRANSIENT));
sqlcatch(sqlite3_bind_text(stmt, n++, nvpair_lookup_string(nv, "total_ops"), -1, SQLITE_TRANSIENT));
sqlcatch(sqlite3_bind_text(stmt, n++, nvpair_lookup_string(nv, "url"), -1, SQLITE_TRANSIENT));
sqlcatch(sqlite3_bind_text(stmt, n++, nvpair_lookup_string(nv, "version"), -1, SQLITE_TRANSIENT));
/* WHERE hostport = ? */
sqlcatch(sqlite3_bind_text(stmt, n++, nvpair_lookup_string(nv, "name"), -1, SQLITE_TRANSIENT));
sqlcatch(sqlite3_bind_text(stmt, n++, nvpair_lookup_string(nv, "port"), -1, SQLITE_TRANSIENT));
sqlcatch(sqlite3_bind_text(stmt, n++, nvpair_lookup_string(nv, "address"), -1, SQLITE_TRANSIENT));
sqlcatch(sqlite3_bind_text(stmt, n++, nvpair_lookup_string(nv, "port"), -1, SQLITE_TRANSIENT));
sqlcatch(sqlite3_bind_text(stmt, n++, nvpair_lookup_string(nv, "url"), -1, SQLITE_TRANSIENT));
sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE);
sqlcatch(sqlite3_reset(stmt));
sqlcatch(sqlite3_clear_bindings(stmt));
}
}
sqlcatch(sqlite3_finalize(stmt); stmt = NULL);
sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t));
sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE);
sqlcatch(sqlite3_finalize(stmt); stmt = NULL);
rc = 0;
goto out;
out:
if (Q)
catalog_query_delete(Q);
sqlite3_finalize(stmt);
sqlend(db);
return rc;
}
static void handle_query(struct link *query_link)
{
FILE *stream;
char line[LINE_MAX];
char url[LINE_MAX];
char path[LINE_MAX];
char action[LINE_MAX];
char version[LINE_MAX];
char hostport[LINE_MAX];
char addr[LINK_ADDRESS_MAX];
char key[LINE_MAX];
int port;
time_t current;
char *hkey;
struct nvpair *nv;
int i, n;
link_address_remote(query_link, addr, &port);
debug(D_DEBUG, "www query from %s:%d", addr, port);
if(link_readline(query_link, line, LINE_MAX, time(0) + HANDLE_QUERY_TIMEOUT)) {
string_chomp(line);
if(sscanf(line, "%s %s %s", action, url, version) != 3) {
return;
}
// Consume the rest of the query
while(1) {
if(!link_readline(query_link, line, LINE_MAX, time(0) + HANDLE_QUERY_TIMEOUT)) {
return;
}
if(line[0] == 0) {
break;
}
}
} else {
return;
}
// Output response
stream = fdopen(link_fd(query_link), "w");
if(!stream) {
return;
}
link_nonblocking(query_link, 0);
current = time(0);
fprintf(stream, "HTTP/1.1 200 OK\n");
fprintf(stream, "Date: %s", ctime(¤t));
fprintf(stream, "Server: catalog_server\n");
fprintf(stream, "Connection: close\n");
if(sscanf(url, "http://%[^/]%s", hostport, path) == 2) {
// continue on
} else {
strcpy(path, url);
}
/* load the hash table entries into one big array */
n = 0;
nvpair_database_firstkey(table);
while(nvpair_database_nextkey(table, &hkey, &nv)) {
array[n] = nv;
n++;
}
/* sort the array by name before displaying */
qsort(array, n, sizeof(struct nvpair *), compare_nvpair);
if(!strcmp(path, "/query.text")) {
fprintf(stream, "Content-type: text/plain\n\n");
for(i = 0; i < n; i++)
nvpair_print_text(array[i], stream);
} else if(!strcmp(path, "/query.json")) {
fprintf(stream, "Content-type: text/plain\n\n");
fprintf(stream,"[\n");
for(i = 0; i < n; i++) {
nvpair_print_json(array[i], stream);
fprintf(stream,",\n");
}
fprintf(stream,"]\n");
} else if(!strcmp(path, "/query.oldclassads")) {
fprintf(stream, "Content-type: text/plain\n\n");
for(i = 0; i < n; i++)
nvpair_print_old_classads(array[i], stream);
} else if(!strcmp(path, "/query.newclassads")) {
fprintf(stream, "Content-type: text/plain\n\n");
for(i = 0; i < n; i++)
nvpair_print_new_classads(array[i], stream);
} else if(!strcmp(path, "/query.xml")) {
fprintf(stream, "Content-type: text/xml\n\n");
fprintf(stream, "<?xml version=\"1.0\" standalone=\"yes\"?>\n");
fprintf(stream, "<catalog>\n");
for(i = 0; i < n; i++)
nvpair_print_xml(array[i], stream);
fprintf(stream, "</catalog>\n");
} else if(sscanf(path, "/detail/%s", key) == 1) {
struct nvpair *nv;
fprintf(stream, "Content-type: text/html\n\n");
nv = nvpair_database_lookup(table, key);
if(nv) {
const char *name = nvpair_lookup_string(nv, "name");
if(!name)
name = "unknown";
fprintf(stream, "<title>%s catalog server: %s</title>\n", preferred_hostname, name);
fprintf(stream, "<center>\n");
fprintf(stream, "<h1>%s catalog server</h1>\n", preferred_hostname);
fprintf(stream, "<h2>%s</h2>\n", name);
fprintf(stream, "<p><a href=/>return to catalog view</a><p>\n");
nvpair_print_html_solo(nv, stream);
fprintf(stream, "</center>\n");
} else {
fprintf(stream, "<title>%s catalog server</title>\n", preferred_hostname);
fprintf(stream, "<center>\n");
fprintf(stream, "<h1>%s catalog server</h1>\n", preferred_hostname);
fprintf(stream, "<h2>Unknown Item!</h2>\n");
fprintf(stream, "</center>\n");
}
} else {
char avail_line[LINE_MAX];
char total_line[LINE_MAX];
INT64_T sum_total = 0;
INT64_T sum_avail = 0;
INT64_T sum_devices = 0;
fprintf(stream, "Content-type: text/html\n\n");
fprintf(stream, "<title>%s catalog server</title>\n", preferred_hostname);
fprintf(stream, "<center>\n");
fprintf(stream, "<h1>%s catalog server</h1>\n", preferred_hostname);
fprintf(stream, "<a href=/query.text>text</a> - ");
fprintf(stream, "<a href=/query.html>html</a> - ");
fprintf(stream, "<a href=/query.xml>xml</a> - ");
fprintf(stream, "<a href=/query.json>json</a> - ");
fprintf(stream, "<a href=/query.oldclassads>oldclassads</a> - ");
fprintf(stream, "<a href=/query.newclassads>newclassads</a>");
fprintf(stream, "<p>\n");
for(i = 0; i < n; i++) {
nv = array[i];
sum_total += nvpair_lookup_integer(nv, "total");
sum_avail += nvpair_lookup_integer(nv, "avail");
sum_devices++;
}
string_metric(sum_avail, -1, avail_line);
string_metric(sum_total, -1, total_line);
fprintf(stream, "<b>%sB available out of %sB on %d devices</b><p>\n", avail_line, total_line, (int) sum_devices);
nvpair_print_html_header(stream, html_headers);
for(i = 0; i < n; i++) {
nv = array[i];
make_hash_key(nv, key);
sprintf(url, "/detail/%s", key);
nvpair_print_html_with_link(nv, stream, html_headers, "name", url);
}
nvpair_print_html_footer(stream, html_headers);
fprintf(stream, "</center>\n");
}
fclose(stream);
}
static void handle_updates(struct datagram *update_port)
{
char data[DATAGRAM_PAYLOAD_MAX * 2];
char addr[DATAGRAM_ADDRESS_MAX];
char key[LINE_MAX];
int port;
int result;
struct nvpair *nv;
while(1) {
result = datagram_recv(update_port, data, DATAGRAM_PAYLOAD_MAX, addr, &port, 0);
if(result <= 0)
return;
data[result] = 0;
nv = nvpair_create();
nvpair_parse(nv, data);
nvpair_insert_string(nv, "address", addr);
nvpair_insert_integer(nv, "lastheardfrom", time(0));
/* If the server reports unbelievable numbers, simply reset them */
if(max_server_size > 0) {
INT64_T total = nvpair_lookup_integer(nv, "total");
INT64_T avail = nvpair_lookup_integer(nv, "avail");
if(total > max_server_size || avail > max_server_size) {
nvpair_insert_integer(nv, "total", max_server_size);
nvpair_insert_integer(nv, "avail", max_server_size);
}
}
/* Do not believe the server's reported name, just resolve it backwards. */
char name[DOMAIN_NAME_MAX];
if(domain_name_cache_lookup_reverse(addr, name)) {
nvpair_insert_string(nv, "name", name);
} else if (nvpair_lookup_string(nv, "name") == NULL) {
/* If rDNS is unsuccessful, then we use the name reported if given.
* This allows for hostnames that are only valid in the subnet of
* the reporting server. Here we set the "name" field to the IP
* Address, addr, because it was not set by the reporting server.
*/
nvpair_insert_string(nv, "name", addr);
}
make_hash_key(nv, key);
if(logfile) {
if(!nvpair_database_lookup(table,key)) {
nvpair_print_text(nv,logfile);
fflush(logfile);
}
}
nvpair_database_insert(table, key, nv);
debug(D_DEBUG, "received udp update from %s", key);
}
}
static int log_play( struct deltadb *db )
{
FILE *stream = stdin;
time_t current = 0;
struct nvpair *nv;
int line_number = 0;
struct hash_table *table = db->table;
char line[NVPAIR_LINE_MAX];
char key[NVPAIR_LINE_MAX];
char name[NVPAIR_LINE_MAX];
char value[NVPAIR_LINE_MAX];
char oper;
int notime = 1;
while(fgets(line,sizeof(line),stream)) {
line_number += 1;
int keep = 0;
if (line[0]=='.') return 0;
int n = sscanf(line,"%c %s %s %[^\n]",&oper,key,name,value);
if(n<1) continue;
switch(oper) {
case 'C':
nv = nvpair_create();
int res = nvpair_parse_stream(nv,stream);
if (res>0){
keep = 1;
struct argument *arg = db->args;
while (arg!=NULL){
const char *var = nvpair_lookup_string(nv,arg->param);
if ( var!=NULL && keep_object(arg,var)==0 ){
keep = 0;
break;
}
arg = arg->next;
}
nvpair_delete(hash_table_remove(db->table,key));
if (keep==1){
if (notime){
printf("T %lld\n",(long long)current);
notime = 0;
}
nvpair_print_text(nv,stdout);
hash_table_insert(db->table,key,nv);
}
}
if (!keep)
nvpair_delete(nv);
break;
case 'D':
nv = hash_table_remove(table,key);
if(nv){
if (notime){
printf("T %lld\n",(long long)current);
notime = 0;
}
printf("%s",line);
}//nvpair_delete(nv);
break;
case 'U':
nv = hash_table_lookup(table,key);
if(nv){
if (notime){
printf("T %lld\n",(long long)current);
notime = 0;
}
printf("%s",line);
}//nvpair_insert_string(nv,name,value);
break;
case 'R':
nv = hash_table_lookup(table,key);
if(nv){
if (notime){
printf("T %lld\n",(long long)current);
notime = 0;
}
printf("%s",line);
}//nvpair_remove(nv,name);
break;
case 'T':
current = atol(key);
notime = 1;
break;
default:
debug(D_NOTICE,"corrupt log data[%i]: %s",line_number,line);
fflush(stderr);
break;
}
}
return 1;
}
