struct jx * nvpair_to_jx( struct nvpair *nv )
{
struct jx *object = jx_object(0);
char *key;
char *value;
struct jx *jvalue;
long long integer_value;
double double_value;
nvpair_first_item(nv);
while(nvpair_next_item(nv,&key,&value)) {
if(!strcmp(value,"true")) {
jvalue = jx_boolean(1);
} else if(!strcmp(value,"false")) {
jvalue = jx_boolean(0);
} else if(!strcmp(value,"null")) {
jvalue = jx_null();
} else if(string_is_integer(value,&integer_value)) {
jvalue = jx_integer(integer_value);
} else if(string_is_float(value,&double_value)) {
jvalue = jx_double(double_value);
} else if(value[0]=='[' || value[0]=='{') {
jvalue = jx_parse_string(value);
if(!jvalue) jvalue = jx_string(value);
} else {
jvalue = jx_string(value);
}
jx_insert(object,jx_string(key),jvalue);
}
return object;
}
static int server_table_load(time_t stoptime)
{
struct catalog_query *q;
struct jx *j;
char *key;
void *item;
if((last_update + update_interval) > time(0)) {
return 1;
}
if(!server_table) {
server_table = hash_table_create(0, 0);
if(!server_table)
return 0;
}
if(inhibit_catalog_queries) {
debug(D_CHIRP, "catalog queries disabled\n");
return 1;
}
hash_table_firstkey(server_table);
while(hash_table_nextkey(server_table, &key, &item)) {
hash_table_remove(server_table, key);
jx_delete(item);
}
debug(D_CHIRP, "querying catalog at %s:%d", CATALOG_HOST, CATALOG_PORT);
struct jx *jexpr = jx_parse_string("type==\"chirp\"");
q = catalog_query_create(CATALOG_HOST, CATALOG_PORT, jexpr, stoptime);
jx_delete(jexpr);
if(!q)
return 0;
while((j = catalog_query_read(q, stoptime))) {
char name[CHIRP_PATH_MAX];
const char *hname;
int port;
hname = jx_lookup_string(j, "name");
if(hname) {
port = jx_lookup_integer(j, "port");
if(!port)
port = CHIRP_PORT;
sprintf(name, "%s:%d", hname, port);
hash_table_insert(server_table, name, j);
} else {
jx_delete(j);
}
}
catalog_query_delete(q);
last_update = time(0);
return 1;
}
static int log_replay( struct jx_database *db, const char *filename, time_t snapshot)
{
char line[LOG_LINE_MAX];
char value[LOG_LINE_MAX];
char name[LOG_LINE_MAX];
char key[LOG_LINE_MAX];
int n;
struct jx *jvalue, *jobject;
long long current = 0;
FILE *file = fopen(filename,"r");
if(!file) return 0;
while(fgets(line,sizeof(line),file)) {
if(line[0]=='C') {
n = sscanf(line,"C %s %[^\n]",key,value);
if(n==1) {
struct nvpair *nv = nvpair_create();
nvpair_parse_stream(nv,file);
jvalue = nvpair_to_jx(nv);
hash_table_insert(db->table,key,jvalue);
} else if(n==2) {
jvalue = jx_parse_string(value);
if(jvalue) {
hash_table_insert(db->table,key,jvalue);
} else {
corrupt_data(filename,line);
}
} else {
corrupt_data(filename,line);
continue;
}
} else if(line[0]=='D') {
n = sscanf(line,"D %s\n",key);
if(n!=1) {
corrupt_data(filename,line);
continue;
}
jx_delete(hash_table_remove(db->table,key));
} else if(line[0]=='U') {
n=sscanf(line,"U %s %s %[^\n],",key,name,value);
if(n!=3) {
corrupt_data(filename,line);
continue;
}
jobject = hash_table_lookup(db->table,key);
if(!jobject) {
corrupt_data(filename,line);
continue;
}
jvalue = jx_parse_string(value);
if(!jvalue) jvalue = jx_string(value);
struct jx *jname = jx_string(name);
jx_delete(jx_remove(jobject,jname));
jx_insert(jobject,jname,jvalue);
} else if(line[0]=='R') {
n=sscanf(line,"R %s %s",key,name);
if(n!=2) {
corrupt_data(filename,line);
continue;
}
jobject = hash_table_lookup(db->table,key);
if(!jobject) {
corrupt_data(filename,line);
continue;
}
struct jx *jname = jx_string(name);
jx_delete(jx_remove(jobject,jname));
jx_delete(jname);
} else if(line[0]=='T') {
n = sscanf(line,"T %lld",¤t);
if(n!=1) {
corrupt_data(filename,line);
continue;
}
if(current>snapshot) break;
} else if(line[0]=='\n') {
continue;
} else {
corrupt_data(filename,line);
}
}
fclose(file);
return 1;
}
int main( int argc, char *argv[] )
{
const char *dbdir=0;
const char *dbfile=0;
struct jx *where_expr = 0;
struct jx *filter_expr = 0;
struct list *output_exprs = list_create();
struct list *reduce_exprs = list_create();
time_t start_time = 0;
time_t stop_time = 0;
int display_every = 0;
int epoch_mode = 0;
char reduce_name[1024];
char reduce_attr[1024];
time_t current = time(0);
int c;
while((c=getopt_long(argc,argv,"D:L:o:w:f:F:T:e:tvh",long_options,0))!=-1) {
switch(c) {
case 'D':
dbdir = optarg;
break;
case 'L':
dbfile = optarg;
break;
case 'o':
if(2==sscanf(optarg,"%[^(](%[^)])",reduce_name,reduce_attr)) {
struct jx *reduce_expr = jx_parse_string(reduce_attr);
if(!reduce_expr) {
fprintf(stderr,"deltadb_query: invalid expression: %s\n",reduce_attr);
return 1;
}
struct deltadb_reduction *r = deltadb_reduction_create(reduce_name,reduce_expr);
if(!r) {
fprintf(stderr,"deltadb_query: invalid reduction: %s\n",reduce_name);
return 1;
}
list_push_tail(reduce_exprs,r);
} else {
struct jx *j = jx_parse_string(optarg);
if(!j) {
fprintf(stderr,"invalid expression: %s\n",optarg);
return 1;
}
list_push_tail(output_exprs,j);
}
break;
case 'w':
if(where_expr) {
fprintf(stderr,"Only one --where expression is allowed. Try joining the expressions with the && (and) operator.");
return 1;
}
where_expr = jx_parse_string(optarg);
if(!where_expr) {
fprintf(stderr,"invalid expression: %s\n",optarg);
return 1;
}
break;
case 'f':
if(filter_expr) {
fprintf(stderr,"Only one --filter expression is allowed. Try joining the expressions with the && (and) operator.");
return 1;
}
filter_expr = jx_parse_string(optarg);
if(!filter_expr) {
fprintf(stderr,"invalid expression: %s\n",optarg);
return 1;
}
break;
case 'F':
start_time = parse_time(optarg,current);
break;
case 'T':
stop_time = parse_time(optarg,current);
break;
case 'e':
display_every = string_time_parse(optarg);
break;
case 't':
epoch_mode = 1;
break;
case 'v':
cctools_version_print(stdout,"deltadb_query");
break;
case 'h':
show_help();
break;
}
}
if(!dbdir && !dbfile) {
fprintf(stderr,"deltadb_query: either --db or --file argument is required\n");
return 1;
}
if(start_time==0) {
fprintf(stderr,"deltadb_query: invalid --from time (must be \"YY-MM-DD\" or \"YY-MM-DD HH:MM:SS\")\n");
return 1;
}
if(stop_time==0) {
stop_time = time(0);
}
struct deltadb *db = deltadb_create(dbdir);
db->where_expr = where_expr;
db->filter_expr = filter_expr;
db->epoch_mode = epoch_mode;
db->output_exprs = output_exprs;
db->reduce_exprs = reduce_exprs;
db->display_every = display_every;
if(list_size(db->reduce_exprs) && list_size(db->output_exprs) ) {
struct deltadb_reduction *r = db->reduce_exprs->head->data;
const char *name = jx_print_string(db->output_exprs->head->data);
fprintf(stderr,"deltadb_query: cannot mix reductions like 'MAX(%s)' with plain outputs like '%s'\n",jx_print_string(r->expr),name);
return 1;
}
if(list_size(db->reduce_exprs)) {
display_mode = MODE_REDUCE;
} else if(list_size(db->output_exprs)) {
display_mode = MODE_OBJECT;
} else {
display_mode = MODE_STREAM;
}
if(dbfile) {
FILE *file = fopen(dbfile,"r");
if(!file) {
fprintf(stderr,"deltadb_query: couldn't open %s: %s\n",dbfile,strerror(errno));
return 1;
}
deltadb_process_stream(db,file,start_time,stop_time);
fclose(file);
} else {
log_play_time(db,start_time,stop_time);
}
return 0;
}
int main(int argc, char *argv[])
{
enum {
LONGOPT_SERVER_LASTHEARDFROM = INT_MAX-0,
LONGOPT_SERVER_PROJECT = INT_MAX-1,
LONGOPT_WHERE = INT_MAX-2,
};
static const struct option long_options[] = {
{"all", no_argument, 0, 'a'},
{"brief", no_argument, 0, 's'},
{"catalog", required_argument, 0, 'c'},
{"debug", required_argument, 0, 'd'},
{"debug-file", required_argument, 0, 'o'},
{"debug-rotate-max", required_argument, 0, 'O'},
{"help", no_argument, 0, 'h'},
{"server-lastheardfrom", required_argument, 0, LONGOPT_SERVER_LASTHEARDFROM},
{"server-project", required_argument, 0, LONGOPT_SERVER_PROJECT},
{"server-space", required_argument, 0, 'A'},
{"timeout", required_argument, 0, 't'},
{"totals", no_argument, 0, 'T'},
{"verbose", no_argument, 0, 'l'},
{"version", no_argument, 0, 'v'},
{"where", required_argument, 0, LONGOPT_WHERE },
{0, 0, 0, 0}
};
struct catalog_query *q;
struct jx *j;
time_t timeout = 60, stoptime;
const char *catalog_host = 0;
int i;
int c;
int count = 0;
int mode = MODE_TABLE;
INT64_T sum_total = 0, sum_avail = 0;
const char *filter_name = 0;
const char *filter_value = 0;
const char *where_expr = "true";
int show_all_types = 0;
const char *server_project = NULL;
time_t server_lastheardfrom = 0;
uint64_t server_avail = 0;
debug_config(argv[0]);
while((c = getopt_long(argc, argv, "aA:c:d:t:o:O:sTlvh", long_options, NULL)) > -1) {
switch (c) {
case 'a':
show_all_types = 1;
break;
case 'c':
catalog_host = optarg;
break;
case 'd':
debug_flags_set(optarg);
break;
case 't':
timeout = string_time_parse(optarg);
break;
case 'A':
server_avail = string_metric_parse(optarg);
break;
case 'o':
debug_config_file(optarg);
break;
case 'O':
debug_config_file_size(string_metric_parse(optarg));
break;
case 'v':
cctools_version_print(stdout, argv[0]);
return 1;
case 's':
mode = MODE_SHORT;
break;
case 'l':
mode = MODE_LONG;
break;
case 'T':
mode = MODE_TOTAL;
break;
case LONGOPT_SERVER_LASTHEARDFROM:
server_lastheardfrom = time(0)-string_time_parse(optarg);
break;
case LONGOPT_SERVER_PROJECT:
server_project = xxstrdup(optarg);
break;
case LONGOPT_WHERE:
where_expr = optarg;
break;
case 'h':
default:
show_help(argv[0]);
return 1;
}
}
cctools_version_debug(D_DEBUG, argv[0]);
if(argc - optind == 0) {
// fine, keep going
} else if((argc - optind) == 1) {
filter_name = "name";
filter_value = argv[optind];
} else if((argc - optind) == 2) {
filter_name = argv[optind];
filter_value = argv[optind + 1];
} else {
show_help(argv[0]);
return 1;
}
stoptime = time(0) + timeout;
const char *query_expr;
if(show_all_types) {
query_expr = where_expr;
} else {
query_expr = string_format("%s && (type==\"chirp\" || type==\"catalog\")",where_expr);
}
struct jx *jexpr = jx_parse_string(query_expr);
if(!jexpr) {
fprintf(stderr,"invalid expression: %s\n",query_expr);
return 1;
}
q = catalog_query_create(catalog_host, 0, jexpr, stoptime);
if(!q) {
fprintf(stderr, "couldn't query catalog: %s\n", strerror(errno));
return 1;
}
if(mode == MODE_TABLE) {
jx_table_print_header(headers,stdout);
} else if(mode==MODE_LONG) {
printf("[\n");
}
while((j = catalog_query_read(q, stoptime))) {
table[count++] = j;
}
catalog_query_delete(q);
qsort(table, count, sizeof(*table), (int (*)(const void *, const void *)) compare_entries);
for(i = 0; i < count; i++) {
const char *lastheardfrom = jx_lookup_string(table[i], "lastheardfrom");
if (lastheardfrom && (time_t)strtoul(lastheardfrom, NULL, 10) < server_lastheardfrom)
continue;
const char *avail = jx_lookup_string(table[i], "avail");
if (avail && strtoul(avail, NULL, 10) < server_avail)
continue;
const char *project = jx_lookup_string(table[i], "project");
if (server_project && (project == NULL || !(strcmp(project, server_project) == 0)))
continue;
if(filter_name) {
const char *v = jx_lookup_string(table[i], filter_name);
if(!v || strcmp(filter_value, v))
continue;
}
if(mode == MODE_SHORT) {
const char *t = jx_lookup_string(table[i], "type");
if(t && !strcmp(t, "chirp")) {
printf("%s:%d\n", jx_lookup_string(table[i], "name"), (int) jx_lookup_integer(table[i], "port"));
}
} else if(mode == MODE_LONG) {
if(i!=0) printf(",\n");
jx_print_stream(table[i],stdout);
} else if(mode == MODE_TABLE) {
jx_table_print(headers, table[i], stdout);
} else if(mode == MODE_TOTAL) {
sum_avail += jx_lookup_integer(table[i], "avail");
sum_total += jx_lookup_integer(table[i], "total");
}
}
for(i=0;i<count;i++) {
jx_delete(table[i]);
}
if(mode == MODE_TOTAL) {
printf("NODES: %4d\n", count);
printf("TOTAL: %6sB\n", string_metric(sum_total, -1, 0));
printf("AVAIL: %6sB\n", string_metric(sum_avail, -1, 0));
printf("INUSE: %6sB\n", string_metric(sum_total - sum_avail, -1, 0));
} else if(mode == MODE_TABLE) {
jx_table_print_footer(headers,stdout);
} else if(mode==MODE_LONG) {
printf("\n]\n");
}
return 0;
}
static void handle_update( const char *addr, int port, const char *raw_data, int raw_data_length, const char *protocol )
{
char key[LINE_MAX];
unsigned long data_length;
struct jx *j;
// If the packet starts with Control-Z (0x1A), it is compressed,
// so uncompress it to data[]. Otherwise just copy to data[];.
if(raw_data[0]==0x1A) {
data_length = sizeof(data);
int success = uncompress((Bytef*)data,&data_length,(const Bytef*)&raw_data[1],raw_data_length-1);
if(success!=Z_OK) {
debug(D_DEBUG,"warning: %s:%d sent invalid compressed data (ignoring it)\n",addr,port);
return;
}
} else {
memcpy(data,raw_data,raw_data_length);
data_length = raw_data_length;
}
// Make sure the string data is null terminated.
data[data_length] = 0;
// Once uncompressed, if it starts with a bracket,
// then it is JX/JSON, otherwise it is the legacy nvpair format.
if(data[0]=='{') {
j = jx_parse_string(data);
if(!j) {
debug(D_DEBUG,"warning: %s:%d sent invalid JSON data (ignoring it)\n%s\n",addr,port,data);
return;
}
if(!jx_is_constant(j)) {
debug(D_DEBUG,"warning: %s:%d sent non-constant JX data (ignoring it)\n%s\n",addr,port,data);
jx_delete(j);
return;
}
} else {
struct nvpair *nv = nvpair_create();
if(!nv) return;
nvpair_parse(nv, data);
j = nvpair_to_jx(nv);
nvpair_delete(nv);
}
jx_insert_string(j, "address", addr);
jx_insert_integer(j, "lastheardfrom", time(0));
/* If the server reports unbelievable numbers, simply reset them */
if(max_server_size > 0) {
INT64_T total = jx_lookup_integer(j, "total");
INT64_T avail = jx_lookup_integer(j, "avail");
if(total > max_server_size || avail > max_server_size) {
jx_insert_integer(j, "total", max_server_size);
jx_insert_integer(j, "avail", max_server_size);
}
}
/* Do not believe the server's reported name, just resolve it backwards. */
char name[DOMAIN_NAME_MAX];
if(domain_name_cache_lookup_reverse(addr, name)) {
/*
Special case: Prior bug resulted in multiple name
entries in logged data. When removing the name property,
keep looking until all items are removed.
*/
struct jx *jname = jx_string("name");
struct jx *n;
while((n=jx_remove(j,jname))) {
jx_delete(n);
}
jx_delete(jname);
jx_insert_string(j,"name",name);
} else if (jx_lookup_string(j, "name") == NULL) {
/* If rDNS is unsuccessful, then we use the name reported if given.
* This allows for hostnames that are only valid in the subnet of
* the reporting server. Here we set the "name" field to the IP
* Address, addr, because it was not set by the reporting server.
*/
jx_insert_string(j, "name", addr);
}
make_hash_key(j, key);
if(logfile) {
if(!jx_database_lookup(table,key)) {
jx_print_stream(j,logfile);
fprintf(logfile,"\n");
fflush(logfile);
}
}
jx_database_insert(table, key, j);
debug(D_DEBUG, "received %s update from %s",protocol,key);
}
