struct jx * nvpair_to_jx( struct nvpair *nv ) { struct jx *object = jx_object(0); char *key; char *value; struct jx *jvalue; long long integer_value; double double_value; nvpair_first_item(nv); while(nvpair_next_item(nv,&key,&value)) { if(!strcmp(value,"true")) { jvalue = jx_boolean(1); } else if(!strcmp(value,"false")) { jvalue = jx_boolean(0); } else if(!strcmp(value,"null")) { jvalue = jx_null(); } else if(string_is_integer(value,&integer_value)) { jvalue = jx_integer(integer_value); } else if(string_is_float(value,&double_value)) { jvalue = jx_double(double_value); } else if(value[0]=='[' || value[0]=='{') { jvalue = jx_parse_string(value); if(!jvalue) jvalue = jx_string(value); } else { jvalue = jx_string(value); } jx_insert(object,jx_string(key),jvalue); } return object; }
static int server_table_load(time_t stoptime) { struct catalog_query *q; struct jx *j; char *key; void *item; if((last_update + update_interval) > time(0)) { return 1; } if(!server_table) { server_table = hash_table_create(0, 0); if(!server_table) return 0; } if(inhibit_catalog_queries) { debug(D_CHIRP, "catalog queries disabled\n"); return 1; } hash_table_firstkey(server_table); while(hash_table_nextkey(server_table, &key, &item)) { hash_table_remove(server_table, key); jx_delete(item); } debug(D_CHIRP, "querying catalog at %s:%d", CATALOG_HOST, CATALOG_PORT); struct jx *jexpr = jx_parse_string("type==\"chirp\""); q = catalog_query_create(CATALOG_HOST, CATALOG_PORT, jexpr, stoptime); jx_delete(jexpr); if(!q) return 0; while((j = catalog_query_read(q, stoptime))) { char name[CHIRP_PATH_MAX]; const char *hname; int port; hname = jx_lookup_string(j, "name"); if(hname) { port = jx_lookup_integer(j, "port"); if(!port) port = CHIRP_PORT; sprintf(name, "%s:%d", hname, port); hash_table_insert(server_table, name, j); } else { jx_delete(j); } } catalog_query_delete(q); last_update = time(0); return 1; }
static int log_replay( struct jx_database *db, const char *filename, time_t snapshot) { char line[LOG_LINE_MAX]; char value[LOG_LINE_MAX]; char name[LOG_LINE_MAX]; char key[LOG_LINE_MAX]; int n; struct jx *jvalue, *jobject; long long current = 0; FILE *file = fopen(filename,"r"); if(!file) return 0; while(fgets(line,sizeof(line),file)) { if(line[0]=='C') { n = sscanf(line,"C %s %[^\n]",key,value); if(n==1) { struct nvpair *nv = nvpair_create(); nvpair_parse_stream(nv,file); jvalue = nvpair_to_jx(nv); hash_table_insert(db->table,key,jvalue); } else if(n==2) { jvalue = jx_parse_string(value); if(jvalue) { hash_table_insert(db->table,key,jvalue); } else { corrupt_data(filename,line); } } else { corrupt_data(filename,line); continue; } } else if(line[0]=='D') { n = sscanf(line,"D %s\n",key); if(n!=1) { corrupt_data(filename,line); continue; } jx_delete(hash_table_remove(db->table,key)); } else if(line[0]=='U') { n=sscanf(line,"U %s %s %[^\n],",key,name,value); if(n!=3) { corrupt_data(filename,line); continue; } jobject = hash_table_lookup(db->table,key); if(!jobject) { corrupt_data(filename,line); continue; } jvalue = jx_parse_string(value); if(!jvalue) jvalue = jx_string(value); struct jx *jname = jx_string(name); jx_delete(jx_remove(jobject,jname)); jx_insert(jobject,jname,jvalue); } else if(line[0]=='R') { n=sscanf(line,"R %s %s",key,name); if(n!=2) { corrupt_data(filename,line); continue; } jobject = hash_table_lookup(db->table,key); if(!jobject) { corrupt_data(filename,line); continue; } struct jx *jname = jx_string(name); jx_delete(jx_remove(jobject,jname)); jx_delete(jname); } else if(line[0]=='T') { n = sscanf(line,"T %lld",¤t); if(n!=1) { corrupt_data(filename,line); continue; } if(current>snapshot) break; } else if(line[0]=='\n') { continue; } else { corrupt_data(filename,line); } } fclose(file); return 1; }
int main( int argc, char *argv[] ) { const char *dbdir=0; const char *dbfile=0; struct jx *where_expr = 0; struct jx *filter_expr = 0; struct list *output_exprs = list_create(); struct list *reduce_exprs = list_create(); time_t start_time = 0; time_t stop_time = 0; int display_every = 0; int epoch_mode = 0; char reduce_name[1024]; char reduce_attr[1024]; time_t current = time(0); int c; while((c=getopt_long(argc,argv,"D:L:o:w:f:F:T:e:tvh",long_options,0))!=-1) { switch(c) { case 'D': dbdir = optarg; break; case 'L': dbfile = optarg; break; case 'o': if(2==sscanf(optarg,"%[^(](%[^)])",reduce_name,reduce_attr)) { struct jx *reduce_expr = jx_parse_string(reduce_attr); if(!reduce_expr) { fprintf(stderr,"deltadb_query: invalid expression: %s\n",reduce_attr); return 1; } struct deltadb_reduction *r = deltadb_reduction_create(reduce_name,reduce_expr); if(!r) { fprintf(stderr,"deltadb_query: invalid reduction: %s\n",reduce_name); return 1; } list_push_tail(reduce_exprs,r); } else { struct jx *j = jx_parse_string(optarg); if(!j) { fprintf(stderr,"invalid expression: %s\n",optarg); return 1; } list_push_tail(output_exprs,j); } break; case 'w': if(where_expr) { fprintf(stderr,"Only one --where expression is allowed. Try joining the expressions with the && (and) operator."); return 1; } where_expr = jx_parse_string(optarg); if(!where_expr) { fprintf(stderr,"invalid expression: %s\n",optarg); return 1; } break; case 'f': if(filter_expr) { fprintf(stderr,"Only one --filter expression is allowed. Try joining the expressions with the && (and) operator."); return 1; } filter_expr = jx_parse_string(optarg); if(!filter_expr) { fprintf(stderr,"invalid expression: %s\n",optarg); return 1; } break; case 'F': start_time = parse_time(optarg,current); break; case 'T': stop_time = parse_time(optarg,current); break; case 'e': display_every = string_time_parse(optarg); break; case 't': epoch_mode = 1; break; case 'v': cctools_version_print(stdout,"deltadb_query"); break; case 'h': show_help(); break; } } if(!dbdir && !dbfile) { fprintf(stderr,"deltadb_query: either --db or --file argument is required\n"); return 1; } if(start_time==0) { fprintf(stderr,"deltadb_query: invalid --from time (must be \"YY-MM-DD\" or \"YY-MM-DD HH:MM:SS\")\n"); return 1; } if(stop_time==0) { stop_time = time(0); } struct deltadb *db = deltadb_create(dbdir); db->where_expr = where_expr; db->filter_expr = filter_expr; db->epoch_mode = epoch_mode; db->output_exprs = output_exprs; db->reduce_exprs = reduce_exprs; db->display_every = display_every; if(list_size(db->reduce_exprs) && list_size(db->output_exprs) ) { struct deltadb_reduction *r = db->reduce_exprs->head->data; const char *name = jx_print_string(db->output_exprs->head->data); fprintf(stderr,"deltadb_query: cannot mix reductions like 'MAX(%s)' with plain outputs like '%s'\n",jx_print_string(r->expr),name); return 1; } if(list_size(db->reduce_exprs)) { display_mode = MODE_REDUCE; } else if(list_size(db->output_exprs)) { display_mode = MODE_OBJECT; } else { display_mode = MODE_STREAM; } if(dbfile) { FILE *file = fopen(dbfile,"r"); if(!file) { fprintf(stderr,"deltadb_query: couldn't open %s: %s\n",dbfile,strerror(errno)); return 1; } deltadb_process_stream(db,file,start_time,stop_time); fclose(file); } else { log_play_time(db,start_time,stop_time); } return 0; }
int main(int argc, char *argv[]) { enum { LONGOPT_SERVER_LASTHEARDFROM = INT_MAX-0, LONGOPT_SERVER_PROJECT = INT_MAX-1, LONGOPT_WHERE = INT_MAX-2, }; static const struct option long_options[] = { {"all", no_argument, 0, 'a'}, {"brief", no_argument, 0, 's'}, {"catalog", required_argument, 0, 'c'}, {"debug", required_argument, 0, 'd'}, {"debug-file", required_argument, 0, 'o'}, {"debug-rotate-max", required_argument, 0, 'O'}, {"help", no_argument, 0, 'h'}, {"server-lastheardfrom", required_argument, 0, LONGOPT_SERVER_LASTHEARDFROM}, {"server-project", required_argument, 0, LONGOPT_SERVER_PROJECT}, {"server-space", required_argument, 0, 'A'}, {"timeout", required_argument, 0, 't'}, {"totals", no_argument, 0, 'T'}, {"verbose", no_argument, 0, 'l'}, {"version", no_argument, 0, 'v'}, {"where", required_argument, 0, LONGOPT_WHERE }, {0, 0, 0, 0} }; struct catalog_query *q; struct jx *j; time_t timeout = 60, stoptime; const char *catalog_host = 0; int i; int c; int count = 0; int mode = MODE_TABLE; INT64_T sum_total = 0, sum_avail = 0; const char *filter_name = 0; const char *filter_value = 0; const char *where_expr = "true"; int show_all_types = 0; const char *server_project = NULL; time_t server_lastheardfrom = 0; uint64_t server_avail = 0; debug_config(argv[0]); while((c = getopt_long(argc, argv, "aA:c:d:t:o:O:sTlvh", long_options, NULL)) > -1) { switch (c) { case 'a': show_all_types = 1; break; case 'c': catalog_host = optarg; break; case 'd': debug_flags_set(optarg); break; case 't': timeout = string_time_parse(optarg); break; case 'A': server_avail = string_metric_parse(optarg); break; case 'o': debug_config_file(optarg); break; case 'O': debug_config_file_size(string_metric_parse(optarg)); break; case 'v': cctools_version_print(stdout, argv[0]); return 1; case 's': mode = MODE_SHORT; break; case 'l': mode = MODE_LONG; break; case 'T': mode = MODE_TOTAL; break; case LONGOPT_SERVER_LASTHEARDFROM: server_lastheardfrom = time(0)-string_time_parse(optarg); break; case LONGOPT_SERVER_PROJECT: server_project = xxstrdup(optarg); break; case LONGOPT_WHERE: where_expr = optarg; break; case 'h': default: show_help(argv[0]); return 1; } } cctools_version_debug(D_DEBUG, argv[0]); if(argc - optind == 0) { // fine, keep going } else if((argc - optind) == 1) { filter_name = "name"; filter_value = argv[optind]; } else if((argc - optind) == 2) { filter_name = argv[optind]; filter_value = argv[optind + 1]; } else { show_help(argv[0]); return 1; } stoptime = time(0) + timeout; const char *query_expr; if(show_all_types) { query_expr = where_expr; } else { query_expr = string_format("%s && (type==\"chirp\" || type==\"catalog\")",where_expr); } struct jx *jexpr = jx_parse_string(query_expr); if(!jexpr) { fprintf(stderr,"invalid expression: %s\n",query_expr); return 1; } q = catalog_query_create(catalog_host, 0, jexpr, stoptime); if(!q) { fprintf(stderr, "couldn't query catalog: %s\n", strerror(errno)); return 1; } if(mode == MODE_TABLE) { jx_table_print_header(headers,stdout); } else if(mode==MODE_LONG) { printf("[\n"); } while((j = catalog_query_read(q, stoptime))) { table[count++] = j; } catalog_query_delete(q); qsort(table, count, sizeof(*table), (int (*)(const void *, const void *)) compare_entries); for(i = 0; i < count; i++) { const char *lastheardfrom = jx_lookup_string(table[i], "lastheardfrom"); if (lastheardfrom && (time_t)strtoul(lastheardfrom, NULL, 10) < server_lastheardfrom) continue; const char *avail = jx_lookup_string(table[i], "avail"); if (avail && strtoul(avail, NULL, 10) < server_avail) continue; const char *project = jx_lookup_string(table[i], "project"); if (server_project && (project == NULL || !(strcmp(project, server_project) == 0))) continue; if(filter_name) { const char *v = jx_lookup_string(table[i], filter_name); if(!v || strcmp(filter_value, v)) continue; } if(mode == MODE_SHORT) { const char *t = jx_lookup_string(table[i], "type"); if(t && !strcmp(t, "chirp")) { printf("%s:%d\n", jx_lookup_string(table[i], "name"), (int) jx_lookup_integer(table[i], "port")); } } else if(mode == MODE_LONG) { if(i!=0) printf(",\n"); jx_print_stream(table[i],stdout); } else if(mode == MODE_TABLE) { jx_table_print(headers, table[i], stdout); } else if(mode == MODE_TOTAL) { sum_avail += jx_lookup_integer(table[i], "avail"); sum_total += jx_lookup_integer(table[i], "total"); } } for(i=0;i<count;i++) { jx_delete(table[i]); } if(mode == MODE_TOTAL) { printf("NODES: %4d\n", count); printf("TOTAL: %6sB\n", string_metric(sum_total, -1, 0)); printf("AVAIL: %6sB\n", string_metric(sum_avail, -1, 0)); printf("INUSE: %6sB\n", string_metric(sum_total - sum_avail, -1, 0)); } else if(mode == MODE_TABLE) { jx_table_print_footer(headers,stdout); } else if(mode==MODE_LONG) { printf("\n]\n"); } return 0; }
static void handle_update( const char *addr, int port, const char *raw_data, int raw_data_length, const char *protocol ) { char key[LINE_MAX]; unsigned long data_length; struct jx *j; // If the packet starts with Control-Z (0x1A), it is compressed, // so uncompress it to data[]. Otherwise just copy to data[];. if(raw_data[0]==0x1A) { data_length = sizeof(data); int success = uncompress((Bytef*)data,&data_length,(const Bytef*)&raw_data[1],raw_data_length-1); if(success!=Z_OK) { debug(D_DEBUG,"warning: %s:%d sent invalid compressed data (ignoring it)\n",addr,port); return; } } else { memcpy(data,raw_data,raw_data_length); data_length = raw_data_length; } // Make sure the string data is null terminated. data[data_length] = 0; // Once uncompressed, if it starts with a bracket, // then it is JX/JSON, otherwise it is the legacy nvpair format. if(data[0]=='{') { j = jx_parse_string(data); if(!j) { debug(D_DEBUG,"warning: %s:%d sent invalid JSON data (ignoring it)\n%s\n",addr,port,data); return; } if(!jx_is_constant(j)) { debug(D_DEBUG,"warning: %s:%d sent non-constant JX data (ignoring it)\n%s\n",addr,port,data); jx_delete(j); return; } } else { struct nvpair *nv = nvpair_create(); if(!nv) return; nvpair_parse(nv, data); j = nvpair_to_jx(nv); nvpair_delete(nv); } jx_insert_string(j, "address", addr); jx_insert_integer(j, "lastheardfrom", time(0)); /* If the server reports unbelievable numbers, simply reset them */ if(max_server_size > 0) { INT64_T total = jx_lookup_integer(j, "total"); INT64_T avail = jx_lookup_integer(j, "avail"); if(total > max_server_size || avail > max_server_size) { jx_insert_integer(j, "total", max_server_size); jx_insert_integer(j, "avail", max_server_size); } } /* Do not believe the server's reported name, just resolve it backwards. */ char name[DOMAIN_NAME_MAX]; if(domain_name_cache_lookup_reverse(addr, name)) { /* Special case: Prior bug resulted in multiple name entries in logged data. When removing the name property, keep looking until all items are removed. */ struct jx *jname = jx_string("name"); struct jx *n; while((n=jx_remove(j,jname))) { jx_delete(n); } jx_delete(jname); jx_insert_string(j,"name",name); } else if (jx_lookup_string(j, "name") == NULL) { /* If rDNS is unsuccessful, then we use the name reported if given. * This allows for hostnames that are only valid in the subnet of * the reporting server. Here we set the "name" field to the IP * Address, addr, because it was not set by the reporting server. */ jx_insert_string(j, "name", addr); } make_hash_key(j, key); if(logfile) { if(!jx_database_lookup(table,key)) { jx_print_stream(j,logfile); fprintf(logfile,"\n"); fflush(logfile); } } jx_database_insert(table, key, j); debug(D_DEBUG, "received %s update from %s",protocol,key); }