static void remove_expired_records() { struct nvpair *nv; char *key; time_t current = time(0); // Only clean every clean_interval seconds. if((current-last_clean_time)<clean_interval) return; // After restarting, all records will have appear to be stale. // Run for a minimum of lifetime seconds before cleaning anything up. if((current-starttime)<lifetime ) return; nvpair_database_firstkey(table); while(nvpair_database_nextkey(table, &key, &nv)) { time_t lastheardfrom = nvpair_lookup_integer(nv,"lastheardfrom"); int this_lifetime = nvpair_lookup_integer(nv,"lifetime"); if(this_lifetime>0) { this_lifetime = MIN(lifetime,this_lifetime); } else { this_lifetime = lifetime; } if( (current-lastheardfrom) > this_lifetime ) { nv = nvpair_database_remove(table,key); if(nv) nvpair_delete(nv); } } last_clean_time = current; }
static int count_workers_needed( struct list *masters_list, int only_waiting ) { int needed_workers=0; int masters=0; struct nvpair *nv; if(!masters_list) { return needed_workers; } list_first_item(masters_list); while((nv=list_next_item(masters_list))) { const char *project = nvpair_lookup_string(nv,"project"); const char *host = nvpair_lookup_string(nv,"name"); const int port = nvpair_lookup_integer(nv,"port"); const char *owner = nvpair_lookup_string(nv,"owner"); const int tr = nvpair_lookup_integer(nv,"tasks_running"); const int tw = nvpair_lookup_integer(nv,"tasks_waiting"); const int tl = nvpair_lookup_integer(nv,"tasks_left"); const int capacity = nvpair_lookup_integer(nv,"capacity"); int tasks = tr+tw+tl; int need; if(only_waiting) { need = tw; } else { need = tasks; } if(consider_capacity && capacity>0) { need = MIN(capacity,tasks); } debug(D_WQ,"%s %s:%d %s %d %d %d",project,host,port,owner,tasks,capacity,need); needed_workers += need; masters++; } needed_workers = (int) ceil(needed_workers / tasks_per_worker); return needed_workers; }
static void make_hash_key(struct nvpair *nv, char *key) { const char *name, *addr; int port; addr = nvpair_lookup_string(nv, "address"); if(!addr) addr = "unknown"; port = nvpair_lookup_integer(nv, "port"); name = nvpair_lookup_string(nv, "name"); if(!name) name = "unknown"; sprintf(key, "%s:%d:%s", addr, port, name); }
int main(int argc, char *argv[]) { struct catalog_query *q; struct nvpair *n; time_t timeout = 60, stoptime; const char *catalog_host = 0; signed char c; int i; int count = 0; int mode = MODE_TABLE; INT64_T total = 0, avail = 0; const char *filter_name = 0; const char *filter_value = 0; debug_config(argv[0]); static struct option long_options[] = { {"catalog", required_argument, 0, 'c'}, {"debug", required_argument, 0, 'd'}, {"debug-file", required_argument, 0, 'o'}, {"debug-rotate-max", required_argument, 0, 'O'}, {"server-space", required_argument, 0, 'A'}, {"all", no_argument, 0, 'a'}, {"timeout", required_argument, 0, 't'}, {"brief", no_argument, 0, 's'}, {"verbose", no_argument, 0, 'l'}, {"totals", no_argument, 0, 'T'}, {"version", no_argument, 0, 'v'}, {"help", no_argument, 0, 'h'}, {0, 0, 0, 0} }; while((c = getopt_long(argc, argv, "aA:c:d:t:o:O:sTlvh", long_options, NULL)) > -1) { switch (c) { case 'a': show_all_types = 1; break; case 'c': catalog_host = optarg; break; case 'd': debug_flags_set(optarg); break; case 't': timeout = string_time_parse(optarg); break; case 'A': minavail = string_metric_parse(optarg); break; case 'o': debug_config_file(optarg); break; case 'O': debug_config_file_size(string_metric_parse(optarg)); break; case 'v': cctools_version_print(stdout, argv[0]); return 1; case 's': mode = MODE_SHORT; break; case 'l': mode = MODE_LONG; break; case 'T': mode = MODE_TOTAL; break; case 'h': default: show_help(argv[0]); return 1; } } cctools_version_debug(D_DEBUG, argv[0]); if(argc - optind == 0) { // fine, keep going } else if((argc - optind) == 1) { filter_name = "name"; filter_value = argv[optind]; } else if((argc - optind) == 2) { filter_name = argv[optind]; filter_value = argv[optind + 1]; } else { show_help(argv[0]); return 1; } stoptime = time(0) + timeout; q = catalog_query_create(catalog_host, 0, stoptime); if(!q) { fprintf(stderr, "couldn't query catalog: %s\n", strerror(errno)); return 1; } if(mode == MODE_TABLE) { nvpair_print_table_header(stdout, headers); } while((n = catalog_query_read(q, stoptime))) { table[count++] = n; } qsort(table, count, sizeof(*table), (int (*)(const void *, const void *)) compare_entries); for(i = 0; i < count; i++) { const char *etype = nvpair_lookup_string(table[i], "type"); if(!show_all_types) { if(etype) { if(!strcmp(etype, "chirp") || !strcmp(etype, "catalog")) { /* ok, keep going */ } else { continue; } } else { continue; } } if(minavail != 0) { if(minavail > nvpair_lookup_integer(table[i], "avail")) { continue; } } if(filter_name) { const char *v = nvpair_lookup_string(table[i], filter_name); if(!v || strcmp(filter_value, v)) continue; } if(mode == MODE_SHORT) { const char *t = nvpair_lookup_string(table[i], "type"); if(t && !strcmp(t, "chirp")) { printf("%s:%d\n", nvpair_lookup_string(table[i], "name"), (int) nvpair_lookup_integer(table[i], "port")); } } else if(mode == MODE_LONG) { nvpair_print_text(table[i], stdout); } else if(mode == MODE_TABLE) { nvpair_print_table(table[i], stdout, headers); } else if(mode == MODE_TOTAL) { avail += nvpair_lookup_integer(table[i], "avail"); total += nvpair_lookup_integer(table[i], "total"); } } if(mode == MODE_TOTAL) { printf("NODES: %4d\n", count); printf("TOTAL: %6sB\n", string_metric(total, -1, 0)); printf("AVAIL: %6sB\n", string_metric(avail, -1, 0)); printf("INUSE: %6sB\n", string_metric(total - avail, -1, 0)); } if(mode == MODE_TABLE) { nvpair_print_table_footer(stdout, headers); } return 0; }
struct work_queue_master *parse_work_queue_master_nvpair(struct nvpair *nv) { struct work_queue_master *m; m = xxmalloc(sizeof(struct work_queue_master)); strncpy(m->addr, nvpair_lookup_string(nv, "address"), LINK_ADDRESS_MAX); const char *project; project = nvpair_lookup_string(nv, "project"); if(project) { strncpy(m->proj, project, WORK_QUEUE_NAME_MAX); } else { strncpy(m->proj, "unknown", WORK_QUEUE_NAME_MAX); } m->port = nvpair_lookup_integer(nv, "port"); m->start_time = nvpair_lookup_integer(nv, "starttime"); m->priority = nvpair_lookup_integer(nv, "priority"); if(m->priority < 0) m->priority = 0; m->capacity = nvpair_lookup_integer(nv, "capacity"); m->tasks_waiting = nvpair_lookup_integer(nv, "tasks_waiting"); m->tasks_running = nvpair_lookup_integer(nv, "tasks_running"); m->tasks_complete = nvpair_lookup_integer(nv, "tasks_complete"); m->total_tasks_dispatched = nvpair_lookup_integer(nv, "total_tasks_dispatched"); m->workers_init = nvpair_lookup_integer(nv, "workers_init"); m->workers_ready = nvpair_lookup_integer(nv, "workers_ready"); m->workers_busy = nvpair_lookup_integer(nv, "workers_busy"); m->workers_full = nvpair_lookup_integer(nv, "workers_full"); m->workers = nvpair_lookup_integer(nv, "workers"); const char *workers_by_pool; workers_by_pool = nvpair_lookup_string(nv, "workers_by_pool"); if(workers_by_pool) { m->workers_by_pool = xxstrdup(workers_by_pool); } else { m->workers_by_pool = xxstrdup("unknown"); } const char *owner; owner = nvpair_lookup_string(nv, "owner"); if(owner) { strncpy(m->owner, nvpair_lookup_string(nv, "owner"), USERNAME_MAX); } else { strncpy(m->owner, "unknown", USERNAME_MAX); } return m; }
static void handle_query(struct link *query_link) { FILE *stream; char line[LINE_MAX]; char url[LINE_MAX]; char path[LINE_MAX]; char action[LINE_MAX]; char version[LINE_MAX]; char hostport[LINE_MAX]; char addr[LINK_ADDRESS_MAX]; char key[LINE_MAX]; int port; time_t current; char *hkey; struct nvpair *nv; int i, n; link_address_remote(query_link, addr, &port); debug(D_DEBUG, "www query from %s:%d", addr, port); if(link_readline(query_link, line, LINE_MAX, time(0) + HANDLE_QUERY_TIMEOUT)) { string_chomp(line); if(sscanf(line, "%s %s %s", action, url, version) != 3) { return; } // Consume the rest of the query while(1) { if(!link_readline(query_link, line, LINE_MAX, time(0) + HANDLE_QUERY_TIMEOUT)) { return; } if(line[0] == 0) { break; } } } else { return; } // Output response stream = fdopen(link_fd(query_link), "w"); if(!stream) { return; } link_nonblocking(query_link, 0); current = time(0); fprintf(stream, "HTTP/1.1 200 OK\n"); fprintf(stream, "Date: %s", ctime(¤t)); fprintf(stream, "Server: catalog_server\n"); fprintf(stream, "Connection: close\n"); if(sscanf(url, "http://%[^/]%s", hostport, path) == 2) { // continue on } else { strcpy(path, url); } /* load the hash table entries into one big array */ n = 0; nvpair_database_firstkey(table); while(nvpair_database_nextkey(table, &hkey, &nv)) { array[n] = nv; n++; } /* sort the array by name before displaying */ qsort(array, n, sizeof(struct nvpair *), compare_nvpair); if(!strcmp(path, "/query.text")) { fprintf(stream, "Content-type: text/plain\n\n"); for(i = 0; i < n; i++) nvpair_print_text(array[i], stream); } else if(!strcmp(path, "/query.json")) { fprintf(stream, "Content-type: text/plain\n\n"); fprintf(stream,"[\n"); for(i = 0; i < n; i++) { nvpair_print_json(array[i], stream); fprintf(stream,",\n"); } fprintf(stream,"]\n"); } else if(!strcmp(path, "/query.oldclassads")) { fprintf(stream, "Content-type: text/plain\n\n"); for(i = 0; i < n; i++) nvpair_print_old_classads(array[i], stream); } else if(!strcmp(path, "/query.newclassads")) { fprintf(stream, "Content-type: text/plain\n\n"); for(i = 0; i < n; i++) nvpair_print_new_classads(array[i], stream); } else if(!strcmp(path, "/query.xml")) { fprintf(stream, "Content-type: text/xml\n\n"); fprintf(stream, "<?xml version=\"1.0\" standalone=\"yes\"?>\n"); fprintf(stream, "<catalog>\n"); for(i = 0; i < n; i++) nvpair_print_xml(array[i], stream); fprintf(stream, "</catalog>\n"); } else if(sscanf(path, "/detail/%s", key) == 1) { struct nvpair *nv; fprintf(stream, "Content-type: text/html\n\n"); nv = nvpair_database_lookup(table, key); if(nv) { const char *name = nvpair_lookup_string(nv, "name"); if(!name) name = "unknown"; fprintf(stream, "<title>%s catalog server: %s</title>\n", preferred_hostname, name); fprintf(stream, "<center>\n"); fprintf(stream, "<h1>%s catalog server</h1>\n", preferred_hostname); fprintf(stream, "<h2>%s</h2>\n", name); fprintf(stream, "<p><a href=/>return to catalog view</a><p>\n"); nvpair_print_html_solo(nv, stream); fprintf(stream, "</center>\n"); } else { fprintf(stream, "<title>%s catalog server</title>\n", preferred_hostname); fprintf(stream, "<center>\n"); fprintf(stream, "<h1>%s catalog server</h1>\n", preferred_hostname); fprintf(stream, "<h2>Unknown Item!</h2>\n"); fprintf(stream, "</center>\n"); } } else { char avail_line[LINE_MAX]; char total_line[LINE_MAX]; INT64_T sum_total = 0; INT64_T sum_avail = 0; INT64_T sum_devices = 0; fprintf(stream, "Content-type: text/html\n\n"); fprintf(stream, "<title>%s catalog server</title>\n", preferred_hostname); fprintf(stream, "<center>\n"); fprintf(stream, "<h1>%s catalog server</h1>\n", preferred_hostname); fprintf(stream, "<a href=/query.text>text</a> - "); fprintf(stream, "<a href=/query.html>html</a> - "); fprintf(stream, "<a href=/query.xml>xml</a> - "); fprintf(stream, "<a href=/query.json>json</a> - "); fprintf(stream, "<a href=/query.oldclassads>oldclassads</a> - "); fprintf(stream, "<a href=/query.newclassads>newclassads</a>"); fprintf(stream, "<p>\n"); for(i = 0; i < n; i++) { nv = array[i]; sum_total += nvpair_lookup_integer(nv, "total"); sum_avail += nvpair_lookup_integer(nv, "avail"); sum_devices++; } string_metric(sum_avail, -1, avail_line); string_metric(sum_total, -1, total_line); fprintf(stream, "<b>%sB available out of %sB on %d devices</b><p>\n", avail_line, total_line, (int) sum_devices); nvpair_print_html_header(stream, html_headers); for(i = 0; i < n; i++) { nv = array[i]; make_hash_key(nv, key); sprintf(url, "/detail/%s", key); nvpair_print_html_with_link(nv, stream, html_headers, "name", url); } nvpair_print_html_footer(stream, html_headers); fprintf(stream, "</center>\n"); } fclose(stream); }
static void handle_updates(struct datagram *update_port) { char data[DATAGRAM_PAYLOAD_MAX * 2]; char addr[DATAGRAM_ADDRESS_MAX]; char key[LINE_MAX]; int port; int result; struct nvpair *nv; while(1) { result = datagram_recv(update_port, data, DATAGRAM_PAYLOAD_MAX, addr, &port, 0); if(result <= 0) return; data[result] = 0; nv = nvpair_create(); nvpair_parse(nv, data); nvpair_insert_string(nv, "address", addr); nvpair_insert_integer(nv, "lastheardfrom", time(0)); /* If the server reports unbelievable numbers, simply reset them */ if(max_server_size > 0) { INT64_T total = nvpair_lookup_integer(nv, "total"); INT64_T avail = nvpair_lookup_integer(nv, "avail"); if(total > max_server_size || avail > max_server_size) { nvpair_insert_integer(nv, "total", max_server_size); nvpair_insert_integer(nv, "avail", max_server_size); } } /* Do not believe the server's reported name, just resolve it backwards. */ char name[DOMAIN_NAME_MAX]; if(domain_name_cache_lookup_reverse(addr, name)) { nvpair_insert_string(nv, "name", name); } else if (nvpair_lookup_string(nv, "name") == NULL) { /* If rDNS is unsuccessful, then we use the name reported if given. * This allows for hostnames that are only valid in the subnet of * the reporting server. Here we set the "name" field to the IP * Address, addr, because it was not set by the reporting server. */ nvpair_insert_string(nv, "name", addr); } make_hash_key(nv, key); if(logfile) { if(!nvpair_database_lookup(table,key)) { nvpair_print_text(nv,logfile); fflush(logfile); } } nvpair_database_insert(table, key, nv); debug(D_DEBUG, "received udp update from %s", key); } }