Ejemplo n.º 1
0
static void remove_expired_records()
{
	struct nvpair *nv;
	char *key;

	time_t current = time(0);

	// Only clean every clean_interval seconds.
	if((current-last_clean_time)<clean_interval) return;

	// After restarting, all records will have appear to be stale.
	// Run for a minimum of lifetime seconds before cleaning anything up.
	if((current-starttime)<lifetime ) return;

	nvpair_database_firstkey(table);
	while(nvpair_database_nextkey(table, &key, &nv)) {
		time_t lastheardfrom = nvpair_lookup_integer(nv,"lastheardfrom");

		int this_lifetime = nvpair_lookup_integer(nv,"lifetime");
		if(this_lifetime>0) {
			this_lifetime = MIN(lifetime,this_lifetime);
		} else {
			this_lifetime = lifetime;
		}

		if( (current-lastheardfrom) > this_lifetime ) {
		    	nv = nvpair_database_remove(table,key);
			if(nv) nvpair_delete(nv);
		}
	}

	last_clean_time = current;
}
Ejemplo n.º 2
0
static int count_workers_needed( struct list *masters_list, int only_waiting )
{
	int needed_workers=0;
	int masters=0;
	struct nvpair *nv;

	if(!masters_list) {
		return needed_workers;
	}

	list_first_item(masters_list);
	while((nv=list_next_item(masters_list))) {

		const char *project =   nvpair_lookup_string(nv,"project");
		const char *host =   nvpair_lookup_string(nv,"name");
		const int  port =    nvpair_lookup_integer(nv,"port");
		const char *owner =  nvpair_lookup_string(nv,"owner");
		const int tr =       nvpair_lookup_integer(nv,"tasks_running");
		const int tw =       nvpair_lookup_integer(nv,"tasks_waiting");
		const int tl =       nvpair_lookup_integer(nv,"tasks_left");
		const int capacity = nvpair_lookup_integer(nv,"capacity");

		int tasks = tr+tw+tl;

		int need;

		if(only_waiting) {
			need = tw;
		} else {
			need = tasks;
		}

		if(consider_capacity && capacity>0) {
			need = MIN(capacity,tasks);
		}

		debug(D_WQ,"%s %s:%d %s %d %d %d",project,host,port,owner,tasks,capacity,need);

		needed_workers += need;
		masters++;
	}

	needed_workers = (int) ceil(needed_workers / tasks_per_worker);

	return needed_workers;
}
Ejemplo n.º 3
0
static void make_hash_key(struct nvpair *nv, char *key)
{
	const char *name, *addr;
	int port;

	addr = nvpair_lookup_string(nv, "address");
	if(!addr)
		addr = "unknown";

	port = nvpair_lookup_integer(nv, "port");

	name = nvpair_lookup_string(nv, "name");
	if(!name)
		name = "unknown";

	sprintf(key, "%s:%d:%s", addr, port, name);
}
Ejemplo n.º 4
0
int main(int argc, char *argv[])
{
	struct catalog_query *q;
	struct nvpair *n;
	time_t timeout = 60, stoptime;
	const char *catalog_host = 0;
	signed char c;
	int i;
	int count = 0;
	int mode = MODE_TABLE;
	INT64_T total = 0, avail = 0;
	const char *filter_name = 0;
	const char *filter_value = 0;

	debug_config(argv[0]);


	static struct option long_options[] = {
		{"catalog", required_argument, 0, 'c'},
		{"debug", required_argument, 0, 'd'},
		{"debug-file", required_argument, 0, 'o'},
		{"debug-rotate-max", required_argument, 0, 'O'},
		{"server-space", required_argument, 0, 'A'},
		{"all", no_argument, 0, 'a'},
		{"timeout", required_argument, 0, 't'},
		{"brief", no_argument, 0, 's'},
		{"verbose", no_argument, 0, 'l'},
		{"totals", no_argument, 0, 'T'},
		{"version", no_argument, 0, 'v'},
		{"help", no_argument, 0, 'h'},
		{0, 0, 0, 0}
	};

	while((c = getopt_long(argc, argv, "aA:c:d:t:o:O:sTlvh", long_options, NULL)) > -1) {
		switch (c) {
		case 'a':
			show_all_types = 1;
			break;
		case 'c':
			catalog_host = optarg;
			break;
		case 'd':
			debug_flags_set(optarg);
			break;
		case 't':
			timeout = string_time_parse(optarg);
			break;
		case 'A':
			minavail = string_metric_parse(optarg);
			break;
		case 'o':
			debug_config_file(optarg);
			break;
		case 'O':
			debug_config_file_size(string_metric_parse(optarg));
			break;
		case 'v':
			cctools_version_print(stdout, argv[0]);
			return 1;
		case 's':
			mode = MODE_SHORT;
			break;
		case 'l':
			mode = MODE_LONG;
			break;
		case 'T':
			mode = MODE_TOTAL;
			break;
		case 'h':
		default:
			show_help(argv[0]);
			return 1;
		}
	}

	cctools_version_debug(D_DEBUG, argv[0]);

	if(argc - optind == 0) {
		// fine, keep going
	} else if((argc - optind) == 1) {
		filter_name = "name";
		filter_value = argv[optind];
	} else if((argc - optind) == 2) {
		filter_name = argv[optind];
		filter_value = argv[optind + 1];
	} else {
		show_help(argv[0]);
		return 1;
	}

	stoptime = time(0) + timeout;

	q = catalog_query_create(catalog_host, 0, stoptime);
	if(!q) {
		fprintf(stderr, "couldn't query catalog: %s\n", strerror(errno));
		return 1;
	}

	if(mode == MODE_TABLE) {
		nvpair_print_table_header(stdout, headers);
	}

	while((n = catalog_query_read(q, stoptime))) {
		table[count++] = n;
	}

	qsort(table, count, sizeof(*table), (int (*)(const void *, const void *)) compare_entries);

	for(i = 0; i < count; i++) {
		const char *etype = nvpair_lookup_string(table[i], "type");
		if(!show_all_types) {
			if(etype) {
				if(!strcmp(etype, "chirp") || !strcmp(etype, "catalog")) {
					/* ok, keep going */
				} else {
					continue;
				}
			} else {
				continue;
			}
		}

		if(minavail != 0) {
			if(minavail > nvpair_lookup_integer(table[i], "avail")) {
				continue;
			}
		}

		if(filter_name) {
			const char *v = nvpair_lookup_string(table[i], filter_name);
			if(!v || strcmp(filter_value, v))
				continue;
		}

		if(mode == MODE_SHORT) {
			const char *t = nvpair_lookup_string(table[i], "type");
			if(t && !strcmp(t, "chirp")) {
				printf("%s:%d\n", nvpair_lookup_string(table[i], "name"), (int) nvpair_lookup_integer(table[i], "port"));
			}
		} else if(mode == MODE_LONG) {
			nvpair_print_text(table[i], stdout);
		} else if(mode == MODE_TABLE) {
			nvpair_print_table(table[i], stdout, headers);
		} else if(mode == MODE_TOTAL) {
			avail += nvpair_lookup_integer(table[i], "avail");
			total += nvpair_lookup_integer(table[i], "total");
		}
	}

	if(mode == MODE_TOTAL) {
		printf("NODES: %4d\n", count);
		printf("TOTAL: %6sB\n", string_metric(total, -1, 0));
		printf("AVAIL: %6sB\n", string_metric(avail, -1, 0));
		printf("INUSE: %6sB\n", string_metric(total - avail, -1, 0));
	}

	if(mode == MODE_TABLE) {
		nvpair_print_table_footer(stdout, headers);
	}

	return 0;
}
Ejemplo n.º 5
0
struct work_queue_master *parse_work_queue_master_nvpair(struct nvpair *nv)
{
	struct work_queue_master *m;

	m = xxmalloc(sizeof(struct work_queue_master));

	strncpy(m->addr, nvpair_lookup_string(nv, "address"), LINK_ADDRESS_MAX);

	const char *project;
	project = nvpair_lookup_string(nv, "project");
	if(project) {
		strncpy(m->proj, project, WORK_QUEUE_NAME_MAX);
	} else {
		strncpy(m->proj, "unknown", WORK_QUEUE_NAME_MAX);
	}

	m->port = nvpair_lookup_integer(nv, "port");
	m->start_time = nvpair_lookup_integer(nv, "starttime");
	m->priority = nvpair_lookup_integer(nv, "priority");
	if(m->priority < 0) m->priority = 0;
	m->capacity = nvpair_lookup_integer(nv, "capacity");
	m->tasks_waiting = nvpair_lookup_integer(nv, "tasks_waiting");
	m->tasks_running = nvpair_lookup_integer(nv, "tasks_running");
	m->tasks_complete = nvpair_lookup_integer(nv, "tasks_complete");
	m->total_tasks_dispatched = nvpair_lookup_integer(nv, "total_tasks_dispatched");
	m->workers_init = nvpair_lookup_integer(nv, "workers_init");
	m->workers_ready = nvpair_lookup_integer(nv, "workers_ready");
	m->workers_busy = nvpair_lookup_integer(nv, "workers_busy");
	m->workers_full = nvpair_lookup_integer(nv, "workers_full");
	m->workers = nvpair_lookup_integer(nv, "workers");

	const char *workers_by_pool;
	workers_by_pool = nvpair_lookup_string(nv, "workers_by_pool");

	if(workers_by_pool) {
		m->workers_by_pool = xxstrdup(workers_by_pool);
	} else {
		m->workers_by_pool = xxstrdup("unknown");
	}

	const char *owner;
	owner = nvpair_lookup_string(nv, "owner");
	if(owner) {
		strncpy(m->owner, nvpair_lookup_string(nv, "owner"), USERNAME_MAX);
	} else {
		strncpy(m->owner, "unknown", USERNAME_MAX);
	}

	return m;
}
Ejemplo n.º 6
0
static void handle_query(struct link *query_link)
{
	FILE *stream;
	char line[LINE_MAX];
	char url[LINE_MAX];
	char path[LINE_MAX];
	char action[LINE_MAX];
	char version[LINE_MAX];
	char hostport[LINE_MAX];
	char addr[LINK_ADDRESS_MAX];
	char key[LINE_MAX];
	int port;
	time_t current;

	char *hkey;
	struct nvpair *nv;
	int i, n;

	link_address_remote(query_link, addr, &port);
	debug(D_DEBUG, "www query from %s:%d", addr, port);

	if(link_readline(query_link, line, LINE_MAX, time(0) + HANDLE_QUERY_TIMEOUT)) {
		string_chomp(line);
		if(sscanf(line, "%s %s %s", action, url, version) != 3) {
			return;
		}

		// Consume the rest of the query
		while(1) {
			if(!link_readline(query_link, line, LINE_MAX, time(0) + HANDLE_QUERY_TIMEOUT)) {
				return;
			}

			if(line[0] == 0) {
				break;
			}
		}
	} else {
		return;
	}

	// Output response
	stream = fdopen(link_fd(query_link), "w");
	if(!stream) {
		return;
	}
	link_nonblocking(query_link, 0);

	current = time(0);
	fprintf(stream, "HTTP/1.1 200 OK\n");
	fprintf(stream, "Date: %s", ctime(&current));
	fprintf(stream, "Server: catalog_server\n");
	fprintf(stream, "Connection: close\n");

	if(sscanf(url, "http://%[^/]%s", hostport, path) == 2) {
		// continue on
	} else {
		strcpy(path, url);
	}

	/* load the hash table entries into one big array */

	n = 0;
	nvpair_database_firstkey(table);
	while(nvpair_database_nextkey(table, &hkey, &nv)) {
		array[n] = nv;
		n++;
	}

	/* sort the array by name before displaying */

	qsort(array, n, sizeof(struct nvpair *), compare_nvpair);

	if(!strcmp(path, "/query.text")) {
		fprintf(stream, "Content-type: text/plain\n\n");
		for(i = 0; i < n; i++)
			nvpair_print_text(array[i], stream);
	} else if(!strcmp(path, "/query.json")) {
		fprintf(stream, "Content-type: text/plain\n\n");
		fprintf(stream,"[\n");
		for(i = 0; i < n; i++) {
			nvpair_print_json(array[i], stream);
			fprintf(stream,",\n");
		}
		fprintf(stream,"]\n");
	} else if(!strcmp(path, "/query.oldclassads")) {
		fprintf(stream, "Content-type: text/plain\n\n");
		for(i = 0; i < n; i++)
			nvpair_print_old_classads(array[i], stream);
	} else if(!strcmp(path, "/query.newclassads")) {
		fprintf(stream, "Content-type: text/plain\n\n");
		for(i = 0; i < n; i++)
			nvpair_print_new_classads(array[i], stream);
	} else if(!strcmp(path, "/query.xml")) {
		fprintf(stream, "Content-type: text/xml\n\n");
		fprintf(stream, "<?xml version=\"1.0\" standalone=\"yes\"?>\n");
		fprintf(stream, "<catalog>\n");
		for(i = 0; i < n; i++)
			nvpair_print_xml(array[i], stream);
		fprintf(stream, "</catalog>\n");
	} else if(sscanf(path, "/detail/%s", key) == 1) {
		struct nvpair *nv;
		fprintf(stream, "Content-type: text/html\n\n");
		nv = nvpair_database_lookup(table, key);
		if(nv) {
			const char *name = nvpair_lookup_string(nv, "name");
			if(!name)
				name = "unknown";
			fprintf(stream, "<title>%s catalog server: %s</title>\n", preferred_hostname, name);
			fprintf(stream, "<center>\n");
			fprintf(stream, "<h1>%s catalog server</h1>\n", preferred_hostname);
			fprintf(stream, "<h2>%s</h2>\n", name);
			fprintf(stream, "<p><a href=/>return to catalog view</a><p>\n");
			nvpair_print_html_solo(nv, stream);
			fprintf(stream, "</center>\n");
		} else {
			fprintf(stream, "<title>%s catalog server</title>\n", preferred_hostname);
			fprintf(stream, "<center>\n");
			fprintf(stream, "<h1>%s catalog server</h1>\n", preferred_hostname);
			fprintf(stream, "<h2>Unknown Item!</h2>\n");
			fprintf(stream, "</center>\n");
		}
	} else {
		char avail_line[LINE_MAX];
		char total_line[LINE_MAX];
		INT64_T sum_total = 0;
		INT64_T sum_avail = 0;
		INT64_T sum_devices = 0;

		fprintf(stream, "Content-type: text/html\n\n");
		fprintf(stream, "<title>%s catalog server</title>\n", preferred_hostname);
		fprintf(stream, "<center>\n");
		fprintf(stream, "<h1>%s catalog server</h1>\n", preferred_hostname);
		fprintf(stream, "<a href=/query.text>text</a> - ");
		fprintf(stream, "<a href=/query.html>html</a> - ");
		fprintf(stream, "<a href=/query.xml>xml</a> - ");
		fprintf(stream, "<a href=/query.json>json</a> - ");
		fprintf(stream, "<a href=/query.oldclassads>oldclassads</a> - ");
		fprintf(stream, "<a href=/query.newclassads>newclassads</a>");
		fprintf(stream, "<p>\n");

		for(i = 0; i < n; i++) {
			nv = array[i];
			sum_total += nvpair_lookup_integer(nv, "total");
			sum_avail += nvpair_lookup_integer(nv, "avail");
			sum_devices++;
		}

		string_metric(sum_avail, -1, avail_line);
		string_metric(sum_total, -1, total_line);
		fprintf(stream, "<b>%sB available out of %sB on %d devices</b><p>\n", avail_line, total_line, (int) sum_devices);

		nvpair_print_html_header(stream, html_headers);
		for(i = 0; i < n; i++) {
			nv = array[i];
			make_hash_key(nv, key);
			sprintf(url, "/detail/%s", key);
			nvpair_print_html_with_link(nv, stream, html_headers, "name", url);
		}
		nvpair_print_html_footer(stream, html_headers);
		fprintf(stream, "</center>\n");
	}
	fclose(stream);
}
Ejemplo n.º 7
0
static void handle_updates(struct datagram *update_port)
{
	char data[DATAGRAM_PAYLOAD_MAX * 2];
	char addr[DATAGRAM_ADDRESS_MAX];
	char key[LINE_MAX];
	int port;
	int result;
	struct nvpair *nv;

	while(1) {
		result = datagram_recv(update_port, data, DATAGRAM_PAYLOAD_MAX, addr, &port, 0);
		if(result <= 0)
			return;

		data[result] = 0;

		nv = nvpair_create();
		nvpair_parse(nv, data);

		nvpair_insert_string(nv, "address", addr);
		nvpair_insert_integer(nv, "lastheardfrom", time(0));

		/* If the server reports unbelievable numbers, simply reset them */

		if(max_server_size > 0) {
			INT64_T total = nvpair_lookup_integer(nv, "total");
			INT64_T avail = nvpair_lookup_integer(nv, "avail");

			if(total > max_server_size || avail > max_server_size) {
				nvpair_insert_integer(nv, "total", max_server_size);
				nvpair_insert_integer(nv, "avail", max_server_size);
			}
		}

		/* Do not believe the server's reported name, just resolve it backwards. */

		char name[DOMAIN_NAME_MAX];
		if(domain_name_cache_lookup_reverse(addr, name)) {
			nvpair_insert_string(nv, "name", name);
		} else if (nvpair_lookup_string(nv, "name") == NULL) {
			/* If rDNS is unsuccessful, then we use the name reported if given.
			 * This allows for hostnames that are only valid in the subnet of
			 * the reporting server.  Here we set the "name" field to the IP
			 * Address, addr, because it was not set by the reporting server.
			 */
			nvpair_insert_string(nv, "name", addr);
		}

		make_hash_key(nv, key);

		if(logfile) {
			if(!nvpair_database_lookup(table,key)) {
				nvpair_print_text(nv,logfile);
				fflush(logfile);
			}
		}

		nvpair_database_insert(table, key, nv);

		debug(D_DEBUG, "received udp update from %s", key);
	}
}