Пример #1
0
void chirp_alloc_flush()
{
	char *path, *root;
	struct alloc_state *s;

	if(!alloc_enabled)
		return;

	debug(D_ALLOC, "flushing allocation states...");

	if(!alloc_table)
		alloc_table = hash_table_create(0, 0);

	hash_table_firstkey(alloc_table);
	while(hash_table_nextkey(alloc_table, &path, (void **) &s)) {
		alloc_state_save(path, s);
		hash_table_remove(alloc_table, path);
	}

	if(!root_table)
		root_table = hash_table_create(0, 0);

	hash_table_firstkey(root_table);
	while(hash_table_nextkey(root_table, &path, (void **) &root)) {
		free(root);
		hash_table_remove(root_table, path);
	}

	last_flush_time = time(0);
}
Пример #2
0
void categories_initialize(struct hash_table *categories, struct rmsummary *top, const char *summaries_file) {
	struct list *summaries = rmsummary_parse_file_multiple(summaries_file);

	if(!summaries) {
		fatal("Could not read '%s' file: %s\n", strerror(errno));
	}

	char *name;
	struct category *c;
	hash_table_firstkey(categories);
	while(hash_table_nextkey(categories, &name, (void **) &c)) {
		category_clear_histograms(c);
		if(c->first_allocation) {
			rmsummary_delete(c->first_allocation);
			c->first_allocation = rmsummary_create(-1);
		}
	}

	struct rmsummary *s;
	list_first_item(summaries);
	while((s = list_pop_head(summaries))) {
		if(s->category) {
			c = category_lookup_or_create(categories, s->category);
			category_accumulate_summary(c, s, NULL);
		}
		rmsummary_delete(s);
	}

	hash_table_firstkey(categories);
	while(hash_table_nextkey(categories, &name, (void **) &c)) {
		category_update_first_allocation(c, NULL);
		category_clear_histograms(c);
	}
}
Пример #3
0
void makeflow_gc_prepare( struct dag *d )
{
	/* Files to be collected:
	 * ((all_files \minus sink_files)) \union collect_list) \minus preserve_list) \minus source_files
	 */

	/* Parse GC_*_LIST and record which target files should be
	 * garbage collected. */
	char *collect_list  = dag_variable_lookup_global_string("GC_COLLECT_LIST", d);
	char *preserve_list = dag_variable_lookup_global_string("GC_PRESERVE_LIST", d);

	struct dag_file *f;
	char *filename;

	/* add all files, but sink_files */
	hash_table_firstkey(d->files);
	while((hash_table_nextkey(d->files, &filename, (void **) &f)))
		if(!dag_file_is_sink(f)) {
			set_insert(d->collect_table, f);
		}

	int i, argc;
	char **argv;

	/* add collect_list, for sink_files that should be removed */
	string_split_quotes(collect_list, &argc, &argv);
	for(i = 0; i < argc; i++) {
		f = dag_file_lookup_or_create(d, argv[i]);
		set_insert(d->collect_table, f);
		debug(D_MAKEFLOW_RUN, "Added %s to garbage collection list", f->filename);
	}
	free(argv);

	/* remove files from preserve_list */
	string_split_quotes(preserve_list, &argc, &argv);
	for(i = 0; i < argc; i++) {
		/* Must initialize to non-zero for hash_table functions to work properly. */
		f = dag_file_lookup_or_create(d, argv[i]);
		set_remove(d->collect_table, f);
		debug(D_MAKEFLOW_RUN, "Removed %s from garbage collection list", f->filename);
	}
	free(argv);

	/* remove source_files from collect_table */
	hash_table_firstkey(d->files);
	while((hash_table_nextkey(d->files, &filename, (void **) &f)))
		if(dag_file_is_source(f)) {
			set_remove(d->collect_table, f);
			debug(D_MAKEFLOW_RUN, "Removed %s from garbage collection list", f->filename);
		}

	/* Print reference counts of files to be collected */
	set_first_element(d->collect_table);
	while((f = set_next_element(d->collect_table)))
		debug(D_MAKEFLOW_RUN, "Added %s to garbage collection list (%d)", f->filename, f->ref_count);
}
Пример #4
0
static int server_table_load(time_t stoptime)
{
	struct catalog_query *q;
	struct jx *j;
	char *key;
	void *item;

	if((last_update + update_interval) > time(0)) {
		return 1;
	}

	if(!server_table) {
		server_table = hash_table_create(0, 0);
		if(!server_table)
			return 0;
	}

	if(inhibit_catalog_queries) {
		debug(D_CHIRP, "catalog queries disabled\n");
		return 1;
	}

	hash_table_firstkey(server_table);
	while(hash_table_nextkey(server_table, &key, &item)) {
		hash_table_remove(server_table, key);
		jx_delete(item);
	}

	debug(D_CHIRP, "querying catalog at %s:%d", CATALOG_HOST, CATALOG_PORT);

	q = catalog_query_create(CATALOG_HOST, CATALOG_PORT, stoptime);
	if(!q)
		return 0;

	while((j = catalog_query_read(q, stoptime))) {
		char name[CHIRP_PATH_MAX];
		const char *type, *hname;
		int port;

		type = jx_lookup_string(j, "type");
		if(type && !strcmp(type, "chirp")) {
			hname = jx_lookup_string(j, "name");
			if(hname) {
				port = jx_lookup_integer(j, "port");
				if(!port)
					port = CHIRP_PORT;
				sprintf(name, "%s:%d", hname, port);
				hash_table_insert(server_table, name, j);
			} else {
				jx_delete(j);
			}
		} else {
			jx_delete(j);
		}
	}
	catalog_query_delete(q);
	last_update = time(0);

	return 1;
}
Пример #5
0
static void makeflow_gc_all( struct dag *d, struct batch_queue *queue, int maxfiles )
{
	int collected = 0;
	struct dag_file *f;
	char *name;

	timestamp_t start_time, stop_time;

	/* This will walk the table of files to collect and will remove any
	 * that are below or equal to the threshold. */
	start_time = timestamp_get();
	hash_table_firstkey(d->files);
	while(hash_table_nextkey(d->files, &name, (void **) &f) && collected < maxfiles) {
		if(f->state == DAG_FILE_STATE_COMPLETE
			&& !dag_file_is_source(f)
			&& !set_lookup(d->outputs, f)
			&& !set_lookup(d->inputs, f)
			&& makeflow_file_clean(d, queue, f, 0)){
			collected++;
		}
	}

	stop_time = timestamp_get();

	/* Record total amount of files collected to Makeflowlog. */
	if(collected > 0) {
		makeflow_gc_collected += collected;
		makeflow_log_gc_event(d,collected,stop_time-start_time,makeflow_gc_collected);
	}
}
Пример #6
0
INT64_T chirp_global_getdir(const char *host, const char *path, chirp_dir_t callback, void *arg, time_t stoptime)
{
	if(is_multi_path(host)) {
		char mhost[CHIRP_PATH_MAX];
		char mpath[CHIRP_PATH_MAX];
		parse_multi_path(path, mhost, mpath);
		return chirp_multi_getdir(mhost, mpath, callback, arg, stoptime);
	} else if(not_empty(path)) {
		return chirp_reli_getdir(host, path, callback, arg, stoptime);
	} else if(not_empty(host)) {
		return chirp_reli_getdir(host, "/", callback, arg, stoptime);
	} else {
		if(server_table_load(stoptime)) {
			char *key;
			void *item;
			hash_table_firstkey(server_table);
			while(hash_table_nextkey(server_table, &key, &item)) {
				callback(key, arg);
			}
			callback("multi", arg);
			return 0;
		} else {
			errno = ENOENT;
			return -1;
		}
	}
}
Пример #7
0
static int checkpoint_write( struct jx_database *db, const char *filename )
{
	char *key;
	struct jx *jobject;
	int first = 1;

	FILE *file = fopen(filename,"w");
	if(!file) return 0;

	fprintf(file,"{\n");

	hash_table_firstkey(db->table);
	while((hash_table_nextkey(db->table,&key,(void**)&jobject))) {
		if(!first) {
			fprintf(file,",\n");
		} else {
			first = 0;
		}
		fprintf(file,"\"%s\":\n",key);
		jx_print_stream(jobject,file);
	}

	fprintf(file,"}\n");

	fclose(file);

	return 1;
}
Пример #8
0
void chirp_stats_summary(char *buf, int length)
{
	int chunk;
	char *addr;
	struct chirp_stats *s;

	if(!stats_table)
		stats_table = hash_table_create(0, 0);

	chunk = snprintf(buf, length, "bytes_written %" PRIu64 "\nbytes_read %" PRIu64 "\ntotal_ops %" PRIu64 "\n", total_bytes_written, total_bytes_read, total_ops);
	length -= chunk;
	buf += chunk;

	chunk = snprintf(buf, length, "clients ");

	length -= chunk;
	buf += chunk;

	hash_table_firstkey(stats_table);
	while(hash_table_nextkey(stats_table, &addr, (void **) &s)) {
		chunk = snprintf(buf, length, "%s,1,1,%" PRIu64 ",%" PRIu64 ",%" PRIu64 "; ", s->addr, s->ops, s->bytes_read, s->bytes_written);
		buf += chunk;
		length -= chunk;
	}

	snprintf(buf, length, "\n");
}
Пример #9
0
void nvpair_print_json(struct nvpair *n, FILE * s)
{
	char *key;
	void *value;

	int i = 0;
	int count = hash_table_size(n->table);

	fprintf(s, "{\n");
	hash_table_firstkey(n->table);
	while(hash_table_nextkey(n->table, &key, &value)) {

		fprintf(s,"\"%s\":",key);

		if(string_is_integer(value)) {
			fprintf(s,"%s",(char*)value);
		} else {
			fprintf(s,"\"%s\"",(char*)value);
		}		

		i++;
		if(i<count) fprintf(s,",\n");
	}
	fprintf(s, "\n}\n");
}
Пример #10
0
static void display_reduce_exprs( struct deltadb *db, time_t current )
{
	struct list_node *n;

	/* Reset all reductions. */
	for(n=db->reduce_exprs->head;n;n=n->next) {
		deltadb_reduction_reset(n->data);
	}

	/* For each item in the hash table: */

	char *key;
	struct jx *jobject;
	hash_table_firstkey(db->table);
	while(hash_table_nextkey(db->table,&key,(void**)&jobject)) {

		/* Skip if the where expression doesn't match */
		if(!deltadb_boolean_expr(db->where_expr,jobject)) continue;

		/* Update each reduction with its value. */
		for(n=db->reduce_exprs->head;n;n=n->next) {
			struct deltadb_reduction *r = n->data;
			struct jx *value = jx_eval(r->expr,jobject);
			if(value && !jx_istype(value, JX_ERROR)) {
				if(value->type==JX_INTEGER) {
					deltadb_reduction_update(n->data,(double)value->u.integer_value);
				} else if(value->type==JX_DOUBLE) {
					deltadb_reduction_update(n->data,value->u.double_value);
				} else {
					// treat non-numerics as 1, to facilitate operations like COUNT
					deltadb_reduction_update(n->data,1);
				}

				jx_delete(value);
			}
		}
	}

	/* Emit the current time */

	if(db->epoch_mode) {
		printf("%lld\t",(long long) current);
	} else {
		char str[32];
		strftime(str,sizeof(str),"%F %T",localtime(&current));
		printf("%s\t",str);
	}

	/* For each reduction, display the final value. */
	for(n=db->reduce_exprs->head;n;n=n->next) {
		printf("%lf\t",deltadb_reduction_value(n->data));
	}

	printf("\n");

}
Пример #11
0
static void log_updates( struct nvpair_database *db, const char *key, struct nvpair *a, struct nvpair *b )
{
	log_select(db);

	char *name, *avalue, *bvalue;

	// For each item in the old nvpair:
	// If the new one is different, log an update event.
	// If the new one is missing, log a remove event.

	hash_table_firstkey(a->table);
	while(hash_table_nextkey(a->table,&name,(void**)&avalue)) {

		// Do not log these special cases, because they do not carry new information:
		if(!strcmp(name,"lastheardfrom")) continue;
		if(!strcmp(name,"uptime")) continue;

		bvalue = hash_table_lookup(b->table,name);
		if(bvalue) {
			if(!strcmp(avalue,bvalue)) {
				// items match, do nothing.
			} else { 
				log_time(db);
				fprintf(db->logfile,"U %s %s %s\n",key,name,bvalue);
			}
		} else {
       			log_time(db);
			fprintf(db->logfile,"R %s %s\n",key,name);
		}
	}

	// For each item in the new nvpair:
	// If it doesn't exist in the old one, log an update event.

	hash_table_firstkey(b->table);
	while(hash_table_nextkey(b->table,&name,(void**)&bvalue)) {
		avalue = hash_table_lookup(a->table,name);
		if(!avalue) {
       			log_time(db);
			fprintf(db->logfile,"U %s %s %s\n",key,name,bvalue);
		}
	}
} 
Пример #12
0
void nvpair_print_old_classads(struct nvpair *n, FILE * s)
{
	char *key;
	void *value;

	hash_table_firstkey(n->table);
	while(hash_table_nextkey(n->table, &key, &value)) {
		fprintf(s, "%s = \"%s\"\n", key, (char *) value);
	}
	fprintf(s, "\n");
}
Пример #13
0
void batch_queue_delete(struct batch_queue *q)
{
    if(q) {
        char *key;
        char *value;

        debug(D_BATCH, "deleting queue %p", q);

        q->module->free(q);

        for (hash_table_firstkey(q->options); hash_table_nextkey(q->options, &key, (void **) &value); free(value))
            ;
        hash_table_delete(q->options);
        for (hash_table_firstkey(q->features); hash_table_nextkey(q->features, &key, (void **) &value); free(value))
            ;
        hash_table_delete(q->features);
        itable_delete(q->job_table);
        itable_delete(q->output_table);
        free(q);
    }
}
Пример #14
0
void chirp_audit_delete(struct hash_table *table)
{
	char *key;
	struct chirp_audit *entry;

	hash_table_firstkey(table);
	while(hash_table_nextkey(table, &key, (void *) &entry)) {
		free(hash_table_remove(table, key));
	}

	hash_table_delete(table);
}
Пример #15
0
static void pfs_resolve_cache_flush()
{
	char *key, *value;

	if(!resolve_cache) return;

	hash_table_firstkey(resolve_cache);
	while(hash_table_nextkey(resolve_cache,&key,(void**)&value)) {
		hash_table_remove(resolve_cache,key);
		free(value);
	}
}
Пример #16
0
void nvpair_print_xml(struct nvpair *n, FILE * s)
{
	char *key;
	void *value;

	fprintf(s, "<item>\n");
	hash_table_firstkey(n->table);
	while(hash_table_nextkey(n->table, &key, &value)) {
		fprintf(s, "<%s>%s</%s>\n", key, (char *) value, key);
	}
	fprintf(s, "</item>\n\n");
}
Пример #17
0
void chirp_stats_cleanup()
{
	char *addr;
	struct chirp_stats *s;

	if(!stats_table)
		stats_table = hash_table_create(0, 0);

	hash_table_firstkey(stats_table);
	while(hash_table_nextkey(stats_table, &addr, (void **) &s)) {
		hash_table_remove(stats_table, addr);
		free(s);
	}
}
Пример #18
0
void nvpair_delete(struct nvpair *n)
{
	char *key;
	void *value;

	if(!n) return;

	hash_table_firstkey(n->table);
	while(hash_table_nextkey(n->table, &key, &value)) {
		hash_table_remove(n->table, key);
		free(value);
	}
	hash_table_delete(n->table);
	free(n);
}
void emit_all_deltadb_reductions( struct deltadb *db, time_t current, int first_output )
{
	int i;
	struct nvpair *nv;
	char *key;
	const char *value;

	/* Reset all deltadb_reduction state. */
	for(i=0;i<db->ndeltadb_reductions;i++) {
		deltadb_reduction_reset(db->deltadb_reductions[i]);
	}

	/* After each event, iterate over all objects... */

	hash_table_firstkey(db->table);
	while(hash_table_nextkey(db->table,&key,(void**)&nv)) {

		/* Update all deltadb_reductions for that object. */
		for(i=0;i<db->ndeltadb_reductions;i++) {
			struct deltadb_reduction *r = db->deltadb_reductions[i];
			value = nvpair_lookup_string(nv,r->attr);
			if(value) deltadb_reduction_update(r,value);
		}
	}

	if(first_output) {
		/* The first time we do this, make it a checkpoint record. */
		printf("T %ld\n",(long)current);
		printf("C 0\n");
		for(i=0;i<db->ndeltadb_reductions;i++) {
			struct deltadb_reduction *r = db->deltadb_reductions[i];
			deltadb_reduction_print(r);
		}
		printf("\n");
		//printf(".Checkpoint End.\n");
		first_output = 0;
	} else {
		/* After that, make it an update record. */
		printf("T %ld\n",(long)current);
		for(i=0;i<db->ndeltadb_reductions;i++) {
			struct deltadb_reduction *r = db->deltadb_reductions[i];
			printf("U 0 ");
			deltadb_reduction_print(r);
		}
	}
}
Пример #20
0
int nvpair_print(struct nvpair *n, char *text, int length)
{
	char *key;
	void *value;

	int actual;
	int total = 0;

	hash_table_firstkey(n->table);
	while(hash_table_nextkey(n->table, &key, &value) && length > 0) {
		actual = snprintf(text, length, "%s %s\n", key, (char *) value);
		total += actual;
		text += actual;
		length -= actual;
	}
	return total;
}
Пример #21
0
/* Returns the list of dag_file's which are not the target of any
 * node */
struct list *dag_input_files(struct dag *d)
{
	struct dag_file *f;
	char *filename;
	struct list *il;

	il = list_create(0);

	hash_table_firstkey(d->file_table);
	while((hash_table_nextkey(d->file_table, &filename, (void **) &f)))
		if(!f->target_of) {
			debug(D_DEBUG, "Found independent input file: %s", f->filename);
			list_push_tail(il, f);
		}

	return il;
}
Пример #22
0
void emit_table_values( struct deltadb *db, time_t current)
{
	char *key;
	struct nvpair *nv;
	int i;

	hash_table_firstkey(db->table);
	while(hash_table_nextkey(db->table, &key, (void **) &nv)) {
		printf("%ld\t",current);
		for(i=0;i<db->nfields;i++) {
			const char *value = nvpair_lookup_string(nv,db->fields[i]);
			if(!value) value = "null";
			printf("%s\t",value);
		}
		printf("\n");
	}
}
Пример #23
0
int hash_cache_nextkey(struct hash_cache *cache, char **key, void **item)
{
	struct entry *e;
	time_t current = time(0);

	while(hash_table_nextkey(cache->table, key, (void **) &e)) {
		if(e->expires < current) {
			hash_cache_remove(cache, *key);
			continue;
		} else {
			*item = e->value;
			return 1;
		}
	}

	return 0;
}
Пример #24
0
static int checkpoint_write( struct nvpair_database *db, const char *filename )
{
	char *key;
	struct nvpair *nv;

	FILE *file = fopen(filename,"w");
	if(!file) return 0;

	hash_table_firstkey(db->table);
	while((hash_table_nextkey(db->table,&key,(void**)&nv))) {
		fprintf(file,"key %s\n",key);
		nvpair_print_text(nv,file);
	}

	fclose(file);

	return 1;
}
Пример #25
0
void chirp_stats_summary(buffer_t *B)
{
	char *addr;
	struct chirp_stats *s;

	if(!stats_table)
		stats_table = hash_table_create(0, 0);

	buffer_putfstring(B, "bytes_written %" PRIu64 "\n", total_bytes_written);
	buffer_putfstring(B, "bytes_read %" PRIu64 "\n", total_bytes_read);
	buffer_putfstring(B, "total_ops %" PRIu64 "\n", total_ops);

	buffer_putliteral(B, "clients ");
	hash_table_firstkey(stats_table);
	while(hash_table_nextkey(stats_table, &addr, (void **) &s)) {
		buffer_putfstring(B, "%s,1,1,%" PRIu64 ",%" PRIu64 ",%" PRIu64 "; ", s->addr, s->ops, s->bytes_read, s->bytes_written);
	}
	buffer_putliteral(B, "\n");
}
Пример #26
0
void dag_compile_ancestors(struct dag *d)
{
	struct dag_node *n, *m;
	struct dag_file *f;
	char *name;

	hash_table_firstkey(d->file_table);
	while(hash_table_nextkey(d->file_table, &name, (void **) &f)) {
		m = f->target_of;

		if(!m)
			continue;

		list_first_item(f->needed_by);
		while((n = list_next_item(f->needed_by))) {
			debug(D_DEBUG, "rule %d ancestor of %d\n", m->nodeid, n->nodeid);
			set_insert(m->descendants, n);
			set_insert(n->ancestors, m);
		}
	}
}
Пример #27
0
int nvpair_print_alloc(struct nvpair *n, char **text)
{
	size_t needed;

	char *key;
	void *value;

	buffer_t b;
	buffer_init(&b);
	buffer_abortonfailure(&b, 0);

	hash_table_firstkey(n->table);
	while(hash_table_nextkey(n->table, &key, &value)) {
		buffer_printf(&b, "%s %s\n", key, (char *) value);
	}

	*text  = xxstrdup(buffer_tostring(&b, &needed));

	buffer_free(&b);

	return needed;
}
Пример #28
0
void makeflow_clean(struct dag *d, struct batch_queue *queue, makeflow_clean_depth clean_depth)
{
	struct dag_file *f;
	char *name;

	hash_table_firstkey(d->files);
	while(hash_table_nextkey(d->files, &name, (void **) &f)) {
		if(dag_file_is_source(f))
			continue;

		int silent = 1;
		if(dag_file_should_exist(f))
			silent = 0;

		if(clean_depth == MAKEFLOW_CLEAN_ALL){
			makeflow_file_clean(d, queue, f, silent);
		} else if(set_lookup(d->outputs, f) && (clean_depth == MAKEFLOW_CLEAN_OUTPUTS)) {
			makeflow_file_clean(d, queue, f, silent);
		} else if(!set_lookup(d->outputs, f) && (clean_depth == MAKEFLOW_CLEAN_INTERMEDIATES)){
			makeflow_file_clean(d, queue, f, silent);
		}
	}

	struct dag_node *n;
	for(n = d->nodes; n; n = n->next) {
		 /* If the node is a Makeflow job, then we should recursively call the *
		  * clean operation on it. */
		 if(n->nested_job) {
			char *command = xxmalloc(sizeof(char) * (strlen(n->command) + 4));
			sprintf(command, "%s -c", n->command);

			/* XXX this should use the batch job interface for consistency */
			makeflow_node_export_variables(d, n);
			system(command);
			free(command);
		}
	}
}
Пример #29
0
void nvpair_print_html_solo(struct nvpair *n, FILE * stream)
{
	char *key;
	void *value;

	fprintf(stream, "<table bgcolor=%s>\n", COLOR_TWO);
	fprintf(stream, "<tr bgcolor=%s>\n", COLOR_ONE);

	color_counter = 0;

	hash_table_firstkey(n->table);
	while(hash_table_nextkey(n->table, &key, &value)) {
		fprintf(stream, "<tr bgcolor=%s>\n", color_counter % 2 ? COLOR_ONE : COLOR_TWO);
		color_counter++;
		fprintf(stream, "<td align=left><b>%s</b>\n", key);
		if(!strcmp(key, "url")) {
			fprintf(stream, "<td align=left><a href=%s>%s</a>\n", (char *) value, (char *) value);
		} else {
			fprintf(stream, "<td align=left>%s\n", (char *) value);
		}
	}
	fprintf(stream, "</table>\n");
}
Пример #30
0
static void display_output_exprs( struct deltadb *db, time_t current )
{
	/* For each item in the table... */

	char *key;
	struct jx *jobject;
	hash_table_firstkey(db->table);
	while(hash_table_nextkey(db->table,&key,(void**)&jobject)) {

		/* Skip if the where expression doesn't match */

		if(!deltadb_boolean_expr(db->where_expr,jobject)) continue;

		/* Emit the current time */

		if(db->epoch_mode) {
			printf("%lld\t",(long long) current);
		} else {
			char str[32];
			strftime(str,sizeof(str),"%F %T",localtime(&current));
			printf("%s\t",str);
		}

		/* For each output expression, compute the value and print. */

		struct list_node *n;
		for(n=db->output_exprs->head;n;n=n->next) {
			struct jx *jvalue = jx_eval(n->data,jobject);
			jx_print_stream(jvalue,stdout);
			printf("\t");
			jx_delete(jvalue);
		}

		printf("\n");
	}
}