void chirp_alloc_flush()
{
char *path, *root;
struct alloc_state *s;
if(!alloc_enabled)
return;
debug(D_ALLOC, "flushing allocation states...");
if(!alloc_table)
alloc_table = hash_table_create(0, 0);
hash_table_firstkey(alloc_table);
while(hash_table_nextkey(alloc_table, &path, (void **) &s)) {
alloc_state_save(path, s);
hash_table_remove(alloc_table, path);
}
if(!root_table)
root_table = hash_table_create(0, 0);
hash_table_firstkey(root_table);
while(hash_table_nextkey(root_table, &path, (void **) &root)) {
free(root);
hash_table_remove(root_table, path);
}
last_flush_time = time(0);
}
void categories_initialize(struct hash_table *categories, struct rmsummary *top, const char *summaries_file) {
struct list *summaries = rmsummary_parse_file_multiple(summaries_file);
if(!summaries) {
fatal("Could not read '%s' file: %s\n", strerror(errno));
}
char *name;
struct category *c;
hash_table_firstkey(categories);
while(hash_table_nextkey(categories, &name, (void **) &c)) {
category_clear_histograms(c);
if(c->first_allocation) {
rmsummary_delete(c->first_allocation);
c->first_allocation = rmsummary_create(-1);
}
}
struct rmsummary *s;
list_first_item(summaries);
while((s = list_pop_head(summaries))) {
if(s->category) {
c = category_lookup_or_create(categories, s->category);
category_accumulate_summary(c, s, NULL);
}
rmsummary_delete(s);
}
hash_table_firstkey(categories);
while(hash_table_nextkey(categories, &name, (void **) &c)) {
category_update_first_allocation(c, NULL);
category_clear_histograms(c);
}
}
void makeflow_gc_prepare( struct dag *d )
{
/* Files to be collected:
* ((all_files \minus sink_files)) \union collect_list) \minus preserve_list) \minus source_files
*/
/* Parse GC_*_LIST and record which target files should be
* garbage collected. */
char *collect_list = dag_variable_lookup_global_string("GC_COLLECT_LIST", d);
char *preserve_list = dag_variable_lookup_global_string("GC_PRESERVE_LIST", d);
struct dag_file *f;
char *filename;
/* add all files, but sink_files */
hash_table_firstkey(d->files);
while((hash_table_nextkey(d->files, &filename, (void **) &f)))
if(!dag_file_is_sink(f)) {
set_insert(d->collect_table, f);
}
int i, argc;
char **argv;
/* add collect_list, for sink_files that should be removed */
string_split_quotes(collect_list, &argc, &argv);
for(i = 0; i < argc; i++) {
f = dag_file_lookup_or_create(d, argv[i]);
set_insert(d->collect_table, f);
debug(D_MAKEFLOW_RUN, "Added %s to garbage collection list", f->filename);
}
free(argv);
/* remove files from preserve_list */
string_split_quotes(preserve_list, &argc, &argv);
for(i = 0; i < argc; i++) {
/* Must initialize to non-zero for hash_table functions to work properly. */
f = dag_file_lookup_or_create(d, argv[i]);
set_remove(d->collect_table, f);
debug(D_MAKEFLOW_RUN, "Removed %s from garbage collection list", f->filename);
}
free(argv);
/* remove source_files from collect_table */
hash_table_firstkey(d->files);
while((hash_table_nextkey(d->files, &filename, (void **) &f)))
if(dag_file_is_source(f)) {
set_remove(d->collect_table, f);
debug(D_MAKEFLOW_RUN, "Removed %s from garbage collection list", f->filename);
}
/* Print reference counts of files to be collected */
set_first_element(d->collect_table);
while((f = set_next_element(d->collect_table)))
debug(D_MAKEFLOW_RUN, "Added %s to garbage collection list (%d)", f->filename, f->ref_count);
}
static int server_table_load(time_t stoptime)
{
struct catalog_query *q;
struct jx *j;
char *key;
void *item;
if((last_update + update_interval) > time(0)) {
return 1;
}
if(!server_table) {
server_table = hash_table_create(0, 0);
if(!server_table)
return 0;
}
if(inhibit_catalog_queries) {
debug(D_CHIRP, "catalog queries disabled\n");
return 1;
}
hash_table_firstkey(server_table);
while(hash_table_nextkey(server_table, &key, &item)) {
hash_table_remove(server_table, key);
jx_delete(item);
}
debug(D_CHIRP, "querying catalog at %s:%d", CATALOG_HOST, CATALOG_PORT);
q = catalog_query_create(CATALOG_HOST, CATALOG_PORT, stoptime);
if(!q)
return 0;
while((j = catalog_query_read(q, stoptime))) {
char name[CHIRP_PATH_MAX];
const char *type, *hname;
int port;
type = jx_lookup_string(j, "type");
if(type && !strcmp(type, "chirp")) {
hname = jx_lookup_string(j, "name");
if(hname) {
port = jx_lookup_integer(j, "port");
if(!port)
port = CHIRP_PORT;
sprintf(name, "%s:%d", hname, port);
hash_table_insert(server_table, name, j);
} else {
jx_delete(j);
}
} else {
jx_delete(j);
}
}
catalog_query_delete(q);
last_update = time(0);
return 1;
}
static void makeflow_gc_all( struct dag *d, struct batch_queue *queue, int maxfiles )
{
int collected = 0;
struct dag_file *f;
char *name;
timestamp_t start_time, stop_time;
/* This will walk the table of files to collect and will remove any
* that are below or equal to the threshold. */
start_time = timestamp_get();
hash_table_firstkey(d->files);
while(hash_table_nextkey(d->files, &name, (void **) &f) && collected < maxfiles) {
if(f->state == DAG_FILE_STATE_COMPLETE
&& !dag_file_is_source(f)
&& !set_lookup(d->outputs, f)
&& !set_lookup(d->inputs, f)
&& makeflow_file_clean(d, queue, f, 0)){
collected++;
}
}
stop_time = timestamp_get();
/* Record total amount of files collected to Makeflowlog. */
if(collected > 0) {
makeflow_gc_collected += collected;
makeflow_log_gc_event(d,collected,stop_time-start_time,makeflow_gc_collected);
}
}
INT64_T chirp_global_getdir(const char *host, const char *path, chirp_dir_t callback, void *arg, time_t stoptime)
{
if(is_multi_path(host)) {
char mhost[CHIRP_PATH_MAX];
char mpath[CHIRP_PATH_MAX];
parse_multi_path(path, mhost, mpath);
return chirp_multi_getdir(mhost, mpath, callback, arg, stoptime);
} else if(not_empty(path)) {
return chirp_reli_getdir(host, path, callback, arg, stoptime);
} else if(not_empty(host)) {
return chirp_reli_getdir(host, "/", callback, arg, stoptime);
} else {
if(server_table_load(stoptime)) {
char *key;
void *item;
hash_table_firstkey(server_table);
while(hash_table_nextkey(server_table, &key, &item)) {
callback(key, arg);
}
callback("multi", arg);
return 0;
} else {
errno = ENOENT;
return -1;
}
}
}
static int checkpoint_write( struct jx_database *db, const char *filename )
{
char *key;
struct jx *jobject;
int first = 1;
FILE *file = fopen(filename,"w");
if(!file) return 0;
fprintf(file,"{\n");
hash_table_firstkey(db->table);
while((hash_table_nextkey(db->table,&key,(void**)&jobject))) {
if(!first) {
fprintf(file,",\n");
} else {
first = 0;
}
fprintf(file,"\"%s\":\n",key);
jx_print_stream(jobject,file);
}
fprintf(file,"}\n");
fclose(file);
return 1;
}
void chirp_stats_summary(char *buf, int length)
{
int chunk;
char *addr;
struct chirp_stats *s;
if(!stats_table)
stats_table = hash_table_create(0, 0);
chunk = snprintf(buf, length, "bytes_written %" PRIu64 "\nbytes_read %" PRIu64 "\ntotal_ops %" PRIu64 "\n", total_bytes_written, total_bytes_read, total_ops);
length -= chunk;
buf += chunk;
chunk = snprintf(buf, length, "clients ");
length -= chunk;
buf += chunk;
hash_table_firstkey(stats_table);
while(hash_table_nextkey(stats_table, &addr, (void **) &s)) {
chunk = snprintf(buf, length, "%s,1,1,%" PRIu64 ",%" PRIu64 ",%" PRIu64 "; ", s->addr, s->ops, s->bytes_read, s->bytes_written);
buf += chunk;
length -= chunk;
}
snprintf(buf, length, "\n");
}
void nvpair_print_json(struct nvpair *n, FILE * s)
{
char *key;
void *value;
int i = 0;
int count = hash_table_size(n->table);
fprintf(s, "{\n");
hash_table_firstkey(n->table);
while(hash_table_nextkey(n->table, &key, &value)) {
fprintf(s,"\"%s\":",key);
if(string_is_integer(value)) {
fprintf(s,"%s",(char*)value);
} else {
fprintf(s,"\"%s\"",(char*)value);
}
i++;
if(i<count) fprintf(s,",\n");
}
fprintf(s, "\n}\n");
}
static void display_reduce_exprs( struct deltadb *db, time_t current )
{
struct list_node *n;
/* Reset all reductions. */
for(n=db->reduce_exprs->head;n;n=n->next) {
deltadb_reduction_reset(n->data);
}
/* For each item in the hash table: */
char *key;
struct jx *jobject;
hash_table_firstkey(db->table);
while(hash_table_nextkey(db->table,&key,(void**)&jobject)) {
/* Skip if the where expression doesn't match */
if(!deltadb_boolean_expr(db->where_expr,jobject)) continue;
/* Update each reduction with its value. */
for(n=db->reduce_exprs->head;n;n=n->next) {
struct deltadb_reduction *r = n->data;
struct jx *value = jx_eval(r->expr,jobject);
if(value && !jx_istype(value, JX_ERROR)) {
if(value->type==JX_INTEGER) {
deltadb_reduction_update(n->data,(double)value->u.integer_value);
} else if(value->type==JX_DOUBLE) {
deltadb_reduction_update(n->data,value->u.double_value);
} else {
// treat non-numerics as 1, to facilitate operations like COUNT
deltadb_reduction_update(n->data,1);
}
jx_delete(value);
}
}
}
/* Emit the current time */
if(db->epoch_mode) {
printf("%lld\t",(long long) current);
} else {
char str[32];
strftime(str,sizeof(str),"%F %T",localtime(¤t));
printf("%s\t",str);
}
/* For each reduction, display the final value. */
for(n=db->reduce_exprs->head;n;n=n->next) {
printf("%lf\t",deltadb_reduction_value(n->data));
}
printf("\n");
}
static void log_updates( struct nvpair_database *db, const char *key, struct nvpair *a, struct nvpair *b )
{
log_select(db);
char *name, *avalue, *bvalue;
// For each item in the old nvpair:
// If the new one is different, log an update event.
// If the new one is missing, log a remove event.
hash_table_firstkey(a->table);
while(hash_table_nextkey(a->table,&name,(void**)&avalue)) {
// Do not log these special cases, because they do not carry new information:
if(!strcmp(name,"lastheardfrom")) continue;
if(!strcmp(name,"uptime")) continue;
bvalue = hash_table_lookup(b->table,name);
if(bvalue) {
if(!strcmp(avalue,bvalue)) {
// items match, do nothing.
} else {
log_time(db);
fprintf(db->logfile,"U %s %s %s\n",key,name,bvalue);
}
} else {
log_time(db);
fprintf(db->logfile,"R %s %s\n",key,name);
}
}
// For each item in the new nvpair:
// If it doesn't exist in the old one, log an update event.
hash_table_firstkey(b->table);
while(hash_table_nextkey(b->table,&name,(void**)&bvalue)) {
avalue = hash_table_lookup(a->table,name);
if(!avalue) {
log_time(db);
fprintf(db->logfile,"U %s %s %s\n",key,name,bvalue);
}
}
}
void nvpair_print_old_classads(struct nvpair *n, FILE * s)
{
char *key;
void *value;
hash_table_firstkey(n->table);
while(hash_table_nextkey(n->table, &key, &value)) {
fprintf(s, "%s = \"%s\"\n", key, (char *) value);
}
fprintf(s, "\n");
}
void batch_queue_delete(struct batch_queue *q)
{
if(q) {
char *key;
char *value;
debug(D_BATCH, "deleting queue %p", q);
q->module->free(q);
for (hash_table_firstkey(q->options); hash_table_nextkey(q->options, &key, (void **) &value); free(value))
;
hash_table_delete(q->options);
for (hash_table_firstkey(q->features); hash_table_nextkey(q->features, &key, (void **) &value); free(value))
;
hash_table_delete(q->features);
itable_delete(q->job_table);
itable_delete(q->output_table);
free(q);
}
}
void chirp_audit_delete(struct hash_table *table)
{
char *key;
struct chirp_audit *entry;
hash_table_firstkey(table);
while(hash_table_nextkey(table, &key, (void *) &entry)) {
free(hash_table_remove(table, key));
}
hash_table_delete(table);
}
static void pfs_resolve_cache_flush()
{
char *key, *value;
if(!resolve_cache) return;
hash_table_firstkey(resolve_cache);
while(hash_table_nextkey(resolve_cache,&key,(void**)&value)) {
hash_table_remove(resolve_cache,key);
free(value);
}
}
void nvpair_print_xml(struct nvpair *n, FILE * s)
{
char *key;
void *value;
fprintf(s, "<item>\n");
hash_table_firstkey(n->table);
while(hash_table_nextkey(n->table, &key, &value)) {
fprintf(s, "<%s>%s</%s>\n", key, (char *) value, key);
}
fprintf(s, "</item>\n\n");
}
void chirp_stats_cleanup()
{
char *addr;
struct chirp_stats *s;
if(!stats_table)
stats_table = hash_table_create(0, 0);
hash_table_firstkey(stats_table);
while(hash_table_nextkey(stats_table, &addr, (void **) &s)) {
hash_table_remove(stats_table, addr);
free(s);
}
}
void nvpair_delete(struct nvpair *n)
{
char *key;
void *value;
if(!n) return;
hash_table_firstkey(n->table);
while(hash_table_nextkey(n->table, &key, &value)) {
hash_table_remove(n->table, key);
free(value);
}
hash_table_delete(n->table);
free(n);
}
void emit_all_deltadb_reductions( struct deltadb *db, time_t current, int first_output )
{
int i;
struct nvpair *nv;
char *key;
const char *value;
/* Reset all deltadb_reduction state. */
for(i=0;i<db->ndeltadb_reductions;i++) {
deltadb_reduction_reset(db->deltadb_reductions[i]);
}
/* After each event, iterate over all objects... */
hash_table_firstkey(db->table);
while(hash_table_nextkey(db->table,&key,(void**)&nv)) {
/* Update all deltadb_reductions for that object. */
for(i=0;i<db->ndeltadb_reductions;i++) {
struct deltadb_reduction *r = db->deltadb_reductions[i];
value = nvpair_lookup_string(nv,r->attr);
if(value) deltadb_reduction_update(r,value);
}
}
if(first_output) {
/* The first time we do this, make it a checkpoint record. */
printf("T %ld\n",(long)current);
printf("C 0\n");
for(i=0;i<db->ndeltadb_reductions;i++) {
struct deltadb_reduction *r = db->deltadb_reductions[i];
deltadb_reduction_print(r);
}
printf("\n");
//printf(".Checkpoint End.\n");
first_output = 0;
} else {
/* After that, make it an update record. */
printf("T %ld\n",(long)current);
for(i=0;i<db->ndeltadb_reductions;i++) {
struct deltadb_reduction *r = db->deltadb_reductions[i];
printf("U 0 ");
deltadb_reduction_print(r);
}
}
}
int nvpair_print(struct nvpair *n, char *text, int length)
{
char *key;
void *value;
int actual;
int total = 0;
hash_table_firstkey(n->table);
while(hash_table_nextkey(n->table, &key, &value) && length > 0) {
actual = snprintf(text, length, "%s %s\n", key, (char *) value);
total += actual;
text += actual;
length -= actual;
}
return total;
}
/* Returns the list of dag_file's which are not the target of any
* node */
struct list *dag_input_files(struct dag *d)
{
struct dag_file *f;
char *filename;
struct list *il;
il = list_create(0);
hash_table_firstkey(d->file_table);
while((hash_table_nextkey(d->file_table, &filename, (void **) &f)))
if(!f->target_of) {
debug(D_DEBUG, "Found independent input file: %s", f->filename);
list_push_tail(il, f);
}
return il;
}
void emit_table_values( struct deltadb *db, time_t current)
{
char *key;
struct nvpair *nv;
int i;
hash_table_firstkey(db->table);
while(hash_table_nextkey(db->table, &key, (void **) &nv)) {
printf("%ld\t",current);
for(i=0;i<db->nfields;i++) {
const char *value = nvpair_lookup_string(nv,db->fields[i]);
if(!value) value = "null";
printf("%s\t",value);
}
printf("\n");
}
}
int hash_cache_nextkey(struct hash_cache *cache, char **key, void **item)
{
struct entry *e;
time_t current = time(0);
while(hash_table_nextkey(cache->table, key, (void **) &e)) {
if(e->expires < current) {
hash_cache_remove(cache, *key);
continue;
} else {
*item = e->value;
return 1;
}
}
return 0;
}
static int checkpoint_write( struct nvpair_database *db, const char *filename )
{
char *key;
struct nvpair *nv;
FILE *file = fopen(filename,"w");
if(!file) return 0;
hash_table_firstkey(db->table);
while((hash_table_nextkey(db->table,&key,(void**)&nv))) {
fprintf(file,"key %s\n",key);
nvpair_print_text(nv,file);
}
fclose(file);
return 1;
}
void chirp_stats_summary(buffer_t *B)
{
char *addr;
struct chirp_stats *s;
if(!stats_table)
stats_table = hash_table_create(0, 0);
buffer_putfstring(B, "bytes_written %" PRIu64 "\n", total_bytes_written);
buffer_putfstring(B, "bytes_read %" PRIu64 "\n", total_bytes_read);
buffer_putfstring(B, "total_ops %" PRIu64 "\n", total_ops);
buffer_putliteral(B, "clients ");
hash_table_firstkey(stats_table);
while(hash_table_nextkey(stats_table, &addr, (void **) &s)) {
buffer_putfstring(B, "%s,1,1,%" PRIu64 ",%" PRIu64 ",%" PRIu64 "; ", s->addr, s->ops, s->bytes_read, s->bytes_written);
}
buffer_putliteral(B, "\n");
}
void dag_compile_ancestors(struct dag *d)
{
struct dag_node *n, *m;
struct dag_file *f;
char *name;
hash_table_firstkey(d->file_table);
while(hash_table_nextkey(d->file_table, &name, (void **) &f)) {
m = f->target_of;
if(!m)
continue;
list_first_item(f->needed_by);
while((n = list_next_item(f->needed_by))) {
debug(D_DEBUG, "rule %d ancestor of %d\n", m->nodeid, n->nodeid);
set_insert(m->descendants, n);
set_insert(n->ancestors, m);
}
}
}
int nvpair_print_alloc(struct nvpair *n, char **text)
{
size_t needed;
char *key;
void *value;
buffer_t b;
buffer_init(&b);
buffer_abortonfailure(&b, 0);
hash_table_firstkey(n->table);
while(hash_table_nextkey(n->table, &key, &value)) {
buffer_printf(&b, "%s %s\n", key, (char *) value);
}
*text = xxstrdup(buffer_tostring(&b, &needed));
buffer_free(&b);
return needed;
}
void makeflow_clean(struct dag *d, struct batch_queue *queue, makeflow_clean_depth clean_depth)
{
struct dag_file *f;
char *name;
hash_table_firstkey(d->files);
while(hash_table_nextkey(d->files, &name, (void **) &f)) {
if(dag_file_is_source(f))
continue;
int silent = 1;
if(dag_file_should_exist(f))
silent = 0;
if(clean_depth == MAKEFLOW_CLEAN_ALL){
makeflow_file_clean(d, queue, f, silent);
} else if(set_lookup(d->outputs, f) && (clean_depth == MAKEFLOW_CLEAN_OUTPUTS)) {
makeflow_file_clean(d, queue, f, silent);
} else if(!set_lookup(d->outputs, f) && (clean_depth == MAKEFLOW_CLEAN_INTERMEDIATES)){
makeflow_file_clean(d, queue, f, silent);
}
}
struct dag_node *n;
for(n = d->nodes; n; n = n->next) {
/* If the node is a Makeflow job, then we should recursively call the *
* clean operation on it. */
if(n->nested_job) {
char *command = xxmalloc(sizeof(char) * (strlen(n->command) + 4));
sprintf(command, "%s -c", n->command);
/* XXX this should use the batch job interface for consistency */
makeflow_node_export_variables(d, n);
system(command);
free(command);
}
}
}
void nvpair_print_html_solo(struct nvpair *n, FILE * stream)
{
char *key;
void *value;
fprintf(stream, "<table bgcolor=%s>\n", COLOR_TWO);
fprintf(stream, "<tr bgcolor=%s>\n", COLOR_ONE);
color_counter = 0;
hash_table_firstkey(n->table);
while(hash_table_nextkey(n->table, &key, &value)) {
fprintf(stream, "<tr bgcolor=%s>\n", color_counter % 2 ? COLOR_ONE : COLOR_TWO);
color_counter++;
fprintf(stream, "<td align=left><b>%s</b>\n", key);
if(!strcmp(key, "url")) {
fprintf(stream, "<td align=left><a href=%s>%s</a>\n", (char *) value, (char *) value);
} else {
fprintf(stream, "<td align=left>%s\n", (char *) value);
}
}
fprintf(stream, "</table>\n");
}
static void display_output_exprs( struct deltadb *db, time_t current )
{
/* For each item in the table... */
char *key;
struct jx *jobject;
hash_table_firstkey(db->table);
while(hash_table_nextkey(db->table,&key,(void**)&jobject)) {
/* Skip if the where expression doesn't match */
if(!deltadb_boolean_expr(db->where_expr,jobject)) continue;
/* Emit the current time */
if(db->epoch_mode) {
printf("%lld\t",(long long) current);
} else {
char str[32];
strftime(str,sizeof(str),"%F %T",localtime(¤t));
printf("%s\t",str);
}
/* For each output expression, compute the value and print. */
struct list_node *n;
for(n=db->output_exprs->head;n;n=n->next) {
struct jx *jvalue = jx_eval(n->data,jobject);
jx_print_stream(jvalue,stdout);
printf("\t");
jx_delete(jvalue);
}
printf("\n");
}
}