void chirp_alloc_flush() { char *path, *root; struct alloc_state *s; if(!alloc_enabled) return; debug(D_ALLOC, "flushing allocation states..."); if(!alloc_table) alloc_table = hash_table_create(0, 0); hash_table_firstkey(alloc_table); while(hash_table_nextkey(alloc_table, &path, (void **) &s)) { alloc_state_save(path, s); hash_table_remove(alloc_table, path); } if(!root_table) root_table = hash_table_create(0, 0); hash_table_firstkey(root_table); while(hash_table_nextkey(root_table, &path, (void **) &root)) { free(root); hash_table_remove(root_table, path); } last_flush_time = time(0); }
void categories_initialize(struct hash_table *categories, struct rmsummary *top, const char *summaries_file) { struct list *summaries = rmsummary_parse_file_multiple(summaries_file); if(!summaries) { fatal("Could not read '%s' file: %s\n", strerror(errno)); } char *name; struct category *c; hash_table_firstkey(categories); while(hash_table_nextkey(categories, &name, (void **) &c)) { category_clear_histograms(c); if(c->first_allocation) { rmsummary_delete(c->first_allocation); c->first_allocation = rmsummary_create(-1); } } struct rmsummary *s; list_first_item(summaries); while((s = list_pop_head(summaries))) { if(s->category) { c = category_lookup_or_create(categories, s->category); category_accumulate_summary(c, s, NULL); } rmsummary_delete(s); } hash_table_firstkey(categories); while(hash_table_nextkey(categories, &name, (void **) &c)) { category_update_first_allocation(c, NULL); category_clear_histograms(c); } }
void makeflow_gc_prepare( struct dag *d ) { /* Files to be collected: * ((all_files \minus sink_files)) \union collect_list) \minus preserve_list) \minus source_files */ /* Parse GC_*_LIST and record which target files should be * garbage collected. */ char *collect_list = dag_variable_lookup_global_string("GC_COLLECT_LIST", d); char *preserve_list = dag_variable_lookup_global_string("GC_PRESERVE_LIST", d); struct dag_file *f; char *filename; /* add all files, but sink_files */ hash_table_firstkey(d->files); while((hash_table_nextkey(d->files, &filename, (void **) &f))) if(!dag_file_is_sink(f)) { set_insert(d->collect_table, f); } int i, argc; char **argv; /* add collect_list, for sink_files that should be removed */ string_split_quotes(collect_list, &argc, &argv); for(i = 0; i < argc; i++) { f = dag_file_lookup_or_create(d, argv[i]); set_insert(d->collect_table, f); debug(D_MAKEFLOW_RUN, "Added %s to garbage collection list", f->filename); } free(argv); /* remove files from preserve_list */ string_split_quotes(preserve_list, &argc, &argv); for(i = 0; i < argc; i++) { /* Must initialize to non-zero for hash_table functions to work properly. */ f = dag_file_lookup_or_create(d, argv[i]); set_remove(d->collect_table, f); debug(D_MAKEFLOW_RUN, "Removed %s from garbage collection list", f->filename); } free(argv); /* remove source_files from collect_table */ hash_table_firstkey(d->files); while((hash_table_nextkey(d->files, &filename, (void **) &f))) if(dag_file_is_source(f)) { set_remove(d->collect_table, f); debug(D_MAKEFLOW_RUN, "Removed %s from garbage collection list", f->filename); } /* Print reference counts of files to be collected */ set_first_element(d->collect_table); while((f = set_next_element(d->collect_table))) debug(D_MAKEFLOW_RUN, "Added %s to garbage collection list (%d)", f->filename, f->ref_count); }
static void makeflow_gc_all( struct dag *d, struct batch_queue *queue, int maxfiles ) { int collected = 0; struct dag_file *f; char *name; timestamp_t start_time, stop_time; /* This will walk the table of files to collect and will remove any * that are below or equal to the threshold. */ start_time = timestamp_get(); hash_table_firstkey(d->files); while(hash_table_nextkey(d->files, &name, (void **) &f) && collected < maxfiles) { if(f->state == DAG_FILE_STATE_COMPLETE && !dag_file_is_source(f) && !set_lookup(d->outputs, f) && !set_lookup(d->inputs, f) && makeflow_file_clean(d, queue, f, 0)){ collected++; } } stop_time = timestamp_get(); /* Record total amount of files collected to Makeflowlog. */ if(collected > 0) { makeflow_gc_collected += collected; makeflow_log_gc_event(d,collected,stop_time-start_time,makeflow_gc_collected); } }
INT64_T chirp_global_getdir(const char *host, const char *path, chirp_dir_t callback, void *arg, time_t stoptime) { if(is_multi_path(host)) { char mhost[CHIRP_PATH_MAX]; char mpath[CHIRP_PATH_MAX]; parse_multi_path(path, mhost, mpath); return chirp_multi_getdir(mhost, mpath, callback, arg, stoptime); } else if(not_empty(path)) { return chirp_reli_getdir(host, path, callback, arg, stoptime); } else if(not_empty(host)) { return chirp_reli_getdir(host, "/", callback, arg, stoptime); } else { if(server_table_load(stoptime)) { char *key; void *item; hash_table_firstkey(server_table); while(hash_table_nextkey(server_table, &key, &item)) { callback(key, arg); } callback("multi", arg); return 0; } else { errno = ENOENT; return -1; } } }
static int checkpoint_write( struct jx_database *db, const char *filename ) { char *key; struct jx *jobject; int first = 1; FILE *file = fopen(filename,"w"); if(!file) return 0; fprintf(file,"{\n"); hash_table_firstkey(db->table); while((hash_table_nextkey(db->table,&key,(void**)&jobject))) { if(!first) { fprintf(file,",\n"); } else { first = 0; } fprintf(file,"\"%s\":\n",key); jx_print_stream(jobject,file); } fprintf(file,"}\n"); fclose(file); return 1; }
void chirp_stats_summary(char *buf, int length) { int chunk; char *addr; struct chirp_stats *s; if(!stats_table) stats_table = hash_table_create(0, 0); chunk = snprintf(buf, length, "bytes_written %" PRIu64 "\nbytes_read %" PRIu64 "\ntotal_ops %" PRIu64 "\n", total_bytes_written, total_bytes_read, total_ops); length -= chunk; buf += chunk; chunk = snprintf(buf, length, "clients "); length -= chunk; buf += chunk; hash_table_firstkey(stats_table); while(hash_table_nextkey(stats_table, &addr, (void **) &s)) { chunk = snprintf(buf, length, "%s,1,1,%" PRIu64 ",%" PRIu64 ",%" PRIu64 "; ", s->addr, s->ops, s->bytes_read, s->bytes_written); buf += chunk; length -= chunk; } snprintf(buf, length, "\n"); }
void nvpair_print_json(struct nvpair *n, FILE * s) { char *key; void *value; int i = 0; int count = hash_table_size(n->table); fprintf(s, "{\n"); hash_table_firstkey(n->table); while(hash_table_nextkey(n->table, &key, &value)) { fprintf(s,"\"%s\":",key); if(string_is_integer(value)) { fprintf(s,"%s",(char*)value); } else { fprintf(s,"\"%s\"",(char*)value); } i++; if(i<count) fprintf(s,",\n"); } fprintf(s, "\n}\n"); }
static int server_table_load(time_t stoptime) { struct catalog_query *q; struct jx *j; char *key; void *item; if((last_update + update_interval) > time(0)) { return 1; } if(!server_table) { server_table = hash_table_create(0, 0); if(!server_table) return 0; } if(inhibit_catalog_queries) { debug(D_CHIRP, "catalog queries disabled\n"); return 1; } hash_table_firstkey(server_table); while(hash_table_nextkey(server_table, &key, &item)) { hash_table_remove(server_table, key); jx_delete(item); } debug(D_CHIRP, "querying catalog at %s:%d", CATALOG_HOST, CATALOG_PORT); q = catalog_query_create(CATALOG_HOST, CATALOG_PORT, stoptime); if(!q) return 0; while((j = catalog_query_read(q, stoptime))) { char name[CHIRP_PATH_MAX]; const char *type, *hname; int port; type = jx_lookup_string(j, "type"); if(type && !strcmp(type, "chirp")) { hname = jx_lookup_string(j, "name"); if(hname) { port = jx_lookup_integer(j, "port"); if(!port) port = CHIRP_PORT; sprintf(name, "%s:%d", hname, port); hash_table_insert(server_table, name, j); } else { jx_delete(j); } } else { jx_delete(j); } } catalog_query_delete(q); last_update = time(0); return 1; }
static void display_reduce_exprs( struct deltadb *db, time_t current ) { struct list_node *n; /* Reset all reductions. */ for(n=db->reduce_exprs->head;n;n=n->next) { deltadb_reduction_reset(n->data); } /* For each item in the hash table: */ char *key; struct jx *jobject; hash_table_firstkey(db->table); while(hash_table_nextkey(db->table,&key,(void**)&jobject)) { /* Skip if the where expression doesn't match */ if(!deltadb_boolean_expr(db->where_expr,jobject)) continue; /* Update each reduction with its value. */ for(n=db->reduce_exprs->head;n;n=n->next) { struct deltadb_reduction *r = n->data; struct jx *value = jx_eval(r->expr,jobject); if(value && !jx_istype(value, JX_ERROR)) { if(value->type==JX_INTEGER) { deltadb_reduction_update(n->data,(double)value->u.integer_value); } else if(value->type==JX_DOUBLE) { deltadb_reduction_update(n->data,value->u.double_value); } else { // treat non-numerics as 1, to facilitate operations like COUNT deltadb_reduction_update(n->data,1); } jx_delete(value); } } } /* Emit the current time */ if(db->epoch_mode) { printf("%lld\t",(long long) current); } else { char str[32]; strftime(str,sizeof(str),"%F %T",localtime(¤t)); printf("%s\t",str); } /* For each reduction, display the final value. */ for(n=db->reduce_exprs->head;n;n=n->next) { printf("%lf\t",deltadb_reduction_value(n->data)); } printf("\n"); }
static void log_updates( struct nvpair_database *db, const char *key, struct nvpair *a, struct nvpair *b ) { log_select(db); char *name, *avalue, *bvalue; // For each item in the old nvpair: // If the new one is different, log an update event. // If the new one is missing, log a remove event. hash_table_firstkey(a->table); while(hash_table_nextkey(a->table,&name,(void**)&avalue)) { // Do not log these special cases, because they do not carry new information: if(!strcmp(name,"lastheardfrom")) continue; if(!strcmp(name,"uptime")) continue; bvalue = hash_table_lookup(b->table,name); if(bvalue) { if(!strcmp(avalue,bvalue)) { // items match, do nothing. } else { log_time(db); fprintf(db->logfile,"U %s %s %s\n",key,name,bvalue); } } else { log_time(db); fprintf(db->logfile,"R %s %s\n",key,name); } } // For each item in the new nvpair: // If it doesn't exist in the old one, log an update event. hash_table_firstkey(b->table); while(hash_table_nextkey(b->table,&name,(void**)&bvalue)) { avalue = hash_table_lookup(a->table,name); if(!avalue) { log_time(db); fprintf(db->logfile,"U %s %s %s\n",key,name,bvalue); } } }
void nvpair_print_old_classads(struct nvpair *n, FILE * s) { char *key; void *value; hash_table_firstkey(n->table); while(hash_table_nextkey(n->table, &key, &value)) { fprintf(s, "%s = \"%s\"\n", key, (char *) value); } fprintf(s, "\n"); }
void chirp_audit_delete(struct hash_table *table) { char *key; struct chirp_audit *entry; hash_table_firstkey(table); while(hash_table_nextkey(table, &key, (void *) &entry)) { free(hash_table_remove(table, key)); } hash_table_delete(table); }
void batch_queue_delete(struct batch_queue *q) { if(q) { char *key; char *value; debug(D_BATCH, "deleting queue %p", q); q->module->free(q); for (hash_table_firstkey(q->options); hash_table_nextkey(q->options, &key, (void **) &value); free(value)) ; hash_table_delete(q->options); for (hash_table_firstkey(q->features); hash_table_nextkey(q->features, &key, (void **) &value); free(value)) ; hash_table_delete(q->features); itable_delete(q->job_table); itable_delete(q->output_table); free(q); } }
void nvpair_print_xml(struct nvpair *n, FILE * s) { char *key; void *value; fprintf(s, "<item>\n"); hash_table_firstkey(n->table); while(hash_table_nextkey(n->table, &key, &value)) { fprintf(s, "<%s>%s</%s>\n", key, (char *) value, key); } fprintf(s, "</item>\n\n"); }
static void pfs_resolve_cache_flush() { char *key, *value; if(!resolve_cache) return; hash_table_firstkey(resolve_cache); while(hash_table_nextkey(resolve_cache,&key,(void**)&value)) { hash_table_remove(resolve_cache,key); free(value); } }
void chirp_stats_cleanup() { char *addr; struct chirp_stats *s; if(!stats_table) stats_table = hash_table_create(0, 0); hash_table_firstkey(stats_table); while(hash_table_nextkey(stats_table, &addr, (void **) &s)) { hash_table_remove(stats_table, addr); free(s); } }
void nvpair_delete(struct nvpair *n) { char *key; void *value; if(!n) return; hash_table_firstkey(n->table); while(hash_table_nextkey(n->table, &key, &value)) { hash_table_remove(n->table, key); free(value); } hash_table_delete(n->table); free(n); }
void emit_all_deltadb_reductions( struct deltadb *db, time_t current, int first_output ) { int i; struct nvpair *nv; char *key; const char *value; /* Reset all deltadb_reduction state. */ for(i=0;i<db->ndeltadb_reductions;i++) { deltadb_reduction_reset(db->deltadb_reductions[i]); } /* After each event, iterate over all objects... */ hash_table_firstkey(db->table); while(hash_table_nextkey(db->table,&key,(void**)&nv)) { /* Update all deltadb_reductions for that object. */ for(i=0;i<db->ndeltadb_reductions;i++) { struct deltadb_reduction *r = db->deltadb_reductions[i]; value = nvpair_lookup_string(nv,r->attr); if(value) deltadb_reduction_update(r,value); } } if(first_output) { /* The first time we do this, make it a checkpoint record. */ printf("T %ld\n",(long)current); printf("C 0\n"); for(i=0;i<db->ndeltadb_reductions;i++) { struct deltadb_reduction *r = db->deltadb_reductions[i]; deltadb_reduction_print(r); } printf("\n"); //printf(".Checkpoint End.\n"); first_output = 0; } else { /* After that, make it an update record. */ printf("T %ld\n",(long)current); for(i=0;i<db->ndeltadb_reductions;i++) { struct deltadb_reduction *r = db->deltadb_reductions[i]; printf("U 0 "); deltadb_reduction_print(r); } } }
/* Returns the list of dag_file's which are not the target of any * node */ struct list *dag_input_files(struct dag *d) { struct dag_file *f; char *filename; struct list *il; il = list_create(0); hash_table_firstkey(d->file_table); while((hash_table_nextkey(d->file_table, &filename, (void **) &f))) if(!f->target_of) { debug(D_DEBUG, "Found independent input file: %s", f->filename); list_push_tail(il, f); } return il; }
void emit_table_values( struct deltadb *db, time_t current) { char *key; struct nvpair *nv; int i; hash_table_firstkey(db->table); while(hash_table_nextkey(db->table, &key, (void **) &nv)) { printf("%ld\t",current); for(i=0;i<db->nfields;i++) { const char *value = nvpair_lookup_string(nv,db->fields[i]); if(!value) value = "null"; printf("%s\t",value); } printf("\n"); } }
int nvpair_print(struct nvpair *n, char *text, int length) { char *key; void *value; int actual; int total = 0; hash_table_firstkey(n->table); while(hash_table_nextkey(n->table, &key, &value) && length > 0) { actual = snprintf(text, length, "%s %s\n", key, (char *) value); total += actual; text += actual; length -= actual; } return total; }
static int checkpoint_write( struct nvpair_database *db, const char *filename ) { char *key; struct nvpair *nv; FILE *file = fopen(filename,"w"); if(!file) return 0; hash_table_firstkey(db->table); while((hash_table_nextkey(db->table,&key,(void**)&nv))) { fprintf(file,"key %s\n",key); nvpair_print_text(nv,file); } fclose(file); return 1; }
void chirp_stats_summary(buffer_t *B) { char *addr; struct chirp_stats *s; if(!stats_table) stats_table = hash_table_create(0, 0); buffer_putfstring(B, "bytes_written %" PRIu64 "\n", total_bytes_written); buffer_putfstring(B, "bytes_read %" PRIu64 "\n", total_bytes_read); buffer_putfstring(B, "total_ops %" PRIu64 "\n", total_ops); buffer_putliteral(B, "clients "); hash_table_firstkey(stats_table); while(hash_table_nextkey(stats_table, &addr, (void **) &s)) { buffer_putfstring(B, "%s,1,1,%" PRIu64 ",%" PRIu64 ",%" PRIu64 "; ", s->addr, s->ops, s->bytes_read, s->bytes_written); } buffer_putliteral(B, "\n"); }
void dag_compile_ancestors(struct dag *d) { struct dag_node *n, *m; struct dag_file *f; char *name; hash_table_firstkey(d->file_table); while(hash_table_nextkey(d->file_table, &name, (void **) &f)) { m = f->target_of; if(!m) continue; list_first_item(f->needed_by); while((n = list_next_item(f->needed_by))) { debug(D_DEBUG, "rule %d ancestor of %d\n", m->nodeid, n->nodeid); set_insert(m->descendants, n); set_insert(n->ancestors, m); } } }
int nvpair_print_alloc(struct nvpair *n, char **text) { size_t needed; char *key; void *value; buffer_t b; buffer_init(&b); buffer_abortonfailure(&b, 0); hash_table_firstkey(n->table); while(hash_table_nextkey(n->table, &key, &value)) { buffer_printf(&b, "%s %s\n", key, (char *) value); } *text = xxstrdup(buffer_tostring(&b, &needed)); buffer_free(&b); return needed; }
void makeflow_clean(struct dag *d, struct batch_queue *queue, makeflow_clean_depth clean_depth) { struct dag_file *f; char *name; hash_table_firstkey(d->files); while(hash_table_nextkey(d->files, &name, (void **) &f)) { if(dag_file_is_source(f)) continue; int silent = 1; if(dag_file_should_exist(f)) silent = 0; if(clean_depth == MAKEFLOW_CLEAN_ALL){ makeflow_file_clean(d, queue, f, silent); } else if(set_lookup(d->outputs, f) && (clean_depth == MAKEFLOW_CLEAN_OUTPUTS)) { makeflow_file_clean(d, queue, f, silent); } else if(!set_lookup(d->outputs, f) && (clean_depth == MAKEFLOW_CLEAN_INTERMEDIATES)){ makeflow_file_clean(d, queue, f, silent); } } struct dag_node *n; for(n = d->nodes; n; n = n->next) { /* If the node is a Makeflow job, then we should recursively call the * * clean operation on it. */ if(n->nested_job) { char *command = xxmalloc(sizeof(char) * (strlen(n->command) + 4)); sprintf(command, "%s -c", n->command); /* XXX this should use the batch job interface for consistency */ makeflow_node_export_variables(d, n); system(command); free(command); } } }
void nvpair_print_html_solo(struct nvpair *n, FILE * stream) { char *key; void *value; fprintf(stream, "<table bgcolor=%s>\n", COLOR_TWO); fprintf(stream, "<tr bgcolor=%s>\n", COLOR_ONE); color_counter = 0; hash_table_firstkey(n->table); while(hash_table_nextkey(n->table, &key, &value)) { fprintf(stream, "<tr bgcolor=%s>\n", color_counter % 2 ? COLOR_ONE : COLOR_TWO); color_counter++; fprintf(stream, "<td align=left><b>%s</b>\n", key); if(!strcmp(key, "url")) { fprintf(stream, "<td align=left><a href=%s>%s</a>\n", (char *) value, (char *) value); } else { fprintf(stream, "<td align=left>%s\n", (char *) value); } } fprintf(stream, "</table>\n"); }
static void display_output_exprs( struct deltadb *db, time_t current ) { /* For each item in the table... */ char *key; struct jx *jobject; hash_table_firstkey(db->table); while(hash_table_nextkey(db->table,&key,(void**)&jobject)) { /* Skip if the where expression doesn't match */ if(!deltadb_boolean_expr(db->where_expr,jobject)) continue; /* Emit the current time */ if(db->epoch_mode) { printf("%lld\t",(long long) current); } else { char str[32]; strftime(str,sizeof(str),"%F %T",localtime(¤t)); printf("%s\t",str); } /* For each output expression, compute the value and print. */ struct list_node *n; for(n=db->output_exprs->head;n;n=n->next) { struct jx *jvalue = jx_eval(n->data,jobject); jx_print_stream(jvalue,stdout); printf("\t"); jx_delete(jvalue); } printf("\n"); } }
void jx_database_firstkey( struct jx_database *db ) { hash_table_firstkey(db->table); }