static int compat_checkpoint_read( struct jx_database *db, const char *filename )
{
FILE * file = fopen(filename,"r");
if(!file) return 0;
while(1) {
struct nvpair *nv = nvpair_create();
if(nvpair_parse_stream(nv,file)) {
const char *key = nvpair_lookup_string(nv,"key");
if(key) {
nvpair_delete(hash_table_remove(db->table,key));
struct jx *j = nvpair_to_jx(nv);
hash_table_insert(db->table,key,j);
}
nvpair_delete(nv);
} else {
nvpair_delete(nv);
break;
}
}
fclose(file);
return 1;
}
static int compat_checkpoint_read( struct deltadb *db, const char *filename )
{
FILE * file = fopen(filename,"r");
if(!file) return 0;
while(1) {
struct nvpair *nv = nvpair_create();
if(nvpair_parse_stream(nv,file)) {
const char *key = nvpair_lookup_string(nv,"key");
if(key) {
nvpair_delete(hash_table_remove(db->table,key));
struct jx *j = nvpair_to_jx(nv);
/* skip objects that don't match the filter */
if(deltadb_boolean_expr(db->filter_expr,j)) {
hash_table_insert(db->table,key,j);
} else {
jx_delete(j);
}
}
nvpair_delete(nv);
} else {
nvpair_delete(nv);
break;
}
}
fclose(file);
return 1;
}
static int log_replay( struct nvpair_database *db, const char *filename, time_t snapshot)
{
FILE *file = fopen(filename,"r");
if(!file) return 0;
time_t current = 0;
struct nvpair *nv;
char line[NVPAIR_LINE_MAX];
char key[NVPAIR_LINE_MAX];
char name[NVPAIR_LINE_MAX];
char value[NVPAIR_LINE_MAX];
char oper;
while(fgets(line,sizeof(line),file)) {
int n = sscanf(line,"%c %s %s %[^\n]",&oper,key,name,value);
if(n<1) continue;
switch(oper) {
case 'C':
nv = nvpair_create();
nvpair_parse_stream(nv,file);
nvpair_delete(hash_table_remove(db->table,name));
hash_table_insert(db->table,key,nv);
break;
case 'D':
nv = hash_table_remove(db->table,key);
if(nv) nvpair_delete(nv);
break;
case 'U':
nv = hash_table_lookup(db->table,key);
if(nv) nvpair_insert_string(nv,name,value);
break;
case 'R':
nv = hash_table_lookup(db->table,key);
if(nv) nvpair_remove(nv,name);
break;
case 'T':
current = atol(key);
if(current>snapshot) break;
break;
default:
debug(D_NOTICE,"corrupt log data in %s: %s",filename,line);
break;
}
}
fclose(file);
return 1;
}
int get_masters(time_t stoptime)
{
struct catalog_query *cq;
struct nvpair *nv;
int i = 0; //nvpair pointer array iterator
if(!catalog_host) {
catalog_host = strdup(CATALOG_HOST);
catalog_port = CATALOG_PORT;
}
cq = catalog_query_create(catalog_host, catalog_port, stoptime );
if(!cq) {
fprintf(stderr, "failed to query catalog server %s:%d: %s \n",catalog_host,catalog_port,strerror(errno));
exit(EXIT_FAILURE);
return 0;
}
while((nv = catalog_query_read(cq,stoptime))) {
resize_catalog(i);
if(strcmp(nvpair_lookup_string(nv, "type"), CATALOG_TYPE_WORK_QUEUE_MASTER) == 0) {
global_catalog[i] = nv; //make the global catalog point to this memory that nv references
i++; //only increment i when a master nvpair is found
}else{
nvpair_delete(nv); //free the memory so something valid can take its place
}
}
resize_catalog(i);
global_catalog[i] = NULL;
catalog_query_delete(cq);
return 1;
}
static void remove_expired_records()
{
struct nvpair *nv;
char *key;
time_t current = time(0);
// Only clean every clean_interval seconds.
if((current-last_clean_time)<clean_interval) return;
// After restarting, all records will have appear to be stale.
// Run for a minimum of lifetime seconds before cleaning anything up.
if((current-starttime)<lifetime ) return;
nvpair_database_firstkey(table);
while(nvpair_database_nextkey(table, &key, &nv)) {
time_t lastheardfrom = nvpair_lookup_integer(nv,"lastheardfrom");
int this_lifetime = nvpair_lookup_integer(nv,"lifetime");
if(this_lifetime>0) {
this_lifetime = MIN(lifetime,this_lifetime);
} else {
this_lifetime = lifetime;
}
if( (current-lastheardfrom) > this_lifetime ) {
nv = nvpair_database_remove(table,key);
if(nv) nvpair_delete(nv);
}
}
last_clean_time = current;
}
static int checkpoint_read( struct deltadb *db )
{
FILE * file = stdin;
if(!file) return 0;
char firstline[NVPAIR_LINE_MAX];
fgets(firstline, sizeof(firstline), file);
printf("%s",firstline);
while(1) {
int keep = 0;
struct nvpair *nv = nvpair_create();
int num_pairs = nvpair_parse_stream(nv,file);
if(num_pairs>0) {
const char *key = nvpair_lookup_string(nv,"key");
if(key) {
keep = 1;
struct argument *arg = db->args;
while (arg!=NULL){
const char *var = nvpair_lookup_string(nv,arg->param);
if ( var!=NULL && keep_object(arg,var)==0 ){
keep = 0;
break;
}
arg = arg->next;
}
nvpair_delete(hash_table_remove(db->table,key));
if (keep==1){
nvpair_print_text(nv,stdout);
hash_table_insert(db->table,key,nv);
}
} else debug(D_NOTICE,"no key in object create.");
} else if (num_pairs == -1) {
return 1;
} else {
break;
}
if (!keep)
nvpair_delete(nv);
}
return 1;
}
static void makeflow_node_export_variables( struct dag *d, struct dag_node *n )
{
struct nvpair *nv = dag_node_env_create(d,n);
if(nv){
nvpair_export(nv);
nvpair_delete(nv);
}
}
void delete_projects_list(struct list *l)
{
if(l) {
struct nvpair *nv;
while((nv=list_pop_head(l))) {
nvpair_delete(nv);
}
list_delete(l);
}
}
void global_catalog_cleanup()
{
int i = 0;
while(global_catalog[i] != NULL)
{
nvpair_delete(global_catalog[i]);
i++;
}
free(global_catalog);
}
struct list *get_masters_from_catalog(const char *catalog_host, int catalog_port, struct list *regex_list)
{
struct catalog_query *q;
struct nvpair *nv;
struct list *ml;
struct work_queue_master *m;
char *regex;
time_t timeout = 60, stoptime;
stoptime = time(0) + timeout;
q = catalog_query_create(catalog_host, catalog_port, stoptime);
if(!q) {
fprintf(stderr, "Failed to query catalog server at %s:%d\n", catalog_host, catalog_port);
return NULL;
}
ml = list_create();
if(!ml)
return NULL;
while((nv = catalog_query_read(q, stoptime))) {
if(strcmp(nvpair_lookup_string(nv, "type"), CATALOG_TYPE_WORK_QUEUE_MASTER) == 0) {
m = parse_work_queue_master_nvpair(nv);
if(m) {
if(regex_list) {
// Matched preferred masters
int match_found = 0;
list_first_item(regex_list);
while((regex = (char *)list_next_item(regex_list))) {
if(whole_string_match_regex(m->proj, regex)) {
debug(D_WQ, "Master matched: %s -> %s\n", regex, m->proj);
list_push_head(ml, m);
match_found = 1;
break;
}
}
if(match_found == 0) {
free_work_queue_master(m);
}
} else {
list_push_head(ml, m);
}
} else {
fprintf(stderr, "Failed to parse a work queue master record!\n");
}
}
nvpair_delete(nv);
}
// Must delete the query otherwise it would occupy 1 tcp connection forever!
catalog_query_delete(q);
return ml;
}
void nvpair_database_insert( struct nvpair_database *db, const char *key, struct nvpair *nv )
{
struct nvpair *old = hash_table_remove(db->table,key);
hash_table_insert(db->table,key,nv);
if(db->logdir) {
if(old) {
log_updates(db,key,old,nv);
nvpair_delete(old);
} else {
log_create(db,key,nv);
}
}
log_flush(db);
}
int do_direct_query( const char *master_host, int master_port, time_t stoptime )
{
static struct nvpair_header *query_headers[4] = { queue_headers, task_headers, worker_headers, master_resource_headers };
static const char * query_strings[4] = {"queue","task","worker", "master_resource"};
struct nvpair_header *query_header = query_headers[query_mode];
const char * query_string = query_strings[query_mode];
struct link *l;
struct nvpair *nv;
char master_addr[LINK_ADDRESS_MAX];
if(!domain_name_cache_lookup(master_host,master_addr)) {
fprintf(stderr,"couldn't find address of %s\n",master_host);
return 1;
}
l = link_connect(master_addr,master_port,stoptime);
if(!l) {
fprintf(stderr,"couldn't connect to %s port %d: %s\n",master_host,master_port,strerror(errno));
return 1;
}
link_putfstring(l,"%s_status\n",stoptime,query_string);
if(format_mode==FORMAT_TABLE) {
nvpair_print_table_header(stdout, query_header);
}
while((nv = link_nvpair_read(l,stoptime))) {
if(format_mode == FORMAT_TABLE) {
nvpair_print_table(nv, stdout, query_header);
} else {
nvpair_print_text(nv, stdout);
}
nvpair_delete(nv);
}
if(format_mode == FORMAT_TABLE) {
nvpair_print_table_footer(stdout, query_header);
}
return EXIT_SUCCESS;
}
int get_pool_decisions_from_catalog(const char *catalog_host, int catalog_port, const char *proj, struct list *decisions) {
struct catalog_query *q;
struct nvpair *nv;
time_t timeout = 60, stoptime;
stoptime = time(0) + timeout;
if(!decisions) {
fprintf(stderr, "No list to store pool decisions.\n");
return 0;
}
q = catalog_query_create(catalog_host, catalog_port, stoptime);
if(!q) {
fprintf(stderr, "Failed to query catalog server at %s:%d\n", catalog_host, catalog_port);
return 0;
}
// multiple pools
while((nv = catalog_query_read(q, stoptime))) {
if(strcmp(nvpair_lookup_string(nv, "type"), CATALOG_TYPE_WORK_QUEUE_POOL) == 0) {
struct work_queue_pool *p;
p = parse_work_queue_pool_nvpair(nv);
debug(D_WQ, "pool %s's decision: %s\n", p->name, p->decision);
int x = workers_by_item(p->decision, proj);
if(x >= 0) {
struct pool_info *pi;
pi = (struct pool_info *)xxmalloc(sizeof(*pi));
strncpy(pi->name, p->name, WORK_QUEUE_POOL_NAME_MAX);
pi->count = x;
list_push_tail(decisions, pi);
}
free(p->decision);
free(p);
}
nvpair_delete(nv);
}
// Must delete the query otherwise it would occupy 1 tcp connection forever!
catalog_query_delete(q);
return 1;
}
struct nvpair *catalog_query_read(struct catalog_query *q, time_t stoptime)
{
struct nvpair *nv = nvpair_create();
char line[65536];
int lines = 0;
while(link_readline(q->link, line, sizeof(line), stoptime)) {
string_chomp(line);
if(!line[0])
break;
nvpair_parse(nv, line);
lines++;
}
if(lines) {
return nv;
} else {
nvpair_delete(nv);
return 0;
}
}
int main(int argc, char *argv[])
{
struct catalog_query *cq;
struct nvpair *nv;
work_queue_status_parse_command_line_arguments(argc, argv);
if(optind > argc) {
work_queue_status_show_help("work_queue_status");
exit(EXIT_FAILURE);
}
cq = catalog_query_create(CATALOG_HOST, CATALOG_PORT, time(0) + Work_Queue_Status_Timeout);
if(!cq) {
fprintf(stderr, "couldn't query catalog %s:%d: %s\n", CATALOG_HOST, CATALOG_PORT, strerror(errno));
return 1;
}
if(Work_Queue_Status_Mode == MODE_TABLE)
nvpair_print_table_header(stdout, headers);
while((nv = catalog_query_read(cq, time(0) + Work_Queue_Status_Timeout))) {
if(strcmp(nvpair_lookup_string(nv, "type"), CATALOG_TYPE_WORK_QUEUE_MASTER) == 0) {
if(Work_Queue_Status_Mode == MODE_TABLE)
nvpair_print_table(nv, stdout, headers);
else
nvpair_print_text(nv, stdout);
}
nvpair_delete(nv);
}
if(Work_Queue_Status_Mode == MODE_TABLE)
nvpair_print_table_footer(stdout, headers);
return EXIT_SUCCESS;
}
static int log_play( struct deltadb *db )
{
FILE *stream = stdin;
time_t current = 0;
struct nvpair *nv;
int line_number = 0;
char line[NVPAIR_LINE_MAX];
char key[NVPAIR_LINE_MAX];
char name[NVPAIR_LINE_MAX];
char value[NVPAIR_LINE_MAX];
char oper;
int notime = 1;
while(fgets(line,sizeof(line),stream)) {
line_number += 1;
if (line[0]=='\n') return 0;
int n = sscanf(line,"%c %s %s %[^\n]",&oper,key,name,value);
if(n<1) continue;
int i,include;
switch(oper) {
case 'C':
nv = nvpair_create();
int num_pairs = nvpair_parse_stream_limited(nv,stream,db->attr_list,db->attr_len);
if (num_pairs>0) {
if (notime) {
printf("T %lld\n",(long long)current);
notime = 0;
}
printf("C %s\n",key);
nvpair_print_text(nv,stdout);
}
nvpair_delete(nv);
break;
case 'D':
if (notime) {
printf("T %lld\n",(long long)current);
notime = 0;
}
printf("%s",line);
break;
case 'U':
include = 0;
for(i=0; i<db->attr_len; i++) {
if(strcmp(name,db->attr_list[i])==0) {
include = 1;
break;
}
}
if (include>0) {
if (notime) {
printf("T %lld\n",(long long)current);
notime = 0;
}
printf("%s",line);
}
break;
case 'R':
include = 0;
for(i=0; i<db->attr_len; i++) {
if(strcmp(name,db->attr_list[i])==0) {
include = 1;
break;
}
}
if (include>0) {
if (notime) {
printf("T %lld\n",(long long)current);
notime = 0;
}
printf("%s",line);
}
break;
case 'T':
current = atol(key);
notime = 1;
break;
default:
debug(D_NOTICE,"corrupt log data[%i]: %s",line_number,line);
fflush(stderr);
break;
}
}
return 0;
}
static int log_play( struct deltadb *db )
{
FILE *stream = stdin;
time_t current = 0;
struct nvpair *nv;
int line_number = 0;
struct hash_table *table = db->table;
char line[NVPAIR_LINE_MAX];
char key[NVPAIR_LINE_MAX];
char name[NVPAIR_LINE_MAX];
char value[NVPAIR_LINE_MAX];
char oper;
int notime = 1;
while(fgets(line,sizeof(line),stream)) {
line_number += 1;
int keep = 0;
if (line[0]=='.') return 0;
int n = sscanf(line,"%c %s %s %[^\n]",&oper,key,name,value);
if(n<1) continue;
switch(oper) {
case 'C':
nv = nvpair_create();
int res = nvpair_parse_stream(nv,stream);
if (res>0){
keep = 1;
struct argument *arg = db->args;
while (arg!=NULL){
const char *var = nvpair_lookup_string(nv,arg->param);
if ( var!=NULL && keep_object(arg,var)==0 ){
keep = 0;
break;
}
arg = arg->next;
}
nvpair_delete(hash_table_remove(db->table,key));
if (keep==1){
if (notime){
printf("T %lld\n",(long long)current);
notime = 0;
}
nvpair_print_text(nv,stdout);
hash_table_insert(db->table,key,nv);
}
}
if (!keep)
nvpair_delete(nv);
break;
case 'D':
nv = hash_table_remove(table,key);
if(nv){
if (notime){
printf("T %lld\n",(long long)current);
notime = 0;
}
printf("%s",line);
}//nvpair_delete(nv);
break;
case 'U':
nv = hash_table_lookup(table,key);
if(nv){
if (notime){
printf("T %lld\n",(long long)current);
notime = 0;
}
printf("%s",line);
}//nvpair_insert_string(nv,name,value);
break;
case 'R':
nv = hash_table_lookup(table,key);
if(nv){
if (notime){
printf("T %lld\n",(long long)current);
notime = 0;
}
printf("%s",line);
}//nvpair_remove(nv,name);
break;
case 'T':
current = atol(key);
notime = 1;
break;
default:
debug(D_NOTICE,"corrupt log data[%i]: %s",line_number,line);
fflush(stderr);
break;
}
}
return 1;
}
/*
Replay a given log file into the hash table, up to the given snapshot time.
Return true if the stoptime was reached.
*/
static int log_play( struct deltadb *db, FILE *stream )
{
struct nvpair *nv;
char line[NVPAIR_LINE_MAX];
char key[NVPAIR_LINE_MAX];
char name[NVPAIR_LINE_MAX];
char value[NVPAIR_LINE_MAX];
char oper;
int first_output = 1;
time_t previous_time = 0;
while(fgets(line,sizeof(line),stream)) {
//debug(D_NOTICE,"Processed line: %s",line);
int n = sscanf(line,"%c %s %s %[^\n]",&oper,key,name,value);
if(n<1) continue;
if(line[0]=='\n') break;
switch(oper) {
case 'C':
nv = nvpair_create();
nvpair_parse_stream(nv,stream);
nvpair_delete(hash_table_remove(db->table,name));
hash_table_insert(db->table,key,nv);
break;
case 'D':
nv = hash_table_remove(db->table,key);
if(nv) nvpair_delete(nv);
break;
case 'U':
nv = hash_table_lookup(db->table,key);
if(nv) nvpair_insert_string(nv,name,value);
break;
case 'R':
nv = hash_table_lookup(db->table,key);
if(nv) nvpair_remove(nv,name);
break;
case 'T':
current = atol(key);
break;
default:
debug(D_NOTICE,"corrupt log data: %s",line);
break;
}
if (previous_time==0){
previous_time = current;
} else if(previous_time!=current) {
emit_all_deltadb_reductions(db,previous_time,first_output);
first_output = 0;
previous_time = current;
}
}
emit_all_deltadb_reductions(db,previous_time,first_output);
return 0;
}
static int log_play( struct deltadb *db, FILE *stream )
{
time_t current = 0;
time_t previous_time = 0;
int line_number = 0;
char line[NVPAIR_LINE_MAX];
char key[NVPAIR_LINE_MAX];
char name[NVPAIR_LINE_MAX];
char value[NVPAIR_LINE_MAX];
char oper;
while(fgets(line,sizeof(line),stream)) {
//debug(D_NOTICE,"Processed line: %s",line);
line_number += 1;
if (line[0]=='\n') break;
int n = sscanf(line,"%c %s %s %[^\n]",&oper,key,name,value);
if(n<1) continue;
struct nvpair *nv;
switch(oper) {
case 'C':
nv = nvpair_create();
int num_pairs = nvpair_parse_stream(nv,stream);
if(num_pairs>0) {
nvpair_delete(hash_table_remove(db->table,key));
hash_table_insert(db->table,key,nv);
} else if (num_pairs == -1) {
nvpair_delete(nv);
break;
} else {
nvpair_delete(nv);
}
break;
case 'D':
nv = hash_table_remove(db->table,key);
if(nv) nvpair_delete(nv);
break;
case 'U':
nv = hash_table_lookup(db->table,key);
if(nv) nvpair_insert_string(nv,name,value);
break;
case 'R':
nv = hash_table_lookup(db->table,key);
if(nv) nvpair_remove(nv,name);
break;
case 'T':
previous_time = current;
current = atol(key);
emit_table_values(db,previous_time);
break;
default:
debug(D_NOTICE,"corrupt log data[%i]: %s",line_number,line);
fflush(stderr);
break;
}
}
emit_table_values(db,current);
return 1;
}
static void handle_update( const char *addr, int port, const char *raw_data, int raw_data_length, const char *protocol )
{
char key[LINE_MAX];
unsigned long data_length;
struct jx *j;
// If the packet starts with Control-Z (0x1A), it is compressed,
// so uncompress it to data[]. Otherwise just copy to data[];.
if(raw_data[0]==0x1A) {
data_length = sizeof(data);
int success = uncompress((Bytef*)data,&data_length,(const Bytef*)&raw_data[1],raw_data_length-1);
if(success!=Z_OK) {
debug(D_DEBUG,"warning: %s:%d sent invalid compressed data (ignoring it)\n",addr,port);
return;
}
} else {
memcpy(data,raw_data,raw_data_length);
data_length = raw_data_length;
}
// Make sure the string data is null terminated.
data[data_length] = 0;
// Once uncompressed, if it starts with a bracket,
// then it is JX/JSON, otherwise it is the legacy nvpair format.
if(data[0]=='{') {
j = jx_parse_string(data);
if(!j) {
debug(D_DEBUG,"warning: %s:%d sent invalid JSON data (ignoring it)\n%s\n",addr,port,data);
return;
}
if(!jx_is_constant(j)) {
debug(D_DEBUG,"warning: %s:%d sent non-constant JX data (ignoring it)\n%s\n",addr,port,data);
jx_delete(j);
return;
}
} else {
struct nvpair *nv = nvpair_create();
if(!nv) return;
nvpair_parse(nv, data);
j = nvpair_to_jx(nv);
nvpair_delete(nv);
}
jx_insert_string(j, "address", addr);
jx_insert_integer(j, "lastheardfrom", time(0));
/* If the server reports unbelievable numbers, simply reset them */
if(max_server_size > 0) {
INT64_T total = jx_lookup_integer(j, "total");
INT64_T avail = jx_lookup_integer(j, "avail");
if(total > max_server_size || avail > max_server_size) {
jx_insert_integer(j, "total", max_server_size);
jx_insert_integer(j, "avail", max_server_size);
}
}
/* Do not believe the server's reported name, just resolve it backwards. */
char name[DOMAIN_NAME_MAX];
if(domain_name_cache_lookup_reverse(addr, name)) {
/*
Special case: Prior bug resulted in multiple name
entries in logged data. When removing the name property,
keep looking until all items are removed.
*/
struct jx *jname = jx_string("name");
struct jx *n;
while((n=jx_remove(j,jname))) {
jx_delete(n);
}
jx_delete(jname);
jx_insert_string(j,"name",name);
} else if (jx_lookup_string(j, "name") == NULL) {
/* If rDNS is unsuccessful, then we use the name reported if given.
* This allows for hostnames that are only valid in the subnet of
* the reporting server. Here we set the "name" field to the IP
* Address, addr, because it was not set by the reporting server.
*/
jx_insert_string(j, "name", addr);
}
make_hash_key(j, key);
if(logfile) {
if(!jx_database_lookup(table,key)) {
jx_print_stream(j,logfile);
fprintf(logfile,"\n");
fflush(logfile);
}
}
jx_database_insert(table, key, j);
debug(D_DEBUG, "received %s update from %s",protocol,key);
}
