int jx_error_valid(struct jx *j) {
if (!jx_istype(j, JX_OBJECT)) return 0;
if (!jx_istype(jx_lookup(j, "source"), JX_STRING)) return 0;
if (!jx_istype(jx_lookup(j, "name"), JX_STRING)) return 0;
if (!jx_istype(jx_lookup(j, "message"), JX_STRING)) return 0;
return 1;
}
struct rmsummary *json_to_rmsummary(struct jx *j) {
if(!j || !jx_istype(j, JX_OBJECT))
return NULL;
struct rmsummary *s = rmsummary_create(-1);
struct jx_pair *head = j->u.pairs;
while(head) {
if(!jx_istype(head->key, JX_STRING))
continue;
char *key = head->key->u.string_value;
struct jx *value = head->value;
if(jx_istype(value, JX_STRING)) {
rmsummary_assign_char_field(s, key, value->u.string_value);
} else if(jx_istype(value, JX_INTEGER)) {
rmsummary_assign_int_field(s, key, value->u.integer_value);
} else if(jx_istype(value, JX_ARRAY)) {
int64_t number;
int status = json_number_of_array(value, key, &number);
if(status) {
rmsummary_assign_int_field(s, key, number);
}
}
head = head->next;
}
return s;
}
static int json_number_of_array(struct jx *array, char *field, int64_t *number) {
struct jx_item *first = array->u.items;
if(!first)
return 0;
double result;
if(jx_istype(first->value, JX_DOUBLE)) {
result = first->value->u.double_value;
} else if(jx_istype(first->value, JX_INTEGER)) {
result = (double) first->value->u.integer_value;
} else {
return 0;
}
struct jx_item *second = first->next;
if(!second)
return 0;
if(!jx_istype(second->value, JX_STRING))
return 0;
char *unit = second->value->u.string_value;
/* all values in MB, or useconds. Incomplete list! */
if(strcmp(unit, "us") == 0) {
result *= 1;
} else if(strcmp(unit, "s") == 0) {
result *= USECOND;
} else if(strcmp(unit, "MB") == 0) {
result *= 1;
} else if(strcmp(unit, "B") == 0) {
result *= MEGABYTE;
}
/* round for worst case. */
if(strcmp(field, "start") == 0) {
*number = (int64_t) floor(result);
} else {
*number = (int64_t) ceil(result);
}
return 1;
}
void initialize_watch_events(struct rmonitor_file_watch_info *f, struct jx *watch_spec) {
struct jx *events_array = jx_lookup(watch_spec, "events");
if(!events_array) {
fatal("File watch for '%s' did not define any events", f->filename);
}
if(!jx_istype(events_array, JX_ARRAY)) {
fatal("Value for key 'events' in file watch for '%s' is not an array.", f->filename);
}
f->events = list_create(0);
struct jx *event_spec;
int error = 0;
for (void *i = NULL; (event_spec = jx_iterate_array(events_array, &i));) {
struct rmonitor_file_watch_event *e = parse_event(f->filename, event_spec);
if(e) {
if(e->on_pattern) {
// at least one event defines a pattern, thus we need line by
// line processing.
f->event_with_pattern = 1;
}
list_push_tail(f->events, e);
debug(D_RMON, "Added event for file '%s', label '%s', max_count %" PRId64, f->filename, e->label, e->max_count);
} else {
error = 1;
}
}
if(error) {
fatal("Error parsing file watch for '%s'.", f->filename);
}
}
int jx_array_length( struct jx *array )
{
if(!jx_istype(array, JX_ARRAY)) return -1;
int count = 0;
for(struct jx_item *i = array->u.items; i; i = i->next) ++count;
return count;
}
struct jx *catalog_query_read(struct catalog_query *q, time_t stoptime)
{
while(q && q->current) {
int keepit = 1;
if(q->filter_expr) {
struct jx * b;
b = jx_eval(q->filter_expr,q->current->value);
if(jx_istype(b, JX_BOOLEAN) && b->u.boolean_value) {
keepit = 1;
} else {
keepit = 0;
}
jx_delete(b);
} else {
keepit = 1;
}
if(keepit) {
struct jx *result = jx_copy(q->current->value);
q->current = q->current->next;
return result;
}
q->current = q->current->next;
}
return 0;
}
double jx_lookup_double( struct jx *object, const char *key )
{
struct jx *j = jx_lookup(object,key);
if(j && jx_istype(j,JX_DOUBLE)) {
return j->u.double_value;
} else {
return 0;
}
}
int jx_lookup_boolean( struct jx *object, const char *key )
{
struct jx *j = jx_lookup(object,key);
if(j && jx_istype(j,JX_BOOLEAN)) {
return !!j->u.boolean_value;
} else {
return 0;
}
}
jx_int_t jx_lookup_integer( struct jx *object, const char *key )
{
struct jx *j = jx_lookup(object,key);
if(j && jx_istype(j,JX_INTEGER)) {
return j->u.integer_value;
} else {
return 0;
}
}
const char * jx_lookup_string( struct jx *object, const char *key )
{
struct jx *j = jx_lookup(object,key);
if(j && jx_istype(j,JX_STRING)) {
return j->u.string_value;
} else {
return 0;
}
}
int deltadb_boolean_expr( struct jx *expr, struct jx *data )
{
if(!expr) return 1;
struct jx *j = jx_eval(expr,data);
int result = j && !jx_istype(j, JX_ERROR) && j->type==JX_BOOLEAN && j->u.boolean_value;
jx_delete(j);
return result;
}
static void display_reduce_exprs( struct deltadb *db, time_t current )
{
struct list_node *n;
/* Reset all reductions. */
for(n=db->reduce_exprs->head;n;n=n->next) {
deltadb_reduction_reset(n->data);
}
/* For each item in the hash table: */
char *key;
struct jx *jobject;
hash_table_firstkey(db->table);
while(hash_table_nextkey(db->table,&key,(void**)&jobject)) {
/* Skip if the where expression doesn't match */
if(!deltadb_boolean_expr(db->where_expr,jobject)) continue;
/* Update each reduction with its value. */
for(n=db->reduce_exprs->head;n;n=n->next) {
struct deltadb_reduction *r = n->data;
struct jx *value = jx_eval(r->expr,jobject);
if(value && !jx_istype(value, JX_ERROR)) {
if(value->type==JX_INTEGER) {
deltadb_reduction_update(n->data,(double)value->u.integer_value);
} else if(value->type==JX_DOUBLE) {
deltadb_reduction_update(n->data,value->u.double_value);
} else {
// treat non-numerics as 1, to facilitate operations like COUNT
deltadb_reduction_update(n->data,1);
}
jx_delete(value);
}
}
}
/* Emit the current time */
if(db->epoch_mode) {
printf("%lld\t",(long long) current);
} else {
char str[32];
strftime(str,sizeof(str),"%F %T",localtime(¤t));
printf("%s\t",str);
}
/* For each reduction, display the final value. */
for(n=db->reduce_exprs->head;n;n=n->next) {
printf("%lf\t",deltadb_reduction_value(n->data));
}
printf("\n");
}
static void update_blacklisted_workers( struct batch_queue *queue, struct list *masters_list ) {
if(!masters_list || list_size(masters_list) < 1)
return;
buffer_t b;
struct jx *j;
buffer_init(&b);
const char *sep = "";
list_first_item(masters_list);
while((j=list_next_item(masters_list))) {
struct jx *blacklisted = jx_lookup(j,"workers_blacklisted");
if(!blacklisted) {
continue;
}
if(jx_istype(blacklisted, JX_STRING)) {
buffer_printf(&b, "%s%s", sep, blacklisted->u.string_value);
sep = " ";
}
if(jx_istype(blacklisted, JX_ARRAY)) {
struct jx *item;
for (void *i = NULL; (item = jx_iterate_array(blacklisted, &i));) {
if(jx_istype(item, JX_STRING)) {
buffer_printf(&b, "%s%s", sep, item->u.string_value);
sep = " ";
}
}
}
}
if(buffer_pos(&b) > 0) {
batch_queue_set_option(queue, "workers-blacklisted", buffer_tostring(&b));
} else {
batch_queue_set_option(queue, "workers-blacklisted", NULL);
}
buffer_free(&b);
}
struct jx * jx_array_index( struct jx *j, int nth )
{
if (!jx_istype(j, JX_ARRAY)) return NULL;
struct jx_item *item = j->u.items;
for(int i = 0; i < nth; i++) {
if (!item) return NULL;
item = item->next;
}
return item ? item->value : NULL;
}
void jx_export( struct jx *j )
{
if(!j || !jx_istype(j,JX_OBJECT)) return;
struct jx_pair *p;
for(p=j->u.pairs;p;p=p->next) {
if(p->key->type==JX_STRING && p->value->type==JX_STRING) {
setenv(p->key->u.string_value,p->value->u.string_value,1);
}
}
}
struct jx *jx_merge(struct jx *j, ...) {
va_list ap;
va_start (ap, j);
struct jx *result = jx_object(NULL);
for (struct jx *next = j; jx_istype(next, JX_OBJECT); next = va_arg(ap, struct jx *)) {
for (struct jx_pair *p = next->u.pairs; p; p = p->next) {
jx_delete(jx_remove(result, p->key));
jx_insert(result, jx_copy(p->key), jx_copy(p->value));
}
}
va_end(ap);
return result;
}
struct jx * jx_iterate_array(struct jx *j, void **i) {
if (!i) return NULL;
if (*i) {
struct jx_item *next = ((struct jx_item *) *i)->next;
if (next) {
*i = next;
return next->value;
} else {
return NULL;
}
} else {
if (!jx_istype(j, JX_ARRAY)) return NULL;
*i = j->u.items;
return *i ? ((struct jx_item *) *i)->value : NULL;
}
}
jx_int_t parse_int(const char *fname, struct jx *spec, const char *key, jx_int_t default_value) {
int found = 0;
struct jx *val = jx_lookup_guard(spec, key, &found);
int result = default_value;
if(found) {
if(jx_istype(val, JX_INTEGER)) {
result = val->u.integer_value;
} else {
fatal("Value of %s for '%s' is not an integer.", key, fname);
}
}
return result;
}
struct jx *jx_array_concat( struct jx *array, ...) {
struct jx *result = jx_array(NULL);
struct jx_item **tail = &result->u.items;
va_list ap;
va_start(ap, array);
for(struct jx *a = array; a; a = va_arg(ap, struct jx *)) {
if (!jx_istype(a, JX_ARRAY)) {
break;
}
*tail = a->u.items;
while(*tail) tail = &(*tail)->next;
free(a);
}
va_end(ap);
return result;
}
int parse_boolean(const char *fname, struct jx *spec, const char *key, int default_value) {
int found = 0;
struct jx *val = jx_lookup_guard(spec, key, &found);
int result = default_value;
if(found) {
if(jx_istype(val, JX_BOOLEAN)) {
result = jx_istrue(val);
} else {
fatal("Value of %s for '%s' is not boolean.", key, fname);
}
}
return result;
}
struct jx * jx_iterate_values(struct jx *j, void **i) {
if (!i) return NULL;
if (*i) {
struct jx_pair *next = ((struct jx_pair *) *i)->next;
if (next) {
*i = next;
return next->value;
} else {
return NULL;
}
} else {
if (!jx_istype(j, JX_OBJECT)) return NULL;
*i = j->u.pairs;
return *i ? ((struct jx_pair *) *i)->value : NULL;
}
}
static batch_job_id_t batch_job_dryrun_submit (struct batch_queue *q, const char *cmd, const char *extra_input_files, const char *extra_output_files, struct jx *envlist, const struct rmsummary *resources )
{
FILE *log;
char *escaped_cmd;
char *env_assignment;
char *escaped_env_assignment;
struct batch_job_info *info;
batch_job_id_t jobid = random();
fflush(NULL);
debug(D_BATCH, "started dry run of job %" PRIbjid ": %s", jobid, cmd);
if ((log = fopen(q->logfile, "a"))) {
if (!(info = calloc(sizeof(*info), 1))) {
fclose(log);
return -1;
}
info->submitted = time(0);
info->started = time(0);
itable_insert(q->job_table, jobid, info);
if(envlist && jx_istype(envlist, JX_OBJECT) && envlist->u.pairs) {
struct jx_pair *p;
fprintf(log, "env ");
for(p=envlist->u.pairs;p;p=p->next) {
if(p->key->type==JX_STRING && p->value->type==JX_STRING) {
env_assignment = string_format("%s=%s", p->key->u.string_value,p->value->u.string_value);
escaped_env_assignment = string_escape_shell(env_assignment);
fprintf(log, "%s", escaped_env_assignment);
fprintf(log, " ");
free(env_assignment);
free(escaped_env_assignment);
}
}
}
escaped_cmd = string_escape_shell(cmd);
fprintf(log, "sh -c %s\n", escaped_cmd);
free(escaped_cmd);
fclose(log);
return jobid;
} else {
return -1;
}
}
void rmsummary_print(FILE *stream, struct rmsummary *s, struct jx *verbatim_fields)
{
struct jx *jsum = rmsummary_to_json(s, 0);
if(verbatim_fields) {
if(!jx_istype(verbatim_fields, JX_OBJECT)) {
fatal("Vebatim fields are not a json object.");
}
struct jx_pair *head = verbatim_fields->u.pairs;
while(head) {
jx_insert(jsum, jx_copy(head->key), jx_copy(head->value));
head = head->next;
}
}
jx_pretty_print_stream(jsum, stream);
jx_delete(jsum);
}
char *parse_str(const char *fname, struct jx *spec, const char *key, const char *default_value) {
int found = 0;
struct jx *val = jx_lookup_guard(spec, key, &found);
const char *result = default_value;
if(found) {
if(jx_istype(val, JX_STRING)) {
result = val->u.string_value;
} else {
fatal("Value of %s for '%s' is not a string.", key, fname);
}
}
if(result) {
return xxstrdup(result);
} else {
return NULL;
}
}
struct jx *catalog_query_send_query(const char *url, time_t stoptime) {
struct link *link = http_query(url, "GET", stoptime);
if(!link) {
return NULL;
}
struct jx *j = jx_parse_link(link,stoptime);
link_close(link);
if(!j) {
debug(D_DEBUG,"query result failed to parse as JSON");
return NULL;
}
if(!jx_istype(j,JX_ARRAY)) {
debug(D_DEBUG,"query result is not a JSON array");
jx_delete(j);
return NULL;
}
return j;
}
void jx_print_args( struct jx *j, buffer_t *b )
{
if(!jx_istype(j, JX_ARRAY)) return;
jx_item_print(j->u.items, b);
}
