void categories_initialize(struct hash_table *categories, struct rmsummary *top, const char *summaries_file) {
struct list *summaries = rmsummary_parse_file_multiple(summaries_file);
if(!summaries) {
fatal("Could not read '%s' file: %s\n", strerror(errno));
}
char *name;
struct category *c;
hash_table_firstkey(categories);
while(hash_table_nextkey(categories, &name, (void **) &c)) {
category_clear_histograms(c);
if(c->first_allocation) {
rmsummary_delete(c->first_allocation);
c->first_allocation = rmsummary_create(-1);
}
}
struct rmsummary *s;
list_first_item(summaries);
while((s = list_pop_head(summaries))) {
if(s->category) {
c = category_lookup_or_create(categories, s->category);
category_accumulate_summary(c, s, NULL);
}
rmsummary_delete(s);
}
hash_table_firstkey(categories);
while(hash_table_nextkey(categories, &name, (void **) &c)) {
category_update_first_allocation(c, NULL);
category_clear_histograms(c);
}
}
/** Deletes task struct and frees contained data. */
void batch_task_delete(struct batch_task *t)
{
if (!t)
return;
free(t->command);
struct batch_file *f;
list_first_item(t->input_files);
while((f = list_next_item(t->input_files))){
batch_file_delete(f);
}
list_delete(t->input_files);
list_first_item(t->output_files);
while((f = list_next_item(t->output_files))){
batch_file_delete(f);
}
list_delete(t->output_files);
rmsummary_delete(t->resources);
jx_delete(t->envlist);
batch_job_info_delete(t->info);
free(t);
}
const struct rmsummary *category_dynamic_task_min_resources(struct category *c, struct rmsummary *user, category_allocation_t request) {
static struct rmsummary *internal = NULL;
const struct rmsummary *max = category_dynamic_task_max_resources(c, user, request);
if(internal) {
rmsummary_delete(internal);
}
internal = rmsummary_create(-1);
/* load seen values */
struct rmsummary *seen = c->max_resources_seen;
if(c->allocation_mode != CATEGORY_ALLOCATION_MODE_FIXED) {
internal->cores = seen->cores;
internal->memory = seen->memory;
internal->disk = seen->disk;
}
rmsummary_merge_override(internal, max);
return internal;
}
const struct rmsummary *category_dynamic_task_max_resources(struct category *c, struct rmsummary *user, category_allocation_t request) {
/* we keep an internal label so that the caller does not have to worry
* about memory leaks. */
static struct rmsummary *internal = NULL;
if(internal) {
rmsummary_delete(internal);
}
internal = rmsummary_create(-1);
struct rmsummary *max = c->max_allocation;
struct rmsummary *first = c->first_allocation;
struct rmsummary *seen = c->max_resources_seen;
if(c->steady_state && c->allocation_mode != CATEGORY_ALLOCATION_MODE_FIXED) {
internal->cores = seen->cores;
internal->memory = seen->memory;
internal->disk = seen->disk;
}
/* load max values */
rmsummary_merge_override(internal, max);
if(c->allocation_mode != CATEGORY_ALLOCATION_MODE_FIXED
&& request == CATEGORY_ALLOCATION_FIRST) {
rmsummary_merge_override(internal, first);
}
/* chip in user values */
rmsummary_merge_override(internal, user);
return internal;
}
int category_accumulate_summary(struct category *c, const struct rmsummary *rs, const struct rmsummary *max_worker) {
int update = 0;
const struct rmsummary *max = c->max_allocation;
const struct rmsummary *seen = c->max_resources_seen;
int new_maximum ;
if(rs
&& (max->cores > 0 || rs->cores <= seen->cores)
&& (max->memory > 0 || rs->memory <= seen->memory)
&& (max->disk > 0 || rs->disk <= seen->disk)) {
new_maximum = 0;
} else {
new_maximum = 1;
}
/* a new maximum has been seen, first-allocation is obsolete. */
if(new_maximum) {
rmsummary_delete(c->first_allocation);
c->first_allocation = NULL;
c->completions_since_last_reset = 0;
update = 1;
}
c->steady_state = c->completions_since_last_reset >= first_allocation_every_n_tasks;
rmsummary_merge_max(c->max_resources_seen, rs);
if(rs && (!rs->exit_type || !strcmp(rs->exit_type, "normal"))) {
category_inc_histogram_count(c, cores, rs);
category_inc_histogram_count(c, cpu_time, rs);
category_inc_histogram_count(c, wall_time, rs);
category_inc_histogram_count(c, virtual_memory, rs);
category_inc_histogram_count(c, memory, rs);
category_inc_histogram_count(c, swap_memory, rs);
category_inc_histogram_count(c, bytes_read, rs);
category_inc_histogram_count(c, bytes_written, rs);
category_inc_histogram_count(c, bytes_sent, rs);
category_inc_histogram_count(c, bytes_received, rs);
category_inc_histogram_count(c, bandwidth, rs);
category_inc_histogram_count(c, total_files, rs);
category_inc_histogram_count(c, disk, rs);
category_inc_histogram_count(c, max_concurrent_processes, rs);
category_inc_histogram_count(c, total_processes,rs);
c->completions_since_last_reset++;
if(new_maximum || c->completions_since_last_reset % first_allocation_every_n_tasks == 0) {
update |= category_update_first_allocation(c, max_worker);
}
/* a task completed using a new maximum, so we consider that the new steady_state. */
if(new_maximum) {
c->steady_state = 1;
}
c->total_tasks++;
}
return update;
}
void rmDsummary_print(FILE *output, struct rmDsummary *so) {
struct rmsummary *s = rmsummary_create(-1);
s->command = xxstrdup(so->command);
if(so->category)
{
s->category = xxstrdup(so->category);
}
else if(so->command)
{
s->category = xxstrdup(so->command);
}
else
{
s->category = xxstrdup(DEFAULT_CATEGORY);
s->command = xxstrdup(DEFAULT_CATEGORY);
}
if(so->task_id) {
s->task_id = xxstrdup(so->task_id);
}
s->start = so->start;
s->end = so->end;
s->wall_time = so->wall_time;
to_internal(so, s, start, "us");
to_internal(so, s, end, "us");
to_internal(so, s, wall_time, "s");
to_internal(so, s, cpu_time, "s");
to_internal(so, s, cores, "cores");
to_internal(so, s, total_processes, "procs");
to_internal(so, s, max_concurrent_processes,"procs");
to_internal(so, s, memory, "MB");
to_internal(so, s, virtual_memory, "MB");
to_internal(so, s, swap_memory, "MB");
to_internal(so, s, bytes_read, "MB");
to_internal(so, s, bytes_written, "MB");
to_internal(so, s, bytes_received, "MB");
to_internal(so, s, bytes_sent, "MB");
to_internal(so, s, bandwidth, "Mbps");
to_internal(so, s, total_files, "files");
to_internal(so, s, disk, "MB");
rmsummary_print(output, s, /* pprint */ 1, /* extra fields */ 0);
rmsummary_delete(s);
return;
}
void category_specify_first_allocation_guess(struct category *c, const struct rmsummary *s) {
/* assume user knows what they are doing. */
c->steady_state = 1;
rmsummary_merge_max(c->max_resources_seen, s);
rmsummary_delete(c->first_allocation);
c->first_allocation = rmsummary_create(-1);
rmsummary_merge_max(c->first_allocation, s);
}
void category_delete(struct hash_table *categories, const char *name) {
struct category *c = hash_table_lookup(categories, name);
if(!c)
return;
hash_table_remove(categories, name);
if(c->name)
free(c->name);
if(c->wq_stats)
free(c->wq_stats);
category_delete_histograms(c);
rmsummary_delete(c->max_allocation);
rmsummary_delete(c->first_allocation);
rmsummary_delete(c->autolabel_resource);
rmsummary_delete(c->max_resources_seen);
free(c);
}
static void batch_queue_wq_option_update (struct batch_queue *q, const char *what, const char *value)
{
if(strcmp(what, "password") == 0) {
if(value)
work_queue_specify_password(q->data, value);
} else if(strcmp(what, "master-mode") == 0) {
if(strcmp(value, "catalog") == 0)
work_queue_specify_master_mode(q->data, WORK_QUEUE_MASTER_MODE_CATALOG);
else if(strcmp(value, "standalone") == 0)
work_queue_specify_master_mode(q->data, WORK_QUEUE_MASTER_MODE_STANDALONE);
} else if(strcmp(what, "name") == 0) {
if(value)
work_queue_specify_name(q->data, value);
} else if(strcmp(what, "priority") == 0) {
if(value)
work_queue_specify_priority(q->data, atoi(value));
else
work_queue_specify_priority(q->data, 0);
} else if(strcmp(what, "fast-abort") == 0) {
if(value)
work_queue_activate_fast_abort(q->data, atof(value));
} else if(strcmp(what, "estimate-capacity") == 0) {
work_queue_specify_estimate_capacity_on(q->data, string_istrue(value));
} else if(strcmp(what, "keepalive-interval") == 0) {
if(value)
work_queue_specify_keepalive_interval(q->data, atoi(value));
else
work_queue_specify_keepalive_interval(q->data, WORK_QUEUE_DEFAULT_KEEPALIVE_INTERVAL);
} else if(strcmp(what, "keepalive-timeout") == 0) {
if(value)
work_queue_specify_keepalive_timeout(q->data, atoi(value));
else
work_queue_specify_keepalive_timeout(q->data, WORK_QUEUE_DEFAULT_KEEPALIVE_TIMEOUT);
} else if(strcmp(what, "master-preferred-connection") == 0) {
if(value)
work_queue_master_preferred_connection(q->data, value);
else
work_queue_master_preferred_connection(q->data, "by_ip");
} else if(strcmp(what, "category-limits") == 0) {
struct rmsummary *s = rmsummary_parse_string(value);
if(s) {
work_queue_specify_category_max_resources(q->data, s->category, s);
rmsummary_delete(s);
} else {
debug(D_NOTICE, "Could no parse '%s' as a summary of resorces encoded in JSON\n", value);
}
}
}
struct rmDsummary *parse_summary(FILE *stream, char *filename, struct hash_table *categories)
{
static FILE *last_stream = NULL;
static int summ_id = 1;
if(last_stream != stream)
{
last_stream = stream;
summ_id = 1;
}
else
{
summ_id++;
}
struct rmsummary *so = rmsummary_parse_next(stream);
if(!so)
return NULL;
if(categories) {
struct category *c = category_lookup_or_create(categories, ALL_SUMMARIES_CATEGORY);
category_accumulate_summary(c, so, NULL);
if(so->category) {
c = category_lookup_or_create(categories, so->category);
category_accumulate_summary(c, so, NULL);
}
}
struct rmDsummary *s = rmsummary_to_rmDsummary(so);
s->file = xxstrdup(filename);
if(!s->task_id) {
s->task_id = get_rule_number(filename);
}
rmsummary_delete(so);
return s;
}
/** Sets the resources of batch_task.
Uses rmsummary_copy to create a deep copy of resources.
*/
void batch_task_set_resources(struct batch_task *t, const struct rmsummary *resources)
{
rmsummary_delete(t->resources);
t->resources = rmsummary_copy(resources);
}
void category_specify_max_allocation(struct category *c, const struct rmsummary *s) {
rmsummary_delete(c->max_allocation);
c->max_allocation = rmsummary_create(-1);
rmsummary_merge_max(c->max_allocation, s);
}
int category_update_first_allocation(struct category *c, const struct rmsummary *max_worker) {
/* buffer used only for debug output. */
static buffer_t *b = NULL;
if(!b) {
b = malloc(sizeof(buffer_t));
buffer_init(b);
}
if(c->allocation_mode == CATEGORY_ALLOCATION_MODE_FIXED)
return 0;
if(c->total_tasks < 1)
return 0;
struct rmsummary *top = rmsummary_create(-1);
rmsummary_merge_override(top, max_worker);
rmsummary_merge_override(top, c->max_resources_seen);
rmsummary_merge_override(top, c->max_allocation);
if(!c->first_allocation) {
c->first_allocation = rmsummary_create(-1);
}
update_first_allocation_field(c, top, 1, cpu_time);
update_first_allocation_field(c, top, 1, wall_time);
update_first_allocation_field(c, top, c->time_peak_independece, cores);
update_first_allocation_field(c, top, c->time_peak_independece, virtual_memory);
update_first_allocation_field(c, top, c->time_peak_independece, memory);
update_first_allocation_field(c, top, c->time_peak_independece, swap_memory);
update_first_allocation_field(c, top, c->time_peak_independece, bytes_read);
update_first_allocation_field(c, top, c->time_peak_independece, bytes_written);
update_first_allocation_field(c, top, c->time_peak_independece, bytes_received);
update_first_allocation_field(c, top, c->time_peak_independece, bytes_sent);
update_first_allocation_field(c, top, c->time_peak_independece, bandwidth);
update_first_allocation_field(c, top, c->time_peak_independece, total_files);
update_first_allocation_field(c, top, c->time_peak_independece, disk);
update_first_allocation_field(c, top, c->time_peak_independece, max_concurrent_processes);
update_first_allocation_field(c, top, c->time_peak_independece, total_processes);
/* From here on we only print debugging info. */
struct jx *jsum = rmsummary_to_json(c->first_allocation, 1);
if(jsum) {
char *str = jx_print_string(jsum);
debug(D_DEBUG, "Updating first allocation '%s':", c->name);
debug(D_DEBUG, "%s", str);
jx_delete(jsum);
free(str);
}
jsum = rmsummary_to_json(top, 1);
if(jsum) {
char *str = jx_print_string(jsum);
debug(D_DEBUG, "From max resources '%s':", c->name);
debug(D_DEBUG, "%s", str);
jx_delete(jsum);
free(str);
}
rmsummary_delete(top);
return 1;
}
