int read_config_file(const char *config_file) {
static time_t last_time_modified = 0;
struct stat s;
time_t new_time_modified;
if(stat(config_file, &s) < 0) {
debug(D_NOTICE, "Error reading file %s (%s)", config_file, strerror(errno));
return 0;
}
new_time_modified = s.st_mtime;
if(new_time_modified == last_time_modified) {
return 1;
}
int error_found = 0;
struct jx *J = jx_parse_file(config_file);
if(!J || J->type!=JX_OBJECT) {
debug(D_NOTICE, "Configuration file is not a valid json object: %s\n", config_file);
return 0;
}
assign_new_value(new_workers_max, workers_max, max-workers, int, JX_INTEGER, integer_value)
assign_new_value(new_workers_min, workers_min, min-workers, int, JX_INTEGER, integer_value)
assign_new_value(new_workers_per_cycle, workers_per_cycle, workers-per-cycle, int, JX_INTEGER, integer_value)
assign_new_value(new_worker_timeout, worker_timeout, timeout, int, JX_INTEGER, integer_value)
assign_new_value(new_num_cores_option, resources->cores, cores, int, JX_INTEGER, integer_value)
assign_new_value(new_num_disk_option, resources->disk, disk, int, JX_INTEGER, integer_value)
assign_new_value(new_num_memory_option, resources->memory, memory, int, JX_INTEGER, integer_value)
assign_new_value(new_autosize_option, autosize, autosize, int, JX_INTEGER, integer_value)
assign_new_value(new_factory_timeout_option, factory_timeout, factory-timeout, int, JX_INTEGER, integer_value)
assign_new_value(new_tasks_per_worker, tasks_per_worker, tasks-per-worker, double, JX_INTEGER, integer_value)
assign_new_value(new_project_regex, project_regex, master-name, const char *, JX_STRING, string_value)
assign_new_value(new_foremen_regex, foremen_regex, foremen-name, const char *, JX_STRING, string_value)
assign_new_value(new_extra_worker_args, extra_worker_args, worker-extra-options, const char *, JX_STRING, string_value)
assign_new_value(new_condor_requirements, condor_requirements, condor-requirements, const char *, JX_STRING, string_value)
if(!new_project_regex || strlen(new_project_regex) == 0) {
debug(D_NOTICE, "%s: master name is missing.\n", config_file);
error_found = 1;
}
if(new_workers_min > new_workers_max) {
debug(D_NOTICE, "%s: min workers (%d) is greater than max workers (%d)\n", config_file, new_workers_min, new_workers_max);
error_found = 1;
}
if(new_workers_min < 0) {
debug(D_NOTICE, "%s: min workers (%d) is less than zero.\n", config_file, new_workers_min);
error_found = 1;
}
if(new_workers_max < 0) {
debug(D_NOTICE, "%s: max workers (%d) is less than zero.\n", config_file, new_workers_max);
error_found = 1;
}
if(new_factory_timeout_option < 0) {
debug(D_NOTICE, "%s: factory timeout (%d) is less than zero.\n", config_file, new_factory_timeout_option);
error_found = 1;
}
if(error_found) {
goto end;
}
workers_max = new_workers_max;
workers_min = new_workers_min;
workers_per_cycle = new_workers_per_cycle;
worker_timeout = new_worker_timeout;
tasks_per_worker = new_tasks_per_worker;
autosize = new_autosize_option;
factory_timeout = new_factory_timeout_option;
resources->cores = new_num_cores_option;
resources->memory = new_num_memory_option;
resources->disk = new_num_disk_option;
if(tasks_per_worker < 1) {
tasks_per_worker = resources->cores > 0 ? resources->cores : 1;
}
if(new_project_regex != project_regex) {
free(project_regex);
project_regex = xxstrdup(new_project_regex);
}
if(new_foremen_regex != foremen_regex) {
free(foremen_regex);
foremen_regex = xxstrdup(new_foremen_regex);
}
if(extra_worker_args != new_extra_worker_args) {
free(extra_worker_args);
extra_worker_args = xxstrdup(new_extra_worker_args);
}
if(new_condor_requirements != condor_requirements) {
free(condor_requirements);
condor_requirements = xxstrdup(new_condor_requirements);
}
last_time_modified = new_time_modified;
fprintf(stdout, "Configuration file '%s' has been loaded.", config_file);
fprintf(stdout, "master-name: %s\n", project_regex);
if(foremen_regex) {
fprintf(stdout, "foremen-name: %s\n", foremen_regex);
}
fprintf(stdout, "max-workers: %d\n", workers_max);
fprintf(stdout, "min-workers: %d\n", workers_min);
fprintf(stdout, "workers-per-cycle: %d\n", workers_per_cycle);
fprintf(stdout, "tasks-per-worker: %" PRId64 "\n", tasks_per_worker > 0 ? tasks_per_worker : (resources->cores > 0 ? resources->cores : 1));
fprintf(stdout, "timeout: %d s\n", worker_timeout);
fprintf(stdout, "cores: %" PRId64 "\n", resources->cores > 0 ? resources->cores : 1);
if(condor_requirements) {
fprintf(stdout, "condor-requirements: %s\n", condor_requirements);
}
if(factory_timeout > 0) {
fprintf(stdout, "factory-timeout: %" PRId64 " MB\n", factory_timeout);
}
if(resources->memory > -1) {
fprintf(stdout, "memory: %" PRId64 " MB\n", resources->memory);
}
if(resources->disk > -1) {
fprintf(stdout, "disk: %" PRId64 " MB\n", resources->disk);
}
if(extra_worker_args) {
fprintf(stdout, "worker-extra-options: %s", extra_worker_args);
}
fprintf(stdout, "\n");
end:
jx_delete(J);
return !error_found;
}
static struct internal_amazon_batch_amazon_ids initialize(struct batch_queue* q){
if(initialized){
return initialized_data;
}
char* config_file = hash_table_lookup(q->options,"amazon-batch-config");
if(!config_file) {
fatal("No amazon config file passed!");
}
struct jx* config = jx_parse_file(config_file);
initialized = 1;
instID = time(NULL);
queue_name = string_format("%i_ccl_amazon_batch_queue",instID);//should be unique
done_jobs = itable_create(0);//default size
amazon_job_ids = itable_create(0);
done_files = itable_create(0);
submitted_files = hash_table_create(0,0);
char* amazon_ami = hash_table_lookup(q->options,"amazon-batch-img");
if(amazon_ami == NULL) {
fatal("No image id passed. Please pass file containing ami image id using --amazon-batch-img flag");
}
char* aws_access_key_id = (char*)jx_lookup_string(config, "aws_id");
char* aws_secret_access_key = (char*)jx_lookup_string(config, "aws_key");
char* aws_region = (char*)jx_lookup_string(config,"aws_reg");
bucket_name = (char*)jx_lookup_string(config,"bucket");
vpc = (char*)jx_lookup_string(config,"vpc");
sec_group = (char*)jx_lookup_string(config,"sec_group");
queue_name = (char*)jx_lookup_string(config,"queue_name");
compute_env_name = (char*)jx_lookup_string(config,"env_name");
subnet = (char*)jx_lookup_string(config,"subnet");
if(!aws_access_key_id)
fatal("credentials file %s does not contain aws_id",config_file);
if(!aws_secret_access_key)
fatal("credentials file %s does not contain aws_key",config_file);
if(!aws_region)
fatal("credentials file %s does not contain aws_reg",config_file);
if(!bucket_name)
fatal("credentials file %s does not contain bucket",config_file);
if(!queue_name)
fatal("credentials file %s does not contain queue_name",config_file);
if(!compute_env_name)
fatal("credentials file %s does not contain env_name",config_file);
if(!vpc)
fatal("credentials file %s does not contain vpc",config_file);
if(!subnet)
fatal("credentials file %s does not contain subnet",config_file);
char* env_var = string_format("AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s AWS_DEFAULT_REGION=%s ",aws_access_key_id,aws_secret_access_key,aws_region);
initialized_data.aws_access_key_id = aws_access_key_id;
initialized_data.aws_secret_access_key = aws_secret_access_key;
initialized_data.aws_region=aws_region;
initialized_data.master_env_prefix = env_var;
return initialized_data;
}
