static int gthsplit_process_files(Gthsplitinfo *gthsplitinfo,
const GthPlugins *plugins, GtError *err)
{
Store_in_subset_file_data store_in_subset_file_data;
GthInput *inputinfo;
unsigned long i;
int had_err;
gt_error_check(err);
/* initialization */
inputinfo = gth_input_new(plugins->file_preprocessor, plugins->seq_con_new);
store_in_subset_file_data.gthsplitinfo = gthsplitinfo;
store_in_subset_file_data.num_of_subset_files = 100 / gthsplitinfo->range;
store_in_subset_file_data.sa_filter = gthsplitinfo->sa_filter;
store_in_subset_file_data.current_outputfilename = NULL;
store_in_subset_file_data.subset_files =
gt_malloc(sizeof (GtFile*) *
store_in_subset_file_data.num_of_subset_files);
store_in_subset_file_data.subset_filenames =
gt_malloc(sizeof (GtStr*) *
store_in_subset_file_data.num_of_subset_files);
store_in_subset_file_data.subset_file_sa_counter =
gt_malloc(sizeof (unsigned long) *
store_in_subset_file_data.num_of_subset_files);
for (i = 0; i < store_in_subset_file_data.num_of_subset_files; i++) {
store_in_subset_file_data.subset_files[i] = NULL;
store_in_subset_file_data.subset_filenames[i] = NULL;
store_in_subset_file_data.subset_file_sa_counter[i] = 0;
}
store_in_subset_file_data.sa_visitor = gth_xml_inter_sa_visitor_new(inputinfo,
0, NULL);
if (gthsplitinfo->showverbose)
gthsplitinfo->showverbose("process all intermediate output files");
/* split up intermediate files */
had_err = gth_process_intermediate_files(inputinfo,
gthsplitinfo->consensusfiles,
store_in_subset_file,
&store_in_subset_file_data,
gthsplitinfo->showverbose, err);
/* close the split files */
close_output_files(&store_in_subset_file_data);
/* free */
gth_sa_visitor_delete(store_in_subset_file_data.sa_visitor);
gth_input_delete_complete(inputinfo);
gt_free(store_in_subset_file_data.current_outputfilename);
gt_free(store_in_subset_file_data.subset_files);
gt_free(store_in_subset_file_data.subset_filenames);
gt_free(store_in_subset_file_data.subset_file_sa_counter);
return had_err;
}
int textual_dict_gen_t::prepare_output_files(void)
{
close_output_files();
xmlfilemain.reset(g_fopen(xmlfilenamemain.c_str(), "wb"));
if(!xmlfilemain) {
g_critical(open_write_file_err, xmlfilenamemain.c_str());
return EXIT_FAILURE;
}
return EXIT_SUCCESS;
}
void handle_rxparse(PARSER_STATE *ps) {
NODE *tok,*ks,*ke,*v;
char *fnglob;
wordexp_t wet;
int start_field,end_field,val_field,reading_ranges;
int start_is_hex=0,end_is_hex=0,val_is_hex=0;
pcre *parse_compiled,*skip_compiled;
pcre_extra *parse_studied,*skip_studied;
const char *errptr;
int erroffset;
int file_num,line_num,seq_num;
FILE *data_file;
char *buf=NULL;
size_t buf_size=0;
ssize_t line_length;
int i,j;
int *capvec,capvec_length;
if (context_stack->parse_regex==NULL) {
parse_error(ps,"must set parserx before rxparse");
return;
}
ps->ignore_semicolon=0;
tok=get_token(ps);
if (tok->type!=nt_string) {
parse_error(ps,"rxparse file pattern isn't a string");
node_delete(tok);
return;
}
fnglob=tok->cp;
tok->cp=NULL;
node_delete(tok);
tok=get_token(ps);
if ((tok->type==nt_keyword) && (strcmp(tok->cp,"x")==0)) {
start_is_hex=1;
node_delete(tok);
tok=get_token(ps);
}
if (tok->type!=nt_int) {
parse_error(ps,"key field number must be integer or x+integer");
free(fnglob);
node_delete(tok);
return;
}
start_field=tok->x;
node_delete(tok);
tok=get_token(ps);
if (tok->type!=nt_keyword) {
parse_error(ps,"rxparse syntax error");
free(fnglob);
node_delete(tok);
return;
} else if (strcmp(tok->cp,"->")==0) {
reading_ranges=0;
end_field=start_field;
} else if (strcmp(tok->cp,"..")==0) {
node_delete(tok);
reading_ranges=1;
tok=get_token(ps);
if ((tok->type==nt_keyword) && (strcmp(tok->cp,"x")==0)) {
end_is_hex=1;
node_delete(tok);
tok=get_token(ps);
}
if (tok->type!=nt_int) {
parse_error(ps,"key field number must be integer or x+integer");
free(fnglob);
node_delete(tok);
return;
}
end_field=tok->x;
node_delete(tok);
tok=get_token(ps);
if ((tok->type!=nt_keyword) || (strcmp(tok->cp,"->")!=0)) {
parse_error(ps,"rxparse syntax error");
free(fnglob);
node_delete(tok);
return;
}
} else {
parse_error(ps,"rxparse syntax error");
free(fnglob);
node_delete(tok);
return;
}
node_delete(tok);
tok=get_token(ps);
if ((tok->type==nt_keyword) && (strcmp(tok->cp,"x")==0)) {
val_is_hex=1;
node_delete(tok);
tok=get_token(ps);
}
if (tok->type!=nt_int) {
parse_error(ps,"value field number must be integer or x+integer");
free(fnglob);
node_delete(tok);
return;
}
val_field=tok->x;
node_delete(tok);
ps->ignore_semicolon=1;
if (wordexp(fnglob,&wet,0)!=0) {
parse_error(ps,"error expanding rxparse file pattern");
free(fnglob);
return;
}
free(fnglob);
parse_compiled=pcre_compile(context_stack->parse_regex,PCRE_UTF8,
&errptr,&erroffset,NULL);
if (parse_compiled==NULL) {
parse_error(ps,"PCRE error in parserx: %s",errptr);
return;
}
parse_studied=pcre_study(parse_compiled,0,&errptr);
if (errptr!=NULL) {
parse_error(ps,"PCRE study error in parserx: %s",errptr);
pcre_free(parse_compiled);
return;
}
capvec_length=10;
pcre_fullinfo(parse_compiled,parse_studied,
PCRE_INFO_CAPTURECOUNT,&capvec_length);
if ((start_field>capvec_length) || (start_field<-1) ||
(end_field>capvec_length) || (end_field<-1) ||
(val_field>capvec_length) || (val_field<-1)) {
parse_error(ps,"parserx field number out of range");
pcre_free(parse_compiled);
if (parse_studied!=NULL)
pcre_free(parse_studied);
return;
}
capvec_length=(capvec_length+1)*3;
capvec=(int *)malloc(sizeof(int)*capvec_length);
if (context_stack->skip_regex!=NULL) {
skip_compiled=pcre_compile(context_stack->skip_regex,PCRE_UTF8,
&errptr,&erroffset,NULL);
if (skip_compiled==NULL) {
parse_error(ps,"PCRE error in skiprx: %s",errptr);
free(capvec);
pcre_free(parse_compiled);
if (parse_studied!=NULL)
pcre_free(parse_studied);
return;
}
skip_studied=pcre_study(skip_compiled,0,&errptr);
if (errptr!=NULL) {
parse_error(ps,"PCRE study error in skiprx: %s",errptr);
free(capvec);
pcre_free(parse_compiled);
if (parse_studied!=NULL)
pcre_free(parse_studied);
return;
}
} else {
skip_compiled=NULL;
skip_studied=NULL;
}
for (file_num=0;file_num<wet.we_wordc;file_num++) {
data_file=fopen(wet.we_wordv[file_num],"r");
if (data_file==NULL) {
parse_error(ps,"can't open rxparse file %s",wet.we_wordv[file_num]);
continue;
}
line_num=0;
seq_num=0;
while ((line_length=getline(&buf,&buf_size,data_file))>0) {
line_num++;
if ((skip_compiled!=NULL) &&
(pcre_exec(skip_compiled,skip_studied,buf,line_length,
0,0,NULL,0)>=0))
continue;
if (pcre_exec(parse_compiled,parse_studied,buf,line_length,
0,0,capvec,capvec_length)>=0) {
ks=extract_captured_field(buf,capvec,start_field,seq_num,
reading_ranges||start_is_hex,
start_is_hex);
v=extract_captured_field(buf,capvec,val_field,seq_num,
val_is_hex,val_is_hex);
if (reading_ranges) {
ke=extract_captured_field(buf,capvec,end_field,seq_num,
1,end_is_hex);
if (ks->x>ke->x)
fprintf(stderr,"%s line %d: malformed range %d..%d\n",
wet.we_wordv[file_num],line_num,ks->x,ke->x);
while (ks->x<=ke->x) {
if (!raw_add_arrow(&(context_stack->am),ks,v,
context_stack->dupe_priority)) {
if (ks->type==nt_string)
fprintf(stderr,"%s line %d: duplicate key %s\n",
wet.we_wordv[file_num],line_num,ks->cp);
else
fprintf(stderr,"%s line %d: duplicate key %d\n",
wet.we_wordv[file_num],line_num,ks->x);
close_output_files();
exit(1);
}
tok=node_new();
tok->type=nt_int;
tok->x=ks->x+1;
node_delete(ks);
ks=tok;
}
node_delete(ke);
} else if (!raw_add_arrow(&(context_stack->am),ks,v,
context_stack->dupe_priority)) {
if (ks->type==nt_string)
fprintf(stderr,"%s line %d: duplicate key %s\n",
wet.we_wordv[file_num],line_num,ks->cp);
else
fprintf(stderr,"%s line %d: duplicate key %d\n",
wet.we_wordv[file_num],line_num,ks->x);
close_output_files();
exit(1);
}
node_delete(ks);
node_delete(v);
seq_num++;
}
}
fclose(data_file);
}
if (buf!=NULL)
free(buf);
pcre_free(parse_compiled);
if (parse_studied!=NULL)
pcre_free(parse_studied);
if (skip_compiled!=NULL)
pcre_free(skip_compiled);
if (skip_studied!=NULL)
pcre_free(skip_studied);
free(capvec);
}
void handle_closing_brace(PARSER_STATE *ps) {
CONTEXT *ctx;
int i,j;
void (*pref_gen)(struct _CONTEXT *);
SUBMAP *sm,*psm;
if (context_stack==NULL) {
parse_error(ps,"unexpected closing brace");
close_output_files();
exit(1);
}
/* only write out the map if we aren't a parent */
if (context_stack->leaves==0) {
/* first, attempt whatever the user asked for */
if (context_stack->generator!=NULL)
context_stack->generator(context_stack);
/* then look for a generator that actually likes this kind of data */
if (context_stack->generator==NULL) {
j=0;
pref_gen=NULL;
if ((i=prefer_wide_cascade(context_stack))>j) {
j=i;
pref_gen=gen_wide_cascade;
}
if ((i=prefer_cascade(context_stack))>j) {
j=i;
pref_gen=gen_cascade;
}
if ((i=prefer_basic_array(context_stack))>j) {
j=i;
pref_gen=gen_basic_array;
}
if (pref_gen!=NULL)
pref_gen(context_stack);
}
/* finally, just try for any generator */
if (context_stack->generator==NULL)
gen_wide_cascade(context_stack);
if (context_stack->generator==NULL)
gen_cascade(context_stack);
if (context_stack->generator==NULL)
gen_basic_array(context_stack);
if (context_stack->generator==NULL) {
if (context_stack->am.num_arrows==0)
parse_error(ps,"no arrows in map (%s)",context_stack->id);
else
parse_error(ps,"no code generator for these data types (%s)",
context_stack->id);
close_output_files();
exit(1);
}
}
arrow_map_delete(&(context_stack->am));
free(context_stack->id);
if (context_stack->skip_regex!=NULL)
free(context_stack->skip_regex);
if (context_stack->parse_regex!=NULL)
free(context_stack->parse_regex);
if (context_stack->c_file!=NULL)
free(context_stack->c_file);
if (context_stack->h_file!=NULL)
free(context_stack->h_file);
if (context_stack->key_c_type!=NULL)
free(context_stack->key_c_type);
if (context_stack->value_c_type!=NULL)
free(context_stack->value_c_type);
if (context_stack->parent==NULL)
psm=NULL;
else
psm=context_stack->parent->submaps;
while (context_stack->submaps!=psm) {
sm=context_stack->submaps;
context_stack->submaps=sm->next;
free(sm->name);
arrow_map_delete(&(sm->am));
free(sm);
}
if (context_stack->default_value!=NULL)
node_delete(context_stack->default_value);
ctx=context_stack->parent;
free(context_stack);
context_stack=ctx;
ps->ignore_semicolon=1;
}
static int store_in_subset_file(void *data, GthSA *sa,
const char *outputfilename, GtError *err)
{
Store_in_subset_file_data *store_in_subset_file_data =
(Store_in_subset_file_data*) data;
double split_determing_percentage = 0.0;
unsigned long filenum;
char filenamesuffix[4];
int had_err = 0;
gt_error_check(err);
/* filter before we do any further processing */
if (gth_sa_filter_filter_sa(store_in_subset_file_data->sa_filter, sa)) {
/* and free it afterwards */
gth_sa_delete(sa);
/* discard */
return 0;
}
/* check whether we got a new output file to process */
if (!store_in_subset_file_data->current_outputfilename) {
store_in_subset_file_data->current_outputfilename =
gt_cstr_dup(outputfilename);
}
else if (strcmp(store_in_subset_file_data->current_outputfilename,
outputfilename)) {
/* close current output files */
close_output_files(store_in_subset_file_data);
gt_free(store_in_subset_file_data->current_outputfilename);
}
/* determine in which file the current sa needs to be put */
switch (store_in_subset_file_data->gthsplitinfo->splitmode) {
case ALIGNMENTSCORE_SPLIT:
split_determing_percentage = gth_sa_score(sa);
strcpy(filenamesuffix, "scr");
break;
case COVERAGE_SPLIT:
split_determing_percentage = gth_sa_coverage(sa);
strcpy(filenamesuffix, "cov");
break;
default: gt_assert(0);
}
gt_assert(split_determing_percentage >= 0.0);
/* XXX: change into an assertion when coverage problem is fixed */
if (split_determing_percentage > 1.0)
split_determing_percentage = 1.0;
if (split_determing_percentage == 1.0)
filenum = store_in_subset_file_data->num_of_subset_files - 1;
else {
filenum = floor(split_determing_percentage * 100.0 /
store_in_subset_file_data->gthsplitinfo->range);
}
gt_assert(filenum < store_in_subset_file_data->num_of_subset_files);
/* make sure the file exists and is open */
if (!store_in_subset_file_data->subset_files[filenum]) {
gt_assert(store_in_subset_file_data->subset_filenames[filenum] == NULL);
store_in_subset_file_data->subset_filenames[filenum] = gt_str_new();
gt_str_append_cstr_nt(store_in_subset_file_data->subset_filenames[filenum],
outputfilename,
gt_file_basename_length(outputfilename));
gt_str_append_char(store_in_subset_file_data->subset_filenames[filenum],
'.');
gt_str_append_cstr(store_in_subset_file_data->subset_filenames[filenum],
filenamesuffix);
gt_str_append_ulong(store_in_subset_file_data->subset_filenames[filenum],
filenum *
store_in_subset_file_data->gthsplitinfo->range);
gt_str_append_char(store_in_subset_file_data->subset_filenames[filenum],
'-');
gt_str_append_ulong(store_in_subset_file_data->subset_filenames[filenum],
(filenum + 1) *
store_in_subset_file_data->gthsplitinfo->range);
gt_str_append_cstr(store_in_subset_file_data->subset_filenames[filenum],
gt_file_mode_suffix(store_in_subset_file_data
->gthsplitinfo->file_mode));
/* if not disabled by -force, check if file already exists */
if (!store_in_subset_file_data->gthsplitinfo->force) {
store_in_subset_file_data->subset_files[filenum] =
gt_file_open(store_in_subset_file_data->gthsplitinfo->file_mode,
gt_str_get(store_in_subset_file_data
->subset_filenames[filenum]), "r", NULL);
if (store_in_subset_file_data->subset_files[filenum]) {
gt_error_set(err, "file \"%s\" exists already. use option -%s to "
"overwrite", gt_str_get(store_in_subset_file_data
->subset_filenames[filenum]),
GT_FORCE_OPT_CSTR);
had_err = -1;
}
}
if (!had_err) {
/* open split file for writing */
store_in_subset_file_data->subset_files[filenum] =
gt_file_xopen_file_mode(store_in_subset_file_data->gthsplitinfo
->file_mode,
gt_str_get(store_in_subset_file_data
->subset_filenames[filenum]), "w");
/* store XML header in file */
gth_xml_show_leader(true,
store_in_subset_file_data->subset_files[filenum]);
}
}
/* put it there */
if (!had_err) {
gth_xml_inter_sa_visitor_set_outfp(store_in_subset_file_data->sa_visitor,
store_in_subset_file_data
->subset_files[filenum]);
gth_sa_visitor_visit_sa(store_in_subset_file_data->sa_visitor, sa);
}
/* adjust counter */
if (!had_err)
store_in_subset_file_data->subset_file_sa_counter[filenum]++;
/* and free it afterwards */
gth_sa_delete(sa);
return had_err;
}
void textual_dict_gen_t::clear(void)
{
close_output_files();
xmlfilenamemain.clear();
parsed_norm_dict = NULL;
}
static void store_line(date_time_t *ptime,
double lat,
double lon,
double alt,
char *callsign,
int ispd,
double tdry,
double lwjw,
double dewpt,
int left_burn,
int right_burn,
int flare_state,
int snowmax_state)
{
static int first_call = TRUE;
char arch_path[MAX_PATH_LEN];
/*
* on first call, init Archive_path
*/
if (first_call) {
strcpy(Archive_path, "");
first_call = FALSE;
}
/*
* compute file path
*/
sprintf(arch_path, "%s%s%.4d%.2d%.2d",
Glob->params.archive_dir_path,
PATH_DELIM,
ptime->year, ptime->month, ptime->day);
if (Archive_file != NULL) {
if (strcmp(arch_path, Archive_path)) {
/*
* names differ, so close the opened files
*/
close_output_files();
}
} /* if (Archive_file != NULL) */
/*
* if output file is not open, open it and store path
* for future use
*/
if (Archive_file == NULL) {
/*
* open archive file for appending
*/
if ((Archive_file = fopen(arch_path, "a")) == NULL) {
fprintf(stderr, "ERROR - %s:process_file\n", Glob->prog_name);
fprintf(stderr, "Cannot open archive file for appending\n");
perror(arch_path);
return;
}
/*
* open fresh realtime file for writing
*/
if ((Realtime_file = fopen(Glob->params.realtime_file_path, "w"))
== NULL) {
fprintf(stderr, "ERROR - %s:process_file\n", Glob->prog_name);
fprintf(stderr, "Cannot open realtime file for writing\n");
perror(Glob->params.realtime_file_path);
return;
}
strcpy(Archive_path, arch_path);
}
/*
* write to file
*/
fprintf(Archive_file, "%8ld %9.4f %9.4f %6.3f %5s %4d "
"%6.2f %5.2f %6.2f %d %d %d %d\n",
ptime->unix_time, lat, lon,
alt, callsign, ispd,
tdry, lwjw, dewpt,
left_burn, right_burn,
flare_state, snowmax_state);
fflush(Archive_file);
fprintf(Realtime_file, "%8ld %9.4f %9.4f %6.3f %5s %4d "
"%6.2f %6.3f %6.2f %d %d %d %d\n",
ptime->unix_time, lat, lon,
alt, callsign, ispd,
tdry, lwjw, dewpt,
left_burn, right_burn,
flare_state, snowmax_state);
fflush(Realtime_file);
return;
}
int dna_ode_run (int argc, char **argv, config_t *simconfig) {
int retcode;
simparameters *simparams;
simcontext *simcon;
/*
* INITIALIZATION
*/
/* check that seed was given */
if (argc < 4) {
print_usage_run (argv [0]);
exit (EXIT_FAILURE);
}
/* memory allocation for the simulation parameters, statistics, and context */
simparams = (simparameters *) malloc (sizeof (simparameters));
simcon = (simcontext *) malloc (sizeof (simcontext));
/* get RNG seed */
simparams->seed = atoi (argv [3]);
/* initialization of ODE */
dInitODE ();
/*
* INITIALIZE SIMULATION PARAMETERS, RANDOM NUMBER GENERATION, AND CONTEXT
*/
init_simulation_parameters (simparams, simconfig);
my_init_RNG (simparams->seed);
init_simulation_context (simcon, simparams);
/*
* NOW WE SWITCH TO THE SYSTEM CHOSEN
*/
switch (simparams->sysid) {
case SDNA :
retcode = sdna_main (simcon);
break;
case FDNA :
retcode = fdna_main (simcon);
break;
case CDNA :
retcode = cdna_main (simcon);
break;
default :
err_message ("Invalid system ID\n");
return 1;
}
/* final stuff to terminal */
finish_output (simcon);
/* close output files */
close_output_files (simcon);
/* free memory */
free_memory (simcon);
/* close ODE */
dCloseODE();
return retcode;
}
static int do_marxcat (char *odir, Data_Dir_Type **dirs, unsigned int ndirs)
{
Marx_Dump_File_Type new_mdfts [NUM_DATA_FILE_NAMES];
char *filenames [NUM_DATA_FILE_NAMES];
unsigned int i, j;
unsigned int num_files;
memset ((char *) new_mdfts, 0, sizeof (new_mdfts));
memset ((char *) filenames, 0, sizeof (filenames));
num_files = 0;
for (i = 0; i < NUM_DATA_FILE_NAMES; i++)
{
char *file;
Marx_Dump_File_Type *mdftp;
if (NULL == (mdftp = dirs[0]->dft_list[i].mdft))
continue;
new_mdfts[num_files].type = mdftp->type;
strcpy (new_mdfts[num_files].colname, mdftp->colname);
file = marx_dircat (odir, Data_Files[i]);
if (file == NULL)
{
close_output_files (new_mdfts, filenames);
return -1;
}
marx_message ("Creating %s\n", file);
if (NULL == marx_create_write_dump_file (file, new_mdfts + num_files))
{
close_output_files (new_mdfts, filenames);
return -1;
}
filenames [num_files] = file;
if (num_files != i)
{
/* Squeeze the dat_file_type for each directory also. */
for (j = 0; j < ndirs; j++)
{
dirs[j]->dft_list [num_files] = dirs[j]->dft_list [i];
memset ((char *) (dirs [j]->dft_list + i),
0, sizeof (Data_File_Type));
}
}
num_files++;
}
/* Now start the real work */
j = 0;
while (j < ndirs)
{
dirs[j]->fp_time = dirs[j]->dft_list[0].mdft->fp;
if (1 == JDMread_float32 (&(dirs[j]->current_time), 1, dirs[j]->fp_time))
{
dirs[j]->num_read = 1;
j++;
continue;
}
if (-1 == check_read_error (dirs[j]))
{
close_output_files (new_mdfts, filenames);
return -1;
}
free_data_dir_type (dirs[j]);
dirs [j] = NULL;
ndirs = compress_dir_list (dirs, ndirs);
}
while (ndirs)
{
unsigned int min_j = 0;
double tmin = dirs [0]->current_time;
for (j = 1; j < ndirs; j++)
{
if (dirs [j]->current_time < tmin)
{
min_j = j;
tmin = dirs [j]->current_time;
}
}
if (-1 == write_row_of_data (dirs[min_j], new_mdfts, filenames, num_files))
{
close_output_files (new_mdfts, filenames);
return -1;
}
if (1 == JDMread_float32 (&(dirs[min_j]->current_time), 1, dirs[min_j]->fp_time))
{
dirs[min_j]->num_read++;
continue;
}
if (-1 == check_read_error (dirs[min_j]))
{
close_output_files (new_mdfts, filenames);
return -1;
}
free_data_dir_type (dirs[min_j]);
dirs [min_j] = NULL;
ndirs = compress_dir_list (dirs, ndirs);
}
return close_output_files (new_mdfts, filenames);
}