/*
* Allow each registered stream transform to bid on whether
* it wants to handle this stream. Repeat until we've finished
* building the pipeline.
*/
static int
choose_filters(struct archive_read *a)
{
int number_bidders, i, bid, best_bid;
struct archive_read_filter_bidder *bidder, *best_bidder;
struct archive_read_filter *filter;
ssize_t avail;
int r;
for (;;) {
number_bidders = sizeof(a->bidders) / sizeof(a->bidders[0]);
best_bid = 0;
best_bidder = NULL;
bidder = a->bidders;
for (i = 0; i < number_bidders; i++, bidder++) {
if (bidder->bid != NULL) {
bid = (bidder->bid)(bidder, a->filter);
if (bid > best_bid) {
best_bid = bid;
best_bidder = bidder;
}
}
}
/* If no bidder, we're done. */
if (best_bidder == NULL) {
/* Verify the filter by asking it for some data. */
__archive_read_filter_ahead(a->filter, 1, &avail);
if (avail < 0) {
close_filters(a);
free_filters(a);
return (ARCHIVE_FATAL);
}
a->archive.compression_name = a->filter->name;
a->archive.compression_code = a->filter->code;
return (ARCHIVE_OK);
}
filter
= (struct archive_read_filter *)calloc(1, sizeof(*filter));
if (filter == NULL)
return (ARCHIVE_FATAL);
filter->bidder = best_bidder;
filter->archive = a;
filter->upstream = a->filter;
a->filter = filter;
r = (best_bidder->init)(a->filter);
if (r != ARCHIVE_OK) {
close_filters(a);
free_filters(a);
return (ARCHIVE_FATAL);
}
}
}
void
free_mathmap (mathmap_t *mathmap)
{
if (mathmap->filters != 0)
free_filters(mathmap->filters);
unload_mathmap(mathmap);
free(mathmap);
}
/*
* Release memory and other resources.
*/
static int
_archive_read_free(struct archive *_a)
{
struct archive_read *a = (struct archive_read *)_a;
int i, n;
int slots;
int r = ARCHIVE_OK;
if (_a == NULL)
return (ARCHIVE_OK);
archive_check_magic(_a, ARCHIVE_READ_MAGIC,
ARCHIVE_STATE_ANY | ARCHIVE_STATE_FATAL, "archive_read_free");
if (a->archive.state != ARCHIVE_STATE_CLOSED
&& a->archive.state != ARCHIVE_STATE_FATAL)
r = archive_read_close(&a->archive);
/* Call cleanup functions registered by optional components. */
if (a->cleanup_archive_extract != NULL)
r = (a->cleanup_archive_extract)(a);
/* Cleanup format-specific data. */
slots = sizeof(a->formats) / sizeof(a->formats[0]);
for (i = 0; i < slots; i++) {
a->format = &(a->formats[i]);
if (a->formats[i].cleanup)
(a->formats[i].cleanup)(a);
}
/* Free the filters */
free_filters(a);
/* Release the bidder objects. */
n = sizeof(a->bidders)/sizeof(a->bidders[0]);
for (i = 0; i < n; i++) {
if (a->bidders[i].free != NULL) {
int r1 = (a->bidders[i].free)(&a->bidders[i]);
if (r1 < r)
r = r1;
}
}
archive_string_free(&a->archive.error_string);
if (a->entry)
archive_entry_free(a->entry);
a->archive.magic = 0;
__archive_clean(&a->archive);
free(a->client.dataset);
free(a);
return (r);
}
operation_t user_input(char* tk)
{
if(tk){
char cmpbuff[256];
int tklen = strcspn(tk," \n\0");
memset(cmpbuff,0,256);
if(tklen > 0 && tklen < 256){
strncpy(cmpbuff,tk,tklen);
strcat(cmpbuff,"\0");
if(strcmp(tk,"run")==0 || strcmp(tk,"r")==0){
return RUNFILTERS;
}else if(strcmp(cmpbuff,"add")==0){
return LOAD_FILTER;
}else if(strcmp(cmpbuff,"delete")==0){
return DELETE_FILTER;
}else if(strcmp(cmpbuff,"clear")==0){
tk = strtok(NULL," \n\0");
if(tk && !strcmp(tk,"queries")){
free_buffers();
printf("Queries cleared.\n");
}else if(tk && !strcmp(tk,"filters")){
printf("Filters cleared.\n");
free_filters();
}else{
printf("All cleared.\n");
free_buffers();
free_filters();
}
return OK;
}else if(strcmp(cmpbuff,"config")==0){
return LOAD_CONFIG;
}else if(strcmp(cmpbuff,"in")==0){
return SET_INFILE;
}else if(strcmp(cmpbuff,"out")==0){
return SET_OUTFILE;
}else if(strcmp(cmpbuff,"exit")==0 || strcmp(cmpbuff,"quit")==0 || strcmp(cmpbuff,"q")==0){
return QUIT;
}else if(strcmp(cmpbuff,"help")==0){
print_help();
return OK;
}else if(strcmp(cmpbuff,"status")==0){
print_status();
return OK;
}else if(strcmp(cmpbuff,"quiet")==0){
instance.verbose = 0;
return OK;
}else if(strcmp(cmpbuff,"verbose")==0){
instance.verbose = 1;
return OK;
}else if(strcmp(cmpbuff,"sessions")==0){
return SESS_COUNT;
}else if(strcmp(cmpbuff,"threads")==0){
return THR_COUNT;
}
}
}
return UNDEFINED;
}
int main(int argc, char** argv){
int i;
char buffer[256];
char* tk;
FILTERCHAIN* tmp_chn;
FILTERCHAIN* del_chn;
HARNESS_INSTANCE* hinstance;
if(harness_init(argc,argv,&hinstance)){
printf("Error: Initialization failed.\n");
skygw_log_write(LOGFILE_ERROR,"Error: Initialization failed.\n");
skygw_logmanager_done();
skygw_logmanager_exit();
return 1;
}
if(instance.verbose){
printf("\n\n\tFilter Test Harness\n\n");
}
while(instance.running){
printf("Harness> ");
memset(buffer,0,256);
fgets(buffer,256,stdin);
tk = strtok(buffer," \n");
switch(user_input(tk))
{
case RUNFILTERS:
if(instance.head->next == NULL){
printf("No filters loaded.\n");
break;
}
if(instance.buffer == NULL){
if(instance.infile<0){
manual_query();
}else{
load_query();
}
}
route_buffers();
break;
case LOAD_FILTER:
tk = strtok(NULL," \n");
tmp_chn = load_filter_module(tk);
if(!tmp_chn || !load_filter(tmp_chn,instance.conf)){
printf("Error creating filter instance.\n");
skygw_log_write(LOGFILE_ERROR,"Error: Error creating filter instance.\n");
}else{
instance.head = tmp_chn;
}
break;
case DELETE_FILTER:
tk = strtok(NULL," \n\0");
tmp_chn = instance.head;
del_chn = instance.head;
if(tk){
if(strcmp(instance.head->name,tk) == 0){
instance.head = instance.head->next;
}else{
while(del_chn->next){
if(strcmp(del_chn->name,tk) == 0){
tmp_chn->next = del_chn->next;
break;
}else{
tmp_chn = del_chn;
del_chn = del_chn->next;
}
}
}
if(del_chn && del_chn->next){
printf("Deleted %s.\n",del_chn->name);
if(del_chn->instance){
del_chn->instance->freeSession(del_chn->filter,del_chn->session);
}
free(del_chn->filter);
free(del_chn->down);
free(del_chn->name);
free(del_chn);
}else{
printf("No matching filter found.\n");
}
}
break;
case LOAD_CONFIG:
tk = strtok(NULL," \n\0");
if(!load_config(tk)){
free_filters();
}
break;
case SET_INFILE:
tk = strtok(NULL," \n\0");
if(instance.infile >= 0){
close(instance.infile);
free(instance.infile_name);
}
if(tk!= NULL){
free_buffers();
instance.infile = open_file(tk,0);
if(instance.infile >= 0){
load_query();
instance.infile_name = strdup(tk);
if(instance.verbose){
printf("Loaded %d queries from file '%s'\n",instance.buffer_count,instance.infile_name);
}
}
}else{
instance.infile = -1;
printf("Queries are read from: command line\n");
}
break;
case SET_OUTFILE:
tk = strtok(NULL," \n\0");
if(instance.outfile >= 0){
close(instance.outfile);
free(instance.outfile_name);
}
if(tk!= NULL){
instance.outfile = open_file(tk,1);
if(instance.outfile >= 0){
instance.outfile_name = strdup(tk);
printf("Output is logged to: %s\n",tk);
}
}else{
instance.outfile = -1;
printf("Output logging disabled.\n");
}
break;
case SESS_COUNT:
tk = strtok(NULL," \n\0");
free_buffers();
free_filters();
instance.session_count = atoi(tk);
printf("Sessions set to: %d\n", instance.session_count);
break;
case THR_COUNT:
instance.running = 0;
pthread_mutex_unlock(&instance.work_mtx);
for(i = 0;i<instance.thrcount;i++){
pthread_join(instance.thrpool[i],NULL);
}
pthread_mutex_lock(&instance.work_mtx);
instance.running = 1;
tk = strtok(NULL," \n\0");
instance.thrcount = atoi(tk);
void* t_thr_pool;
if(!(t_thr_pool = realloc(instance.thrpool,instance.thrcount * sizeof(pthread_t)))){
printf("Error: Out of memory\n");
skygw_log_write(LOGFILE_ERROR,"Error: Out of memory\n");
instance.running = 0;
break;
}
instance.thrpool = t_thr_pool;
int thr_num = 1;
for(i = 0;i<instance.thrcount;i++){
pthread_create(&instance.thrpool[i],
NULL,
(void*)work_buffer,
(void*)thr_num++);
}
printf("Threads set to: %d\n", instance.thrcount);
break;
case QUIT:
instance.running = 0;
pthread_mutex_unlock(&instance.work_mtx);
for(i = 0;i<instance.thrcount;i++){
pthread_join(instance.thrpool[i],NULL);
}
break;
case UNDEFINED:
printf("Command not found, enter \"help\" for a list of commands\n");
break;
default:
break;
}
}
if(instance.infile >= 0){
close(instance.infile);
}
if(instance.outfile >= 0){
close(instance.outfile);
}
free_buffers();
free_filters();
skygw_logmanager_done();
skygw_logmanager_exit();
free(instance.head);
return 0;
}
int run_filter(shared_options_data_t *shared_options_data, filter_options_data_t *options_data) {
int ret_code;
double start, stop, total;
vcf_file_t *file = vcf_open(shared_options_data->vcf_filename, shared_options_data->max_batches);
if (!file) {
LOG_FATAL("VCF file does not exist!\n");
}
ret_code = create_directory(shared_options_data->output_directory);
if (ret_code != 0 && errno != EEXIST) {
LOG_FATAL_F("Can't create output directory: %s\n", shared_options_data->output_directory);
}
#pragma omp parallel sections private(start, stop, total)
{
#pragma omp section
{
LOG_DEBUG_F("Thread %d reads the VCF file\n", omp_get_thread_num());
// Reading
start = omp_get_wtime();
if (shared_options_data->batch_bytes > 0) {
ret_code = vcf_parse_batches_in_bytes(shared_options_data->batch_bytes, file);
} else if (shared_options_data->batch_lines > 0) {
ret_code = vcf_parse_batches(shared_options_data->batch_lines, file);
}
stop = omp_get_wtime();
total = stop - start;
if (ret_code) { LOG_FATAL_F("[%dR] Error code = %d\n", omp_get_thread_num(), ret_code); }
LOG_INFO_F("[%dR] Time elapsed = %f s\n", omp_get_thread_num(), total);
LOG_INFO_F("[%dR] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000);
notify_end_parsing(file);
}
#pragma omp section
{
filter_t **filters = NULL;
int num_filters = 0;
if (shared_options_data->chain != NULL) {
filters = sort_filter_chain(shared_options_data->chain, &num_filters);
}
FILE *passed_file = NULL, *failed_file = NULL;
get_filtering_output_files(shared_options_data, &passed_file, &failed_file);
if (!options_data->save_rejected) {
fclose(failed_file);
}
LOG_DEBUG("File streams created\n");
start = omp_get_wtime();
int i = 0;
vcf_batch_t *batch = NULL;
while ((batch = fetch_vcf_batch(file)) != NULL) {
if (i == 0) {
// Add headers associated to the defined filters
vcf_header_entry_t **filter_headers = get_filters_as_vcf_headers(filters, num_filters);
for (int j = 0; j < num_filters; j++) {
add_vcf_header_entry(filter_headers[j], file);
}
// Write file format, header entries and delimiter
write_vcf_header(file, passed_file);
if (options_data->save_rejected) {
write_vcf_header(file, failed_file);
}
LOG_DEBUG("VCF header written created\n");
}
array_list_t *input_records = batch->records;
array_list_t *passed_records, *failed_records;
if (i % 100 == 0) {
LOG_INFO_F("Batch %d reached by thread %d - %zu/%zu records \n",
i, omp_get_thread_num(),
batch->records->size, batch->records->capacity);
}
if (filters == NULL) {
passed_records = input_records;
} else {
failed_records = array_list_new(input_records->size + 1, 1, COLLECTION_MODE_ASYNCHRONIZED);
passed_records = run_filter_chain(input_records, failed_records, filters, num_filters);
}
// Write records that passed and failed to 2 new separated files
if (passed_records != NULL && passed_records->size > 0) {
LOG_DEBUG_F("[batch %d] %zu passed records\n", i, passed_records->size);
#pragma omp critical
{
for (int r = 0; r < passed_records->size; r++) {
write_vcf_record(passed_records->items[r], passed_file);
}
// write_batch(passed_records, passed_file);
}
}
if (options_data->save_rejected && failed_records != NULL && failed_records->size > 0) {
LOG_DEBUG_F("[batch %d] %zu failed records\n", i, failed_records->size);
#pragma omp critical
{
for (int r = 0; r < failed_records->size; r++) {
write_vcf_record(failed_records->items[r], failed_file);
}
// write_batch(failed_records, failed_file);
}
}
// Free batch and its contents
vcf_batch_free(batch);
// Free items in both lists (not their internal data)
if (passed_records != input_records) {
array_list_free(passed_records, NULL);
}
if (failed_records) {
array_list_free(failed_records, NULL);
}
i++;
}
stop = omp_get_wtime();
total = stop - start;
LOG_INFO_F("[%d] Time elapsed = %f s\n", omp_get_thread_num(), total);
LOG_INFO_F("[%d] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000);
// Free resources
if (passed_file) {
fclose(passed_file);
}
if (options_data->save_rejected && failed_file) {
fclose(failed_file);
}
free_filters(filters, num_filters);
}
}
vcf_close(file);
return 0;
}
mathmap_t*
parse_mathmap (char *expression)
{
static mathmap_t *mathmap; /* this is static to avoid problems with longjmp. */
volatile gboolean need_end_scan = FALSE;
mathmap = g_new0(mathmap_t, 1);
the_mathmap = mathmap;
register_native_filters(mathmap);
DO_JUMP_CODE {
filter_t *filter;
scanFromString(expression);
need_end_scan = TRUE;
yyparse();
endScanningFromString();
need_end_scan = FALSE;
if (mathmap->filters == NULL || mathmap->filters->kind != FILTER_MATHMAP)
{
free_filters(mathmap->filters);
mathmap->filters = 0;
sprintf(error_string, _("At least one filter must be defined."));
error_region = scanner_null_region;
JUMP(1);
}
for (filter = mathmap->filters; filter != 0; filter = filter->next)
{
exprtree *expr;
if (filter->kind != FILTER_MATHMAP)
continue;
if (filter->v.mathmap.decl->type != TOP_LEVEL_FILTER)
{
free_filters(mathmap->filters);
mathmap->filters = 0;
sprintf(error_string, _("Top-level declarations can only be filters."));
error_region = scanner_null_region;
JUMP(1);
}
expr = filter->v.mathmap.decl->v.filter.body;
if (expr->result.number != rgba_tag_number
|| expr->result.length != 4)
{
sprintf(error_string, _("The filter `%s' must have the result type rgba:4."), filter->name);
error_region = expr->region;
free_filters(mathmap->filters);
mathmap->filters = 0;
JUMP(1);
}
}
mathmap->main_filter = mathmap->filters;
mathmap->flags = 0;
} WITH_JUMP_HANDLER {
if (need_end_scan)
endScanningFromString();
free_mathmap(mathmap);
mathmap = 0;
} END_JUMP_HANDLER;
the_mathmap = 0;
return mathmap;
}
int load_config( char* fname)
{
CONFIG* iter;
CONFIG_ITEM* item;
int config_ok = 1,inirval;
free_filters();
if((inirval = ini_parse(fname,handler,instance.conf)) < 0){
printf("Error parsing configuration file!\n");
if(inirval == -1)
printf("Inih file open error.\n");
else if(inirval == -2)
printf("inih memory error.\n");
MXS_ERROR("Error parsing configuration file!\n");
config_ok = 0;
goto cleanup;
}
if(instance.verbose){
printf("Configuration loaded from %s\n\n",fname);
}
if(instance.conf == NULL){
printf("Nothing valid was read from the file.\n");
MXS_NOTICE("Nothing valid was read from the file.\n");
config_ok = 0;
goto cleanup;
}
instance.conf = process_config(instance.conf);
if(instance.conf){
if(instance.verbose){
printf("Modules Loaded:\n");
}
iter = instance.conf;
}else{
printf("No filters found in the configuration file.\n");
MXS_NOTICE("No filters found in the configuration file.\n");
config_ok = 0;
goto cleanup;
}
while(iter){
item = iter->item;
while(item){
if(!strcmp("module",item->name)){
if(instance.mod_dir){
char* modstr = malloc(sizeof(char)*(strlen(instance.mod_dir) + strlen(item->value) + 1));
strcpy(modstr,instance.mod_dir);
strcat(modstr,"/");
strcat(modstr,item->value);
instance.head = load_filter_module(modstr);
free(modstr);
}else{
instance.head = load_filter_module(item->value);
}
if(!instance.head || !load_filter(instance.head,instance.conf)){
printf("Error creating filter instance!\nModule: %s\n",item->value);
MXS_ERROR("Error creating filter instance!\nModule: %s\n",item->value);
config_ok = 0;
goto cleanup;
}else{
if(instance.verbose){
printf("\t%s\n",iter->section);
}
}
}
item = item->next;
}
iter = iter->next;
}
while(instance.conf){
item = instance.conf->item;
while(item){
item = instance.conf->item;
instance.conf->item = instance.conf->item->next;
free(item->name);
free(item->value);
free(item);
item = instance.conf->item;
}
instance.conf = instance.conf->next;
}
cleanup:
while(instance.conf){
iter = instance.conf;
instance.conf = instance.conf->next;
item = iter->item;
while(item){
free(item->name);
free(item->value);
free(item);
iter->item = iter->item->next;
item = iter->item;
}
free(iter);
}
instance.conf = NULL;
return config_ok;
}