/* * Allow each registered stream transform to bid on whether * it wants to handle this stream. Repeat until we've finished * building the pipeline. */ static int choose_filters(struct archive_read *a) { int number_bidders, i, bid, best_bid; struct archive_read_filter_bidder *bidder, *best_bidder; struct archive_read_filter *filter; ssize_t avail; int r; for (;;) { number_bidders = sizeof(a->bidders) / sizeof(a->bidders[0]); best_bid = 0; best_bidder = NULL; bidder = a->bidders; for (i = 0; i < number_bidders; i++, bidder++) { if (bidder->bid != NULL) { bid = (bidder->bid)(bidder, a->filter); if (bid > best_bid) { best_bid = bid; best_bidder = bidder; } } } /* If no bidder, we're done. */ if (best_bidder == NULL) { /* Verify the filter by asking it for some data. */ __archive_read_filter_ahead(a->filter, 1, &avail); if (avail < 0) { close_filters(a); free_filters(a); return (ARCHIVE_FATAL); } a->archive.compression_name = a->filter->name; a->archive.compression_code = a->filter->code; return (ARCHIVE_OK); } filter = (struct archive_read_filter *)calloc(1, sizeof(*filter)); if (filter == NULL) return (ARCHIVE_FATAL); filter->bidder = best_bidder; filter->archive = a; filter->upstream = a->filter; a->filter = filter; r = (best_bidder->init)(a->filter); if (r != ARCHIVE_OK) { close_filters(a); free_filters(a); return (ARCHIVE_FATAL); } } }
void free_mathmap (mathmap_t *mathmap) { if (mathmap->filters != 0) free_filters(mathmap->filters); unload_mathmap(mathmap); free(mathmap); }
/* * Release memory and other resources. */ static int _archive_read_free(struct archive *_a) { struct archive_read *a = (struct archive_read *)_a; int i, n; int slots; int r = ARCHIVE_OK; if (_a == NULL) return (ARCHIVE_OK); archive_check_magic(_a, ARCHIVE_READ_MAGIC, ARCHIVE_STATE_ANY | ARCHIVE_STATE_FATAL, "archive_read_free"); if (a->archive.state != ARCHIVE_STATE_CLOSED && a->archive.state != ARCHIVE_STATE_FATAL) r = archive_read_close(&a->archive); /* Call cleanup functions registered by optional components. */ if (a->cleanup_archive_extract != NULL) r = (a->cleanup_archive_extract)(a); /* Cleanup format-specific data. */ slots = sizeof(a->formats) / sizeof(a->formats[0]); for (i = 0; i < slots; i++) { a->format = &(a->formats[i]); if (a->formats[i].cleanup) (a->formats[i].cleanup)(a); } /* Free the filters */ free_filters(a); /* Release the bidder objects. */ n = sizeof(a->bidders)/sizeof(a->bidders[0]); for (i = 0; i < n; i++) { if (a->bidders[i].free != NULL) { int r1 = (a->bidders[i].free)(&a->bidders[i]); if (r1 < r) r = r1; } } archive_string_free(&a->archive.error_string); if (a->entry) archive_entry_free(a->entry); a->archive.magic = 0; __archive_clean(&a->archive); free(a->client.dataset); free(a); return (r); }
operation_t user_input(char* tk) { if(tk){ char cmpbuff[256]; int tklen = strcspn(tk," \n\0"); memset(cmpbuff,0,256); if(tklen > 0 && tklen < 256){ strncpy(cmpbuff,tk,tklen); strcat(cmpbuff,"\0"); if(strcmp(tk,"run")==0 || strcmp(tk,"r")==0){ return RUNFILTERS; }else if(strcmp(cmpbuff,"add")==0){ return LOAD_FILTER; }else if(strcmp(cmpbuff,"delete")==0){ return DELETE_FILTER; }else if(strcmp(cmpbuff,"clear")==0){ tk = strtok(NULL," \n\0"); if(tk && !strcmp(tk,"queries")){ free_buffers(); printf("Queries cleared.\n"); }else if(tk && !strcmp(tk,"filters")){ printf("Filters cleared.\n"); free_filters(); }else{ printf("All cleared.\n"); free_buffers(); free_filters(); } return OK; }else if(strcmp(cmpbuff,"config")==0){ return LOAD_CONFIG; }else if(strcmp(cmpbuff,"in")==0){ return SET_INFILE; }else if(strcmp(cmpbuff,"out")==0){ return SET_OUTFILE; }else if(strcmp(cmpbuff,"exit")==0 || strcmp(cmpbuff,"quit")==0 || strcmp(cmpbuff,"q")==0){ return QUIT; }else if(strcmp(cmpbuff,"help")==0){ print_help(); return OK; }else if(strcmp(cmpbuff,"status")==0){ print_status(); return OK; }else if(strcmp(cmpbuff,"quiet")==0){ instance.verbose = 0; return OK; }else if(strcmp(cmpbuff,"verbose")==0){ instance.verbose = 1; return OK; }else if(strcmp(cmpbuff,"sessions")==0){ return SESS_COUNT; }else if(strcmp(cmpbuff,"threads")==0){ return THR_COUNT; } } } return UNDEFINED; }
int main(int argc, char** argv){ int i; char buffer[256]; char* tk; FILTERCHAIN* tmp_chn; FILTERCHAIN* del_chn; HARNESS_INSTANCE* hinstance; if(harness_init(argc,argv,&hinstance)){ printf("Error: Initialization failed.\n"); skygw_log_write(LOGFILE_ERROR,"Error: Initialization failed.\n"); skygw_logmanager_done(); skygw_logmanager_exit(); return 1; } if(instance.verbose){ printf("\n\n\tFilter Test Harness\n\n"); } while(instance.running){ printf("Harness> "); memset(buffer,0,256); fgets(buffer,256,stdin); tk = strtok(buffer," \n"); switch(user_input(tk)) { case RUNFILTERS: if(instance.head->next == NULL){ printf("No filters loaded.\n"); break; } if(instance.buffer == NULL){ if(instance.infile<0){ manual_query(); }else{ load_query(); } } route_buffers(); break; case LOAD_FILTER: tk = strtok(NULL," \n"); tmp_chn = load_filter_module(tk); if(!tmp_chn || !load_filter(tmp_chn,instance.conf)){ printf("Error creating filter instance.\n"); skygw_log_write(LOGFILE_ERROR,"Error: Error creating filter instance.\n"); }else{ instance.head = tmp_chn; } break; case DELETE_FILTER: tk = strtok(NULL," \n\0"); tmp_chn = instance.head; del_chn = instance.head; if(tk){ if(strcmp(instance.head->name,tk) == 0){ instance.head = instance.head->next; }else{ while(del_chn->next){ if(strcmp(del_chn->name,tk) == 0){ tmp_chn->next = del_chn->next; break; }else{ tmp_chn = del_chn; del_chn = del_chn->next; } } } if(del_chn && del_chn->next){ printf("Deleted %s.\n",del_chn->name); if(del_chn->instance){ del_chn->instance->freeSession(del_chn->filter,del_chn->session); } free(del_chn->filter); free(del_chn->down); free(del_chn->name); free(del_chn); }else{ printf("No matching filter found.\n"); } } break; case LOAD_CONFIG: tk = strtok(NULL," \n\0"); if(!load_config(tk)){ free_filters(); } break; case SET_INFILE: tk = strtok(NULL," \n\0"); if(instance.infile >= 0){ close(instance.infile); free(instance.infile_name); } if(tk!= NULL){ free_buffers(); instance.infile = open_file(tk,0); if(instance.infile >= 0){ load_query(); instance.infile_name = strdup(tk); if(instance.verbose){ printf("Loaded %d queries from file '%s'\n",instance.buffer_count,instance.infile_name); } } }else{ instance.infile = -1; printf("Queries are read from: command line\n"); } break; case SET_OUTFILE: tk = strtok(NULL," \n\0"); if(instance.outfile >= 0){ close(instance.outfile); free(instance.outfile_name); } if(tk!= NULL){ instance.outfile = open_file(tk,1); if(instance.outfile >= 0){ instance.outfile_name = strdup(tk); printf("Output is logged to: %s\n",tk); } }else{ instance.outfile = -1; printf("Output logging disabled.\n"); } break; case SESS_COUNT: tk = strtok(NULL," \n\0"); free_buffers(); free_filters(); instance.session_count = atoi(tk); printf("Sessions set to: %d\n", instance.session_count); break; case THR_COUNT: instance.running = 0; pthread_mutex_unlock(&instance.work_mtx); for(i = 0;i<instance.thrcount;i++){ pthread_join(instance.thrpool[i],NULL); } pthread_mutex_lock(&instance.work_mtx); instance.running = 1; tk = strtok(NULL," \n\0"); instance.thrcount = atoi(tk); void* t_thr_pool; if(!(t_thr_pool = realloc(instance.thrpool,instance.thrcount * sizeof(pthread_t)))){ printf("Error: Out of memory\n"); skygw_log_write(LOGFILE_ERROR,"Error: Out of memory\n"); instance.running = 0; break; } instance.thrpool = t_thr_pool; int thr_num = 1; for(i = 0;i<instance.thrcount;i++){ pthread_create(&instance.thrpool[i], NULL, (void*)work_buffer, (void*)thr_num++); } printf("Threads set to: %d\n", instance.thrcount); break; case QUIT: instance.running = 0; pthread_mutex_unlock(&instance.work_mtx); for(i = 0;i<instance.thrcount;i++){ pthread_join(instance.thrpool[i],NULL); } break; case UNDEFINED: printf("Command not found, enter \"help\" for a list of commands\n"); break; default: break; } } if(instance.infile >= 0){ close(instance.infile); } if(instance.outfile >= 0){ close(instance.outfile); } free_buffers(); free_filters(); skygw_logmanager_done(); skygw_logmanager_exit(); free(instance.head); return 0; }
int run_filter(shared_options_data_t *shared_options_data, filter_options_data_t *options_data) { int ret_code; double start, stop, total; vcf_file_t *file = vcf_open(shared_options_data->vcf_filename, shared_options_data->max_batches); if (!file) { LOG_FATAL("VCF file does not exist!\n"); } ret_code = create_directory(shared_options_data->output_directory); if (ret_code != 0 && errno != EEXIST) { LOG_FATAL_F("Can't create output directory: %s\n", shared_options_data->output_directory); } #pragma omp parallel sections private(start, stop, total) { #pragma omp section { LOG_DEBUG_F("Thread %d reads the VCF file\n", omp_get_thread_num()); // Reading start = omp_get_wtime(); if (shared_options_data->batch_bytes > 0) { ret_code = vcf_parse_batches_in_bytes(shared_options_data->batch_bytes, file); } else if (shared_options_data->batch_lines > 0) { ret_code = vcf_parse_batches(shared_options_data->batch_lines, file); } stop = omp_get_wtime(); total = stop - start; if (ret_code) { LOG_FATAL_F("[%dR] Error code = %d\n", omp_get_thread_num(), ret_code); } LOG_INFO_F("[%dR] Time elapsed = %f s\n", omp_get_thread_num(), total); LOG_INFO_F("[%dR] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000); notify_end_parsing(file); } #pragma omp section { filter_t **filters = NULL; int num_filters = 0; if (shared_options_data->chain != NULL) { filters = sort_filter_chain(shared_options_data->chain, &num_filters); } FILE *passed_file = NULL, *failed_file = NULL; get_filtering_output_files(shared_options_data, &passed_file, &failed_file); if (!options_data->save_rejected) { fclose(failed_file); } LOG_DEBUG("File streams created\n"); start = omp_get_wtime(); int i = 0; vcf_batch_t *batch = NULL; while ((batch = fetch_vcf_batch(file)) != NULL) { if (i == 0) { // Add headers associated to the defined filters vcf_header_entry_t **filter_headers = get_filters_as_vcf_headers(filters, num_filters); for (int j = 0; j < num_filters; j++) { add_vcf_header_entry(filter_headers[j], file); } // Write file format, header entries and delimiter write_vcf_header(file, passed_file); if (options_data->save_rejected) { write_vcf_header(file, failed_file); } LOG_DEBUG("VCF header written created\n"); } array_list_t *input_records = batch->records; array_list_t *passed_records, *failed_records; if (i % 100 == 0) { LOG_INFO_F("Batch %d reached by thread %d - %zu/%zu records \n", i, omp_get_thread_num(), batch->records->size, batch->records->capacity); } if (filters == NULL) { passed_records = input_records; } else { failed_records = array_list_new(input_records->size + 1, 1, COLLECTION_MODE_ASYNCHRONIZED); passed_records = run_filter_chain(input_records, failed_records, filters, num_filters); } // Write records that passed and failed to 2 new separated files if (passed_records != NULL && passed_records->size > 0) { LOG_DEBUG_F("[batch %d] %zu passed records\n", i, passed_records->size); #pragma omp critical { for (int r = 0; r < passed_records->size; r++) { write_vcf_record(passed_records->items[r], passed_file); } // write_batch(passed_records, passed_file); } } if (options_data->save_rejected && failed_records != NULL && failed_records->size > 0) { LOG_DEBUG_F("[batch %d] %zu failed records\n", i, failed_records->size); #pragma omp critical { for (int r = 0; r < failed_records->size; r++) { write_vcf_record(failed_records->items[r], failed_file); } // write_batch(failed_records, failed_file); } } // Free batch and its contents vcf_batch_free(batch); // Free items in both lists (not their internal data) if (passed_records != input_records) { array_list_free(passed_records, NULL); } if (failed_records) { array_list_free(failed_records, NULL); } i++; } stop = omp_get_wtime(); total = stop - start; LOG_INFO_F("[%d] Time elapsed = %f s\n", omp_get_thread_num(), total); LOG_INFO_F("[%d] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000); // Free resources if (passed_file) { fclose(passed_file); } if (options_data->save_rejected && failed_file) { fclose(failed_file); } free_filters(filters, num_filters); } } vcf_close(file); return 0; }
mathmap_t* parse_mathmap (char *expression) { static mathmap_t *mathmap; /* this is static to avoid problems with longjmp. */ volatile gboolean need_end_scan = FALSE; mathmap = g_new0(mathmap_t, 1); the_mathmap = mathmap; register_native_filters(mathmap); DO_JUMP_CODE { filter_t *filter; scanFromString(expression); need_end_scan = TRUE; yyparse(); endScanningFromString(); need_end_scan = FALSE; if (mathmap->filters == NULL || mathmap->filters->kind != FILTER_MATHMAP) { free_filters(mathmap->filters); mathmap->filters = 0; sprintf(error_string, _("At least one filter must be defined.")); error_region = scanner_null_region; JUMP(1); } for (filter = mathmap->filters; filter != 0; filter = filter->next) { exprtree *expr; if (filter->kind != FILTER_MATHMAP) continue; if (filter->v.mathmap.decl->type != TOP_LEVEL_FILTER) { free_filters(mathmap->filters); mathmap->filters = 0; sprintf(error_string, _("Top-level declarations can only be filters.")); error_region = scanner_null_region; JUMP(1); } expr = filter->v.mathmap.decl->v.filter.body; if (expr->result.number != rgba_tag_number || expr->result.length != 4) { sprintf(error_string, _("The filter `%s' must have the result type rgba:4."), filter->name); error_region = expr->region; free_filters(mathmap->filters); mathmap->filters = 0; JUMP(1); } } mathmap->main_filter = mathmap->filters; mathmap->flags = 0; } WITH_JUMP_HANDLER { if (need_end_scan) endScanningFromString(); free_mathmap(mathmap); mathmap = 0; } END_JUMP_HANDLER; the_mathmap = 0; return mathmap; }
int load_config( char* fname) { CONFIG* iter; CONFIG_ITEM* item; int config_ok = 1,inirval; free_filters(); if((inirval = ini_parse(fname,handler,instance.conf)) < 0){ printf("Error parsing configuration file!\n"); if(inirval == -1) printf("Inih file open error.\n"); else if(inirval == -2) printf("inih memory error.\n"); MXS_ERROR("Error parsing configuration file!\n"); config_ok = 0; goto cleanup; } if(instance.verbose){ printf("Configuration loaded from %s\n\n",fname); } if(instance.conf == NULL){ printf("Nothing valid was read from the file.\n"); MXS_NOTICE("Nothing valid was read from the file.\n"); config_ok = 0; goto cleanup; } instance.conf = process_config(instance.conf); if(instance.conf){ if(instance.verbose){ printf("Modules Loaded:\n"); } iter = instance.conf; }else{ printf("No filters found in the configuration file.\n"); MXS_NOTICE("No filters found in the configuration file.\n"); config_ok = 0; goto cleanup; } while(iter){ item = iter->item; while(item){ if(!strcmp("module",item->name)){ if(instance.mod_dir){ char* modstr = malloc(sizeof(char)*(strlen(instance.mod_dir) + strlen(item->value) + 1)); strcpy(modstr,instance.mod_dir); strcat(modstr,"/"); strcat(modstr,item->value); instance.head = load_filter_module(modstr); free(modstr); }else{ instance.head = load_filter_module(item->value); } if(!instance.head || !load_filter(instance.head,instance.conf)){ printf("Error creating filter instance!\nModule: %s\n",item->value); MXS_ERROR("Error creating filter instance!\nModule: %s\n",item->value); config_ok = 0; goto cleanup; }else{ if(instance.verbose){ printf("\t%s\n",iter->section); } } } item = item->next; } iter = iter->next; } while(instance.conf){ item = instance.conf->item; while(item){ item = instance.conf->item; instance.conf->item = instance.conf->item->next; free(item->name); free(item->value); free(item); item = instance.conf->item; } instance.conf = instance.conf->next; } cleanup: while(instance.conf){ iter = instance.conf; instance.conf = instance.conf->next; item = iter->item; while(item){ free(item->name); free(item->value); free(item); iter->item = iter->item->next; item = iter->item; } free(iter); } instance.conf = NULL; return config_ok; }