Ejemplo n.º 1
0
/*
 * Allow each registered stream transform to bid on whether
 * it wants to handle this stream.  Repeat until we've finished
 * building the pipeline.
 */
static int
choose_filters(struct archive_read *a)
{
	int number_bidders, i, bid, best_bid;
	struct archive_read_filter_bidder *bidder, *best_bidder;
	struct archive_read_filter *filter;
	ssize_t avail;
	int r;

	for (;;) {
		number_bidders = sizeof(a->bidders) / sizeof(a->bidders[0]);

		best_bid = 0;
		best_bidder = NULL;

		bidder = a->bidders;
		for (i = 0; i < number_bidders; i++, bidder++) {
			if (bidder->bid != NULL) {
				bid = (bidder->bid)(bidder, a->filter);
				if (bid > best_bid) {
					best_bid = bid;
					best_bidder = bidder;
				}
			}
		}

		/* If no bidder, we're done. */
		if (best_bidder == NULL) {
			/* Verify the filter by asking it for some data. */
			__archive_read_filter_ahead(a->filter, 1, &avail);
			if (avail < 0) {
				close_filters(a);
				free_filters(a);
				return (ARCHIVE_FATAL);
			}
			a->archive.compression_name = a->filter->name;
			a->archive.compression_code = a->filter->code;
			return (ARCHIVE_OK);
		}

		filter
		    = (struct archive_read_filter *)calloc(1, sizeof(*filter));
		if (filter == NULL)
			return (ARCHIVE_FATAL);
		filter->bidder = best_bidder;
		filter->archive = a;
		filter->upstream = a->filter;
		a->filter = filter;
		r = (best_bidder->init)(a->filter);
		if (r != ARCHIVE_OK) {
			close_filters(a);
			free_filters(a);
			return (ARCHIVE_FATAL);
		}
	}
}
Ejemplo n.º 2
0
void
free_mathmap (mathmap_t *mathmap)
{
    if (mathmap->filters != 0)
	free_filters(mathmap->filters);
    unload_mathmap(mathmap);

    free(mathmap);
}
Ejemplo n.º 3
0
/*
 * Release memory and other resources.
 */
static int
_archive_read_free(struct archive *_a)
{
	struct archive_read *a = (struct archive_read *)_a;
	int i, n;
	int slots;
	int r = ARCHIVE_OK;

	if (_a == NULL)
		return (ARCHIVE_OK);
	archive_check_magic(_a, ARCHIVE_READ_MAGIC,
	    ARCHIVE_STATE_ANY | ARCHIVE_STATE_FATAL, "archive_read_free");
	if (a->archive.state != ARCHIVE_STATE_CLOSED
	    && a->archive.state != ARCHIVE_STATE_FATAL)
		r = archive_read_close(&a->archive);

	/* Call cleanup functions registered by optional components. */
	if (a->cleanup_archive_extract != NULL)
		r = (a->cleanup_archive_extract)(a);

	/* Cleanup format-specific data. */
	slots = sizeof(a->formats) / sizeof(a->formats[0]);
	for (i = 0; i < slots; i++) {
		a->format = &(a->formats[i]);
		if (a->formats[i].cleanup)
			(a->formats[i].cleanup)(a);
	}

	/* Free the filters */
	free_filters(a);

	/* Release the bidder objects. */
	n = sizeof(a->bidders)/sizeof(a->bidders[0]);
	for (i = 0; i < n; i++) {
		if (a->bidders[i].free != NULL) {
			int r1 = (a->bidders[i].free)(&a->bidders[i]);
			if (r1 < r)
				r = r1;
		}
	}

	archive_string_free(&a->archive.error_string);
	if (a->entry)
		archive_entry_free(a->entry);
	a->archive.magic = 0;
	__archive_clean(&a->archive);
	free(a->client.dataset);
	free(a);
	return (r);
}
Ejemplo n.º 4
0
operation_t user_input(char* tk)
{  

  if(tk){

    char cmpbuff[256];
    int tklen = strcspn(tk," \n\0");
    memset(cmpbuff,0,256);
    if(tklen > 0 && tklen < 256){
      strncpy(cmpbuff,tk,tklen);
      strcat(cmpbuff,"\0");
      if(strcmp(tk,"run")==0 || strcmp(tk,"r")==0){
	return RUNFILTERS;

      }else if(strcmp(cmpbuff,"add")==0){
	return LOAD_FILTER;

      }else if(strcmp(cmpbuff,"delete")==0){
	return DELETE_FILTER;

      }else if(strcmp(cmpbuff,"clear")==0){
	tk = strtok(NULL," \n\0");
	if(tk && !strcmp(tk,"queries")){
	  free_buffers();
	  printf("Queries cleared.\n");
	}else if(tk && !strcmp(tk,"filters")){
	  printf("Filters cleared.\n");
	  free_filters();
	}else{
	  printf("All cleared.\n");
	  free_buffers();
	  free_filters();
	}
	
	
	return OK;

      }else if(strcmp(cmpbuff,"config")==0){
	return LOAD_CONFIG;

      }else if(strcmp(cmpbuff,"in")==0){
	return SET_INFILE;

      }else if(strcmp(cmpbuff,"out")==0){
	return SET_OUTFILE;

      }else if(strcmp(cmpbuff,"exit")==0 || strcmp(cmpbuff,"quit")==0 || strcmp(cmpbuff,"q")==0){
	return QUIT;

      }else if(strcmp(cmpbuff,"help")==0){
	print_help();	
	return OK;
      }else if(strcmp(cmpbuff,"status")==0){
	print_status();
	return OK;
      }else if(strcmp(cmpbuff,"quiet")==0){
	instance.verbose = 0;
	return OK;
      }else if(strcmp(cmpbuff,"verbose")==0){
	instance.verbose = 1;
	return OK;
      }else if(strcmp(cmpbuff,"sessions")==0){
	return SESS_COUNT;
      }else if(strcmp(cmpbuff,"threads")==0){
	return THR_COUNT;
      }
    }
  }
  return UNDEFINED;
}
Ejemplo n.º 5
0
int main(int argc, char** argv){
  int i;
  char buffer[256];
  char* tk;
  FILTERCHAIN* tmp_chn;
  FILTERCHAIN* del_chn;  
	HARNESS_INSTANCE* hinstance;

	if(harness_init(argc,argv,&hinstance)){
		printf("Error: Initialization failed.\n");
		skygw_log_write(LOGFILE_ERROR,"Error: Initialization failed.\n");
		skygw_logmanager_done();
		skygw_logmanager_exit();
		return 1;
	}

  if(instance.verbose){
    printf("\n\n\tFilter Test Harness\n\n");
  }
  

  while(instance.running){
    printf("Harness> ");
    memset(buffer,0,256);
    fgets(buffer,256,stdin);
    tk = strtok(buffer," \n");
    switch(user_input(tk))
      {
      case RUNFILTERS:
	if(instance.head->next == NULL){
	  printf("No filters loaded.\n");
	  break;
	}
	if(instance.buffer == NULL){
	  if(instance.infile<0){
	    manual_query();
	  }else{
	    load_query();
	  }
	}
	
	route_buffers();
	break;

      case LOAD_FILTER:

	tk = strtok(NULL," \n");
	tmp_chn = load_filter_module(tk);
	if(!tmp_chn || !load_filter(tmp_chn,instance.conf)){
	  printf("Error creating filter instance.\n");	  
	  skygw_log_write(LOGFILE_ERROR,"Error: Error creating filter instance.\n");
	}else{
	  instance.head =  tmp_chn;
	}
	break;

      case DELETE_FILTER:

	tk = strtok(NULL," \n\0");
	tmp_chn = instance.head;
	del_chn = instance.head;
	if(tk){
	  if(strcmp(instance.head->name,tk) == 0){

	    instance.head = instance.head->next;

	  }else{
	
	    while(del_chn->next){

	      if(strcmp(del_chn->name,tk) == 0){

		tmp_chn->next = del_chn->next;
		break;
	      
	      }else{
		tmp_chn = del_chn;
		del_chn = del_chn->next;
	      

	      }

	    }
	  }

	  if(del_chn && del_chn->next){

	    printf("Deleted %s.\n",del_chn->name);

	    if(del_chn->instance){

	      del_chn->instance->freeSession(del_chn->filter,del_chn->session);

	    }

	    free(del_chn->filter);
	    free(del_chn->down);
	    free(del_chn->name);
	    free(del_chn);
	  }else{
	    printf("No matching filter found.\n");
	  }
	}
	break;

      case LOAD_CONFIG:
	tk = strtok(NULL,"  \n\0");
	if(!load_config(tk)){
	  free_filters();
	}
	break;

      case SET_INFILE:

	tk = strtok(NULL,"  \n\0");
	if(instance.infile >= 0){
	  close(instance.infile);
	  free(instance.infile_name);
	}
	if(tk!= NULL){
	  free_buffers();
	  instance.infile = open_file(tk,0);
	  if(instance.infile >= 0){
	    load_query();
	    instance.infile_name = strdup(tk);
	    if(instance.verbose){
	      printf("Loaded %d queries from file '%s'\n",instance.buffer_count,instance.infile_name);
	    }
	  }
	}else{
	  instance.infile = -1;
	  printf("Queries are read from: command line\n");
	}

	break;

      case SET_OUTFILE:

	tk = strtok(NULL,"  \n\0");
	if(instance.outfile >= 0){
	  close(instance.outfile);
	  free(instance.outfile_name);
	}
	if(tk!= NULL){
	  
	  instance.outfile = open_file(tk,1);
	  if(instance.outfile >= 0){
	    instance.outfile_name = strdup(tk);
	    printf("Output is logged to: %s\n",tk);
	  }
	}else{
	  instance.outfile = -1;
	  printf("Output logging disabled.\n");
	}

	break;

      case SESS_COUNT:

	tk = strtok(NULL,"  \n\0");
	free_buffers();
	free_filters();
	instance.session_count = atoi(tk);
	printf("Sessions set to: %d\n", instance.session_count);
	break;

      case THR_COUNT:

	instance.running = 0;
	pthread_mutex_unlock(&instance.work_mtx);
	for(i = 0;i<instance.thrcount;i++){
	  pthread_join(instance.thrpool[i],NULL);
	}
	pthread_mutex_lock(&instance.work_mtx);

	instance.running = 1;
	tk = strtok(NULL,"  \n\0");
	instance.thrcount = atoi(tk);
	void* t_thr_pool;

	if(!(t_thr_pool = realloc(instance.thrpool,instance.thrcount * sizeof(pthread_t)))){
	  printf("Error: Out of memory\n");
	  skygw_log_write(LOGFILE_ERROR,"Error: Out of memory\n");
	  instance.running = 0;
	  break;
	}

	instance.thrpool = t_thr_pool;
	int thr_num = 1;

	for(i = 0;i<instance.thrcount;i++){

	  pthread_create(&instance.thrpool[i],
			 NULL,
			 (void*)work_buffer,
			 (void*)thr_num++);

	}
	printf("Threads set to: %d\n", instance.thrcount);

	break;

      case QUIT:

	instance.running = 0;
	pthread_mutex_unlock(&instance.work_mtx);
	for(i = 0;i<instance.thrcount;i++){
	  pthread_join(instance.thrpool[i],NULL);
	}
	break;
      case UNDEFINED:

	printf("Command not found, enter \"help\" for a list of commands\n");

	break;
      default:
	
	break;
	
      }  
  }

  if(instance.infile >= 0){
    close(instance.infile);
  }
  if(instance.outfile >= 0){
    close(instance.outfile);
  }

  free_buffers();
  free_filters();
  skygw_logmanager_done();
  skygw_logmanager_exit();
  free(instance.head);

  return 0;
}
Ejemplo n.º 6
0
int run_filter(shared_options_data_t *shared_options_data, filter_options_data_t *options_data) {
    int ret_code;
    double start, stop, total;
    
    vcf_file_t *file = vcf_open(shared_options_data->vcf_filename, shared_options_data->max_batches);
    if (!file) {
        LOG_FATAL("VCF file does not exist!\n");
    }
    
    ret_code = create_directory(shared_options_data->output_directory);
    if (ret_code != 0 && errno != EEXIST) {
        LOG_FATAL_F("Can't create output directory: %s\n", shared_options_data->output_directory);
    }
    
#pragma omp parallel sections private(start, stop, total)
    {
#pragma omp section
        {
            LOG_DEBUG_F("Thread %d reads the VCF file\n", omp_get_thread_num());
            // Reading
            start = omp_get_wtime();

            if (shared_options_data->batch_bytes > 0) {
                ret_code = vcf_parse_batches_in_bytes(shared_options_data->batch_bytes, file);
            } else if (shared_options_data->batch_lines > 0) {
                ret_code = vcf_parse_batches(shared_options_data->batch_lines, file);
            }

            stop = omp_get_wtime();
            total = stop - start;

            if (ret_code) { LOG_FATAL_F("[%dR] Error code = %d\n", omp_get_thread_num(), ret_code); }

            LOG_INFO_F("[%dR] Time elapsed = %f s\n", omp_get_thread_num(), total);
            LOG_INFO_F("[%dR] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000);

            notify_end_parsing(file);
        }
        
#pragma omp section
        {
            filter_t **filters = NULL;
            int num_filters = 0;
            if (shared_options_data->chain != NULL) {
                filters = sort_filter_chain(shared_options_data->chain, &num_filters);
            }
    
            FILE *passed_file = NULL, *failed_file = NULL;
            get_filtering_output_files(shared_options_data, &passed_file, &failed_file);
            if (!options_data->save_rejected) {
                fclose(failed_file);
            }
            LOG_DEBUG("File streams created\n");
            
            start = omp_get_wtime();

            int i = 0;
            vcf_batch_t *batch = NULL;
            while ((batch = fetch_vcf_batch(file)) != NULL) {
                if (i == 0) {
                    // Add headers associated to the defined filters
                    vcf_header_entry_t **filter_headers = get_filters_as_vcf_headers(filters, num_filters);
                    for (int j = 0; j < num_filters; j++) {
                        add_vcf_header_entry(filter_headers[j], file);
                    }
                    
                    // Write file format, header entries and delimiter
                    write_vcf_header(file, passed_file);
                    if (options_data->save_rejected) {
                        write_vcf_header(file, failed_file);
                    }

                    LOG_DEBUG("VCF header written created\n");
                }
                
                array_list_t *input_records = batch->records;
                array_list_t *passed_records, *failed_records;

                if (i % 100 == 0) {
                    LOG_INFO_F("Batch %d reached by thread %d - %zu/%zu records \n", 
                                i, omp_get_thread_num(),
                                batch->records->size, batch->records->capacity);
                }

                if (filters == NULL) {
                    passed_records = input_records;
                } else {
                    failed_records = array_list_new(input_records->size + 1, 1, COLLECTION_MODE_ASYNCHRONIZED);
                    passed_records = run_filter_chain(input_records, failed_records, filters, num_filters);
                }

                // Write records that passed and failed to 2 new separated files
                if (passed_records != NULL && passed_records->size > 0) {
                    LOG_DEBUG_F("[batch %d] %zu passed records\n", i, passed_records->size);
                #pragma omp critical 
                    {
                        for (int r = 0; r < passed_records->size; r++) {
                            write_vcf_record(passed_records->items[r], passed_file);
                        }
//                         write_batch(passed_records, passed_file);
                    }
                }
                
                if (options_data->save_rejected && failed_records != NULL && failed_records->size > 0) {
                    LOG_DEBUG_F("[batch %d] %zu failed records\n", i, failed_records->size);
                #pragma omp critical 
                    {
                        for (int r = 0; r < failed_records->size; r++) {
                            write_vcf_record(failed_records->items[r], failed_file);
                        }
//                         write_batch(failed_records, failed_file);
                    }
                }
                
                // Free batch and its contents
                vcf_batch_free(batch);
                
                // Free items in both lists (not their internal data)
                if (passed_records != input_records) {
                    array_list_free(passed_records, NULL);
                }
                if (failed_records) {
                    array_list_free(failed_records, NULL);
                }
                
                i++;
            }

            stop = omp_get_wtime();

            total = stop - start;

            LOG_INFO_F("[%d] Time elapsed = %f s\n", omp_get_thread_num(), total);
            LOG_INFO_F("[%d] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000);

            // Free resources
            if (passed_file) {
            	fclose(passed_file);
            }
            if (options_data->save_rejected && failed_file) {
            	fclose(failed_file);
            }

            free_filters(filters, num_filters);
        }
    }
    
    vcf_close(file);
    
    return 0;
}
Ejemplo n.º 7
0
mathmap_t*
parse_mathmap (char *expression)
{
    static mathmap_t *mathmap;	/* this is static to avoid problems with longjmp.  */
    volatile gboolean need_end_scan = FALSE;

    mathmap = g_new0(mathmap_t, 1);

    the_mathmap = mathmap;

    register_native_filters(mathmap);

    DO_JUMP_CODE {
	filter_t *filter;

	scanFromString(expression);
	need_end_scan = TRUE;
	yyparse();
	endScanningFromString();
	need_end_scan = FALSE;

	if (mathmap->filters == NULL || mathmap->filters->kind != FILTER_MATHMAP)
	{
	    free_filters(mathmap->filters);
	    mathmap->filters = 0;

	    sprintf(error_string, _("At least one filter must be defined."));
	    error_region = scanner_null_region;
	    JUMP(1);
	}

	for (filter = mathmap->filters; filter != 0; filter = filter->next)
	{
	    exprtree *expr;

	    if (filter->kind != FILTER_MATHMAP)
		continue;

	    if (filter->v.mathmap.decl->type != TOP_LEVEL_FILTER)
	    {
		free_filters(mathmap->filters);
		mathmap->filters = 0;

		sprintf(error_string, _("Top-level declarations can only be filters."));
		error_region = scanner_null_region;
		JUMP(1);
	    }

	    expr = filter->v.mathmap.decl->v.filter.body;

	    if (expr->result.number != rgba_tag_number
		|| expr->result.length != 4)
	    {
		sprintf(error_string, _("The filter `%s' must have the result type rgba:4."), filter->name);
		error_region = expr->region;

		free_filters(mathmap->filters);
		mathmap->filters = 0;

		JUMP(1);
	    }
	}

	mathmap->main_filter = mathmap->filters;

	mathmap->flags = 0;
    } WITH_JUMP_HANDLER {
	if (need_end_scan)
	    endScanningFromString();
	free_mathmap(mathmap);
	mathmap = 0;
    } END_JUMP_HANDLER;

    the_mathmap = 0;

    return mathmap;
}
Ejemplo n.º 8
0
int load_config( char* fname)
{
	CONFIG* iter;
	CONFIG_ITEM* item;
	int config_ok = 1,inirval;
	free_filters();
	if((inirval = ini_parse(fname,handler,instance.conf)) < 0){
		printf("Error parsing configuration file!\n");
        if(inirval == -1)
            printf("Inih file open error.\n");
        else if(inirval == -2)
            printf("inih memory error.\n");
        MXS_ERROR("Error parsing configuration file!\n");
        config_ok = 0;
        goto cleanup;
	}
	if(instance.verbose){
		printf("Configuration loaded from %s\n\n",fname);
	}
	if(instance.conf == NULL){
		printf("Nothing valid was read from the file.\n");
		MXS_NOTICE("Nothing valid was read from the file.\n");
		config_ok = 0;
		goto cleanup;
	}

	instance.conf = process_config(instance.conf);
	if(instance.conf){
		if(instance.verbose){
			printf("Modules Loaded:\n");
		}
		iter = instance.conf;
	}else{
		printf("No filters found in the configuration file.\n");
		MXS_NOTICE("No filters found in the configuration file.\n");
		config_ok = 0;
		goto cleanup;
	}

	while(iter){
		item = iter->item;
		while(item){
      
			if(!strcmp("module",item->name)){

				if(instance.mod_dir){
					char* modstr = malloc(sizeof(char)*(strlen(instance.mod_dir) + strlen(item->value) + 1));
					strcpy(modstr,instance.mod_dir);
					strcat(modstr,"/");
					strcat(modstr,item->value);
					instance.head = load_filter_module(modstr);
					free(modstr);
				}else{
					instance.head = load_filter_module(item->value);
				}


				if(!instance.head || !load_filter(instance.head,instance.conf)){

					printf("Error creating filter instance!\nModule: %s\n",item->value);
					MXS_ERROR("Error creating filter instance!\nModule: %s\n",item->value);
					config_ok = 0;
					goto cleanup;

				}else{
					if(instance.verbose){
						printf("\t%s\n",iter->section);  
					}
				}
			}
			item = item->next;
		}
		iter = iter->next;
	}

	while(instance.conf){
		item = instance.conf->item;
		while(item){
			item = instance.conf->item;
			instance.conf->item = instance.conf->item->next;
			free(item->name);
			free(item->value);
			free(item);
			item = instance.conf->item;
		}
		instance.conf = instance.conf->next;
    
	}

	cleanup:
	while(instance.conf){
		iter = instance.conf;
		instance.conf = instance.conf->next;
		item = iter->item;

		while(item){      
			free(item->name);
			free(item->value);
			free(item);
			iter->item = iter->item->next;
			item = iter->item;
		}

		free(iter);
	}
	instance.conf = NULL;

	return config_ok;
}