static int same_header(char *mrt_filename, unsigned char *previous_header) { off_t read_bytes; io_t *io_h = wandio_create(mrt_filename); if(io_h == NULL) { bgpstream_log_err("\t\tBSDS_SINGLEFILE: can't open file!"); return -1; } read_bytes = wandio_read(io_h, (void *) &(buffer[0]), MAX_HEADER_READ_BYTES); if(read_bytes < 0) { bgpstream_log_err("\t\tBSDS_SINGLEFILE: can't read file!"); wandio_destroy(io_h); return -1; } int ret = memcmp(buffer, previous_header, sizeof(unsigned char) * read_bytes); wandio_destroy(io_h); /* if there is no difference, then they have the same header */ if(ret == 0) { /* fprintf(stderr, "same header\n"); */ return 1; } memcpy(previous_header, buffer, sizeof(unsigned char) * read_bytes); return 0; }
static int add_prefixes_from_file(bgpstream_ip_counter_t *poi, char *pfx_file_string) { char pfx_line[MAX_LOG_BUFFER_LEN]; io_t *file = NULL; bgpstream_pfx_storage_t pfx_st; if (pfx_file_string == NULL) { return -1; } if ((file = wandio_create(pfx_file_string)) == NULL) { fprintf(stderr, "ERROR: Could not open prefix file (%s)\n", pfx_file_string); return -1; } while (wandio_fgets(file, &pfx_line, MAX_LOG_BUFFER_LEN, 1) > 0) { /* treat # as comment line, and ignore empty lines */ if (pfx_line[0] == '#' || pfx_line[0] == '\0') { continue; } if (bgpstream_str2pfx(pfx_line, &pfx_st) == NULL || bgpstream_ip_counter_add(poi, (bgpstream_pfx_t *)&pfx_st) != 0) { /* failed to parse/insert the prefix */ return -1; } } wandio_destroy(file); return 0; }
static void bz_close(io_t *io) { BZ2_bzDecompressEnd(&DATA(io)->strm); wandio_destroy(DATA(io)->parent); free(io->data); free(io); }
static void lzma_close(io_t *io) { lzma_end(&DATA(io)->strm); wandio_destroy(DATA(io)->parent); free(io->data); free(io); }
static void peek_close(io_t *io) { /* Make sure we close the child that is doing the actual reading! */ wandio_destroy(DATA(io)->child); if (DATA(io)->buffer) free(DATA(io)->buffer); free(io->data); free(io); }
int bgpstream_csvfile_datasource_update_input_queue(bgpstream_csvfile_datasource_t* csvfile_ds, bgpstream_input_mgr_t *input_mgr) { bgpstream_debug("\t\tBSDS_CSVFILE: csvfile_ds update input queue start"); io_t *file_io = NULL; char buffer[BUFFER_LEN]; int read = 0; struct timeval tv; gettimeofday(&tv, NULL); /* we accept all timestamp earlier than now() - 1 second */ csvfile_ds->max_accepted_ts = tv.tv_sec - 1; csvfile_ds->num_results = 0; csvfile_ds->max_ts_infile = 0; csvfile_ds->input_mgr = input_mgr; if((file_io = wandio_create(csvfile_ds->csvfile_file)) == NULL) { bgpstream_log_err("\t\tBSDS_CSVFILE: create csvfile_ds can't open file %s", csvfile_ds->csvfile_file); return -1; } while((read = wandio_read(file_io, &buffer, BUFFER_LEN)) > 0) { if(csv_parse(&(csvfile_ds->parser), buffer, read, parse_csvfile_field, parse_csvfile_rowend, csvfile_ds) != read) { bgpstream_log_err("\t\tBSDS_CSVFILE: CSV error %s", csv_strerror(csv_error(&(csvfile_ds->parser)))); return -1; } } if(csv_fini(&(csvfile_ds->parser), parse_csvfile_field, parse_csvfile_rowend, csvfile_ds) != 0) { bgpstream_log_err("\t\tBSDS_CSVFILE: CSV error %s", csv_strerror(csv_error(&(csvfile_ds->parser)))); return -1; } wandio_destroy(file_io); csvfile_ds->input_mgr = NULL; csvfile_ds->last_processed_ts = csvfile_ds->max_ts_infile; bgpstream_debug("\t\tBSDS_CSVFILE: csvfile_ds update input queue end"); return csvfile_ds->num_results; }
/** Close a libftrace source */ void ftrace_destroy(libftrace_t *ft) { /* FIXME need to destroy uniflow if currently reading */ if (ft) { pthread_cond_destroy(&ft->wok); pthread_cond_destroy(&ft->rok); pthread_mutex_destroy(&ft->mux); if (ft->tg) libfc_template_group_delete(ft->tg); if (ft->wio) wandio_destroy(ft->wio); free(ft); } }
void corsaro_file_rclose(corsaro_file_in_t *file) { switch(file->mode) { case CORSARO_FILE_MODE_ASCII: case CORSARO_FILE_MODE_BINARY: case CORSARO_FILE_MODE_UNKNOWN: /* close the wandio object */ if(file->wand_io != NULL) { wandio_destroy(file->wand_io); file->wand_io = NULL; } /* just for sanity */ file->trace_io = NULL; break; case CORSARO_FILE_MODE_TRACE: if(file->trace_io != NULL) { trace_destroy(file->trace_io); file->trace_io = NULL; } /* just for sanity */ file->wand_io = NULL; break; default: /* do nothing - it was already freed? */ /* leave the assert here for when people are debugging */ assert(0); return; } /* this will prevent an idiot from using this object again */ file->mode = -1; free(file); return; }
/* The reading thread */ static void *thread_producer(void* userdata) { io_t *state = (io_t*) userdata; int buffer=0; bool running = true; #ifdef PR_SET_NAME char namebuf[17]; if (prctl(PR_GET_NAME, namebuf, 0,0,0) == 0) { namebuf[16] = '\0'; /* Make sure it's NUL terminated */ /* If the filename is too long, overwrite the last few bytes */ if (strlen(namebuf)>9) { strcpy(namebuf+10,"[ior]"); } else { strncat(namebuf," [ior]",16); } prctl(PR_SET_NAME, namebuf, 0,0,0); } #endif pthread_mutex_lock(&DATA(state)->mutex); do { /* If all the buffers are full, we need to wait for one to * become free otherwise we have nowhere to write to! */ while (DATA(state)->buffer[buffer].state == FULL) { if (DATA(state)->closing) break; pthread_cond_wait(&DATA(state)->space_avail, &DATA(state)->mutex); } /* Don't bother reading any more data if we are shutting up * shop */ if (DATA(state)->closing) { break; } pthread_mutex_unlock(&DATA(state)->mutex); /* Get the parent reader to fill the buffer */ DATA(state)->buffer[buffer].len=wandio_read( DATA(state)->io, DATA(state)->buffer[buffer].buffer, sizeof(DATA(state)->buffer[buffer].buffer)); pthread_mutex_lock(&DATA(state)->mutex); DATA(state)->buffer[buffer].state = FULL; /* If we've not reached the end of the file keep going */ running = (DATA(state)->buffer[buffer].len > 0 ); /* Signal that there is data available for the main thread */ pthread_cond_signal(&DATA(state)->data_ready); /* Move on to the next buffer */ buffer=(buffer+1) % max_buffers; } while(running); /* If we reach here, it's all over so start tidying up */ wandio_destroy(DATA(state)->io); pthread_cond_signal(&DATA(state)->data_ready); pthread_mutex_unlock(&DATA(state)->mutex); return NULL; }
int bgpstream_broker_datasource_update_input_queue( bgpstream_broker_datasource_t *broker_ds, bgpstream_input_mgr_t *input_mgr) { // we need to set two parameters: // - dataAddedSince ("time" from last response we got) // - minInitialTime (max("initialTime"+"duration") of any file we've ever seen) #define BUFLEN 20 char buf[BUFLEN]; io_t *jsonfile = NULL; ; int num_results; int attempts = 0; int wait_time = 1; int success = 0; if (broker_ds->last_response_time > 0) { // need to add dataAddedSince if (snprintf(buf, BUFLEN, "%" PRIu32, broker_ds->last_response_time) >= BUFLEN) { fprintf(stderr, "ERROR: Could not build dataAddedSince param string\n"); goto err; } AMPORQ; APPEND_STR("dataAddedSince="); APPEND_STR(buf); } if (broker_ds->current_window_end > 0) { // need to add minInitialTime if (snprintf(buf, BUFLEN, "%" PRIu32, broker_ds->current_window_end) >= BUFLEN) { fprintf(stderr, "ERROR: Could not build minInitialTime param string\n"); goto err; } AMPORQ; APPEND_STR("minInitialTime="); APPEND_STR(buf); } do { if (attempts > 0) { fprintf(stderr, "WARN: Broker request failed, waiting %ds before retry\n", wait_time); sleep(wait_time); if (wait_time < MAX_WAIT_TIME) { wait_time *= 2; } } attempts++; #ifdef WITH_BROKER_DEBUG fprintf(stderr, "\nQuery URL: \"%s\"\n", broker_ds->query_url_buf); #endif if ((jsonfile = wandio_create(broker_ds->query_url_buf)) == NULL) { fprintf(stderr, "ERROR: Could not open %s for reading\n", broker_ds->query_url_buf); goto retry; } if ((num_results = read_json(broker_ds, input_mgr, jsonfile)) == ERR_FATAL) { fprintf(stderr, "ERROR: Received fatal error code from read_json\n"); goto err; } else if (num_results == ERR_RETRY) { goto retry; } else { // success! success = 1; } retry: if (jsonfile != NULL) { wandio_destroy(jsonfile); jsonfile = NULL; } } while (success == 0); // reset the variable params *broker_ds->query_url_end = '\0'; broker_ds->query_url_remaining = URL_BUFLEN - strlen(broker_ds->query_url_end); return num_results; err: fprintf(stderr, "ERROR: Fatal error in broker data source\n"); if (jsonfile != NULL) { wandio_destroy(jsonfile); } return -1; }