static int
same_header(char *mrt_filename, unsigned char *previous_header)
{
off_t read_bytes;
io_t *io_h = wandio_create(mrt_filename);
if(io_h == NULL)
{
bgpstream_log_err("\t\tBSDS_SINGLEFILE: can't open file!");
return -1;
}
read_bytes = wandio_read(io_h, (void *) &(buffer[0]), MAX_HEADER_READ_BYTES);
if(read_bytes < 0)
{
bgpstream_log_err("\t\tBSDS_SINGLEFILE: can't read file!");
wandio_destroy(io_h);
return -1;
}
int ret = memcmp(buffer, previous_header, sizeof(unsigned char) * read_bytes);
wandio_destroy(io_h);
/* if there is no difference, then they have the same header */
if(ret == 0)
{
/* fprintf(stderr, "same header\n"); */
return 1;
}
memcpy(previous_header, buffer, sizeof(unsigned char) * read_bytes);
return 0;
}
static int add_prefixes_from_file(bgpstream_ip_counter_t *poi,
char *pfx_file_string)
{
char pfx_line[MAX_LOG_BUFFER_LEN];
io_t *file = NULL;
bgpstream_pfx_storage_t pfx_st;
if (pfx_file_string == NULL) {
return -1;
}
if ((file = wandio_create(pfx_file_string)) == NULL) {
fprintf(stderr, "ERROR: Could not open prefix file (%s)\n",
pfx_file_string);
return -1;
}
while (wandio_fgets(file, &pfx_line, MAX_LOG_BUFFER_LEN, 1) > 0) {
/* treat # as comment line, and ignore empty lines */
if (pfx_line[0] == '#' || pfx_line[0] == '\0') {
continue;
}
if (bgpstream_str2pfx(pfx_line, &pfx_st) == NULL ||
bgpstream_ip_counter_add(poi, (bgpstream_pfx_t *)&pfx_st) != 0) {
/* failed to parse/insert the prefix */
return -1;
}
}
wandio_destroy(file);
return 0;
}
static void bz_close(io_t *io)
{
BZ2_bzDecompressEnd(&DATA(io)->strm);
wandio_destroy(DATA(io)->parent);
free(io->data);
free(io);
}
static void lzma_close(io_t *io)
{
lzma_end(&DATA(io)->strm);
wandio_destroy(DATA(io)->parent);
free(io->data);
free(io);
}
static void peek_close(io_t *io)
{
/* Make sure we close the child that is doing the actual reading! */
wandio_destroy(DATA(io)->child);
if (DATA(io)->buffer)
free(DATA(io)->buffer);
free(io->data);
free(io);
}
int
bgpstream_csvfile_datasource_update_input_queue(bgpstream_csvfile_datasource_t* csvfile_ds,
bgpstream_input_mgr_t *input_mgr) {
bgpstream_debug("\t\tBSDS_CSVFILE: csvfile_ds update input queue start");
io_t *file_io = NULL;
char buffer[BUFFER_LEN];
int read = 0;
struct timeval tv;
gettimeofday(&tv, NULL);
/* we accept all timestamp earlier than now() - 1 second */
csvfile_ds->max_accepted_ts = tv.tv_sec - 1;
csvfile_ds->num_results = 0;
csvfile_ds->max_ts_infile = 0;
csvfile_ds->input_mgr = input_mgr;
if((file_io = wandio_create(csvfile_ds->csvfile_file)) == NULL)
{
bgpstream_log_err("\t\tBSDS_CSVFILE: create csvfile_ds can't open file %s", csvfile_ds->csvfile_file);
return -1;
}
while((read = wandio_read(file_io, &buffer, BUFFER_LEN)) > 0)
{
if(csv_parse(&(csvfile_ds->parser), buffer, read,
parse_csvfile_field,
parse_csvfile_rowend,
csvfile_ds) != read)
{
bgpstream_log_err("\t\tBSDS_CSVFILE: CSV error %s", csv_strerror(csv_error(&(csvfile_ds->parser))));
return -1;
}
}
if(csv_fini(&(csvfile_ds->parser),
parse_csvfile_field,
parse_csvfile_rowend,
csvfile_ds) != 0)
{
bgpstream_log_err("\t\tBSDS_CSVFILE: CSV error %s", csv_strerror(csv_error(&(csvfile_ds->parser))));
return -1;
}
wandio_destroy(file_io);
csvfile_ds->input_mgr = NULL;
csvfile_ds->last_processed_ts = csvfile_ds->max_ts_infile;
bgpstream_debug("\t\tBSDS_CSVFILE: csvfile_ds update input queue end");
return csvfile_ds->num_results;
}
/** Close a libftrace source */
void ftrace_destroy(libftrace_t *ft)
{
/* FIXME need to destroy uniflow if currently reading */
if (ft) {
pthread_cond_destroy(&ft->wok);
pthread_cond_destroy(&ft->rok);
pthread_mutex_destroy(&ft->mux);
if (ft->tg) libfc_template_group_delete(ft->tg);
if (ft->wio) wandio_destroy(ft->wio);
free(ft);
}
}
void corsaro_file_rclose(corsaro_file_in_t *file)
{
switch(file->mode)
{
case CORSARO_FILE_MODE_ASCII:
case CORSARO_FILE_MODE_BINARY:
case CORSARO_FILE_MODE_UNKNOWN:
/* close the wandio object */
if(file->wand_io != NULL)
{
wandio_destroy(file->wand_io);
file->wand_io = NULL;
}
/* just for sanity */
file->trace_io = NULL;
break;
case CORSARO_FILE_MODE_TRACE:
if(file->trace_io != NULL)
{
trace_destroy(file->trace_io);
file->trace_io = NULL;
}
/* just for sanity */
file->wand_io = NULL;
break;
default:
/* do nothing - it was already freed? */
/* leave the assert here for when people are debugging */
assert(0);
return;
}
/* this will prevent an idiot from using this object again */
file->mode = -1;
free(file);
return;
}
/* The reading thread */
static void *thread_producer(void* userdata)
{
io_t *state = (io_t*) userdata;
int buffer=0;
bool running = true;
#ifdef PR_SET_NAME
char namebuf[17];
if (prctl(PR_GET_NAME, namebuf, 0,0,0) == 0) {
namebuf[16] = '\0'; /* Make sure it's NUL terminated */
/* If the filename is too long, overwrite the last few bytes */
if (strlen(namebuf)>9) {
strcpy(namebuf+10,"[ior]");
}
else {
strncat(namebuf," [ior]",16);
}
prctl(PR_SET_NAME, namebuf, 0,0,0);
}
#endif
pthread_mutex_lock(&DATA(state)->mutex);
do {
/* If all the buffers are full, we need to wait for one to
* become free otherwise we have nowhere to write to! */
while (DATA(state)->buffer[buffer].state == FULL) {
if (DATA(state)->closing)
break;
pthread_cond_wait(&DATA(state)->space_avail, &DATA(state)->mutex);
}
/* Don't bother reading any more data if we are shutting up
* shop */
if (DATA(state)->closing) {
break;
}
pthread_mutex_unlock(&DATA(state)->mutex);
/* Get the parent reader to fill the buffer */
DATA(state)->buffer[buffer].len=wandio_read(
DATA(state)->io,
DATA(state)->buffer[buffer].buffer,
sizeof(DATA(state)->buffer[buffer].buffer));
pthread_mutex_lock(&DATA(state)->mutex);
DATA(state)->buffer[buffer].state = FULL;
/* If we've not reached the end of the file keep going */
running = (DATA(state)->buffer[buffer].len > 0 );
/* Signal that there is data available for the main thread */
pthread_cond_signal(&DATA(state)->data_ready);
/* Move on to the next buffer */
buffer=(buffer+1) % max_buffers;
} while(running);
/* If we reach here, it's all over so start tidying up */
wandio_destroy(DATA(state)->io);
pthread_cond_signal(&DATA(state)->data_ready);
pthread_mutex_unlock(&DATA(state)->mutex);
return NULL;
}
int bgpstream_broker_datasource_update_input_queue(
bgpstream_broker_datasource_t *broker_ds, bgpstream_input_mgr_t *input_mgr)
{
// we need to set two parameters:
// - dataAddedSince ("time" from last response we got)
// - minInitialTime (max("initialTime"+"duration") of any file we've ever seen)
#define BUFLEN 20
char buf[BUFLEN];
io_t *jsonfile = NULL;
;
int num_results;
int attempts = 0;
int wait_time = 1;
int success = 0;
if (broker_ds->last_response_time > 0) {
// need to add dataAddedSince
if (snprintf(buf, BUFLEN, "%" PRIu32, broker_ds->last_response_time) >=
BUFLEN) {
fprintf(stderr, "ERROR: Could not build dataAddedSince param string\n");
goto err;
}
AMPORQ;
APPEND_STR("dataAddedSince=");
APPEND_STR(buf);
}
if (broker_ds->current_window_end > 0) {
// need to add minInitialTime
if (snprintf(buf, BUFLEN, "%" PRIu32, broker_ds->current_window_end) >=
BUFLEN) {
fprintf(stderr, "ERROR: Could not build minInitialTime param string\n");
goto err;
}
AMPORQ;
APPEND_STR("minInitialTime=");
APPEND_STR(buf);
}
do {
if (attempts > 0) {
fprintf(stderr, "WARN: Broker request failed, waiting %ds before retry\n",
wait_time);
sleep(wait_time);
if (wait_time < MAX_WAIT_TIME) {
wait_time *= 2;
}
}
attempts++;
#ifdef WITH_BROKER_DEBUG
fprintf(stderr, "\nQuery URL: \"%s\"\n", broker_ds->query_url_buf);
#endif
if ((jsonfile = wandio_create(broker_ds->query_url_buf)) == NULL) {
fprintf(stderr, "ERROR: Could not open %s for reading\n",
broker_ds->query_url_buf);
goto retry;
}
if ((num_results = read_json(broker_ds, input_mgr, jsonfile)) ==
ERR_FATAL) {
fprintf(stderr, "ERROR: Received fatal error code from read_json\n");
goto err;
} else if (num_results == ERR_RETRY) {
goto retry;
} else {
// success!
success = 1;
}
retry:
if (jsonfile != NULL) {
wandio_destroy(jsonfile);
jsonfile = NULL;
}
} while (success == 0);
// reset the variable params
*broker_ds->query_url_end = '\0';
broker_ds->query_url_remaining =
URL_BUFLEN - strlen(broker_ds->query_url_end);
return num_results;
err:
fprintf(stderr, "ERROR: Fatal error in broker data source\n");
if (jsonfile != NULL) {
wandio_destroy(jsonfile);
}
return -1;
}
