static int
same_header(char *mrt_filename, unsigned char *previous_header)
{
off_t read_bytes;
io_t *io_h = wandio_create(mrt_filename);
if(io_h == NULL)
{
bgpstream_log_err("\t\tBSDS_SINGLEFILE: can't open file!");
return -1;
}
read_bytes = wandio_read(io_h, (void *) &(buffer[0]), MAX_HEADER_READ_BYTES);
if(read_bytes < 0)
{
bgpstream_log_err("\t\tBSDS_SINGLEFILE: can't read file!");
wandio_destroy(io_h);
return -1;
}
int ret = memcmp(buffer, previous_header, sizeof(unsigned char) * read_bytes);
wandio_destroy(io_h);
/* if there is no difference, then they have the same header */
if(ret == 0)
{
/* fprintf(stderr, "same header\n"); */
return 1;
}
memcpy(previous_header, buffer, sizeof(unsigned char) * read_bytes);
return 0;
}
int
bgpstream_csvfile_datasource_update_input_queue(bgpstream_csvfile_datasource_t* csvfile_ds,
bgpstream_input_mgr_t *input_mgr) {
bgpstream_debug("\t\tBSDS_CSVFILE: csvfile_ds update input queue start");
io_t *file_io = NULL;
char buffer[BUFFER_LEN];
int read = 0;
struct timeval tv;
gettimeofday(&tv, NULL);
/* we accept all timestamp earlier than now() - 1 second */
csvfile_ds->max_accepted_ts = tv.tv_sec - 1;
csvfile_ds->num_results = 0;
csvfile_ds->max_ts_infile = 0;
csvfile_ds->input_mgr = input_mgr;
if((file_io = wandio_create(csvfile_ds->csvfile_file)) == NULL)
{
bgpstream_log_err("\t\tBSDS_CSVFILE: create csvfile_ds can't open file %s", csvfile_ds->csvfile_file);
return -1;
}
while((read = wandio_read(file_io, &buffer, BUFFER_LEN)) > 0)
{
if(csv_parse(&(csvfile_ds->parser), buffer, read,
parse_csvfile_field,
parse_csvfile_rowend,
csvfile_ds) != read)
{
bgpstream_log_err("\t\tBSDS_CSVFILE: CSV error %s", csv_strerror(csv_error(&(csvfile_ds->parser))));
return -1;
}
}
if(csv_fini(&(csvfile_ds->parser),
parse_csvfile_field,
parse_csvfile_rowend,
csvfile_ds) != 0)
{
bgpstream_log_err("\t\tBSDS_CSVFILE: CSV error %s", csv_strerror(csv_error(&(csvfile_ds->parser))));
return -1;
}
wandio_destroy(file_io);
csvfile_ds->input_mgr = NULL;
csvfile_ds->last_processed_ts = csvfile_ds->max_ts_infile;
bgpstream_debug("\t\tBSDS_CSVFILE: csvfile_ds update input queue end");
return csvfile_ds->num_results;
}
bgpstream_csvfile_datasource_t *
bgpstream_csvfile_datasource_create(bgpstream_filter_mgr_t *filter_mgr,
char *csvfile_file)
{
bgpstream_debug("\t\tBSDS_CSVFILE: create csvfile_ds start");
bgpstream_csvfile_datasource_t *csvfile_ds = (bgpstream_csvfile_datasource_t*) malloc_zero(sizeof(bgpstream_csvfile_datasource_t));
if(csvfile_ds == NULL) {
bgpstream_log_err("\t\tBSDS_CSVFILE: create csvfile_ds can't allocate memory");
goto err;
}
if(csvfile_file == NULL)
{
bgpstream_log_err("\t\tBSDS_CSVFILE: create csvfile_ds no file provided");
goto err;
}
if((csvfile_ds->csvfile_file = strdup(csvfile_file)) == NULL)
{
bgpstream_log_err("\t\tBSDS_CSVFILE: can't allocate memory for filename");
goto err;
}
/* cvs file parser options */
unsigned char options = CSV_STRICT | CSV_REPALL_NL | CSV_STRICT_FINI |
CSV_APPEND_NULL | CSV_EMPTY_IS_NULL;
if(csv_init(&(csvfile_ds->parser), options) !=0)
{
bgpstream_log_err("\t\tBSDS_CSVFILE: can't initialize csv parser");
goto err;
}
csvfile_ds->current_field = CSVFILE_PATH;
csvfile_ds->filter_mgr = filter_mgr;
csvfile_ds->input_mgr = NULL;
csvfile_ds->num_results = 0;
csvfile_ds->max_ts_infile = 0;
csvfile_ds->last_processed_ts = 0;
csvfile_ds->max_accepted_ts = 0;
bgpstream_debug("\t\tBSDS_CSVFILE: create csvfile_ds end");
return csvfile_ds;
err:
bgpstream_csvfile_datasource_destroy(csvfile_ds);
return NULL;
}
bgpstream_singlefile_datasource_t *
bgpstream_singlefile_datasource_create(bgpstream_filter_mgr_t *filter_mgr,
char *singlefile_rib_mrtfile,
char *singlefile_upd_mrtfile)
{
bgpstream_debug("\t\tBSDS_CLIST: create singlefile_ds start");
bgpstream_singlefile_datasource_t *singlefile_ds = (bgpstream_singlefile_datasource_t*) malloc(sizeof(bgpstream_singlefile_datasource_t));
if(singlefile_ds == NULL) {
bgpstream_log_err("\t\tBSDS_CLIST: create singlefile_ds can't allocate memory");
return NULL; // can't allocate memory
}
singlefile_ds->filter_mgr = filter_mgr;
singlefile_ds->rib_filename[0] = '\0';
singlefile_ds->rib_header[0] = '\0';
singlefile_ds->last_rib_filetime = 0;
if(singlefile_rib_mrtfile != NULL)
{
strcpy(singlefile_ds->rib_filename, singlefile_rib_mrtfile);
}
singlefile_ds->update_filename[0] = '\0';
singlefile_ds->update_header[0] = '\0';
singlefile_ds->last_update_filetime = 0;
if(singlefile_upd_mrtfile != NULL)
{
strcpy(singlefile_ds->update_filename, singlefile_upd_mrtfile);
}
bgpstream_debug("\t\tBSDS_CLIST: create customlist_ds end");
return singlefile_ds;
}
static int bgpstream_parse_endvalue(char *conj, fp_state_t *state,
bgpstream_filter_item_t **curr)
{
if (*curr) {
free((*curr)->value);
free(*curr);
*curr = NULL;
}
/* Check for a valid conjunction */
if (strcmp(conj, "and") == 0) {
*state = TERM;
}
/* TODO allow 'or', anything else? */
if (*state != TERM) {
bgpstream_log_err("Bad conjunction in bgpstream filter string: %s", conj);
*state = FAIL;
return FAIL;
}
*curr = (bgpstream_filter_item_t *)calloc(1, sizeof(bgpstream_filter_item_t));
return *state;
}
static int bgpstream_parse_filter_term(char *term, fp_state_t *state,
bgpstream_filter_item_t *curr)
{
/* Painful list of strcmps... */
/* TODO, can we save some time by checking the first character to rule
* out most of the terms quickly?? */
if (strcmp(term, "project") == 0 || strcmp(term, "proj") == 0) {
/* Project */
bgpstream_debug("Got a project term");
curr->termtype = BGPSTREAM_FILTER_TYPE_PROJECT;
*state = VALUE;
return *state;
}
if (strcmp(term, "collector") == 0 || strcmp(term, "coll") == 0) {
/* Collector */
bgpstream_debug("Got a collector term");
curr->termtype = BGPSTREAM_FILTER_TYPE_COLLECTOR;
*state = VALUE;
return *state;
}
if (strcmp(term, "type") == 0) {
/* Type */
bgpstream_debug("Got a type term");
curr->termtype = BGPSTREAM_FILTER_TYPE_RECORD_TYPE;
*state = VALUE;
return *state;
}
if (strcmp(term, "peer") == 0) {
/* Peer */
bgpstream_debug("Got a peer term");
curr->termtype = BGPSTREAM_FILTER_TYPE_ELEM_PEER_ASN;
*state = VALUE;
return *state;
}
if (strcmp(term, "prefix") == 0 || strcmp(term, "pref") == 0) {
/* prefix */
bgpstream_debug("Got a prefix term");
/* XXX is this the best default? */
curr->termtype = BGPSTREAM_FILTER_TYPE_ELEM_PREFIX_MORE;
*state = PREFIXEXT;
return *state;
}
if (strcmp(term, "community") == 0 || strcmp(term, "comm") == 0) {
/* Community */
bgpstream_debug("Got a community term");
curr->termtype = BGPSTREAM_FILTER_TYPE_ELEM_COMMUNITY;
*state = VALUE;
return *state;
}
if (strcmp(term, "aspath") == 0 || strcmp(term, "path") == 0) {
/* AS Path */
bgpstream_debug("Got an aspath term");
curr->termtype = BGPSTREAM_FILTER_TYPE_ELEM_ASPATH;
*state = VALUE;
return *state;
}
if (strcmp(term, "extcommunity") == 0 || strcmp(term, "extc") == 0) {
/* Extended Community */
bgpstream_debug("Got a extended community term");
curr->termtype = BGPSTREAM_FILTER_TYPE_ELEM_EXTENDED_COMMUNITY;
*state = VALUE;
return *state;
}
if (strcmp(term, "ipversion") == 0 || strcmp(term, "ipv") == 0) {
/* IP version */
bgpstream_debug("Got a ip version term");
curr->termtype = BGPSTREAM_FILTER_TYPE_ELEM_IP_VERSION;
*state = VALUE;
return *state;
}
if (strcmp(term, "elemtype") == 0) {
/* Element type */
bgpstream_debug("Got an element type term");
curr->termtype = BGPSTREAM_FILTER_TYPE_ELEM_TYPE;
*state = VALUE;
return *state;
}
bgpstream_log_err("Expected a valid term, got %s", term);
*state = FAIL;
return FAIL;
}
int bgpstream_parse_filter_string(bgpstream_t *bs, const char *fstring)
{
char *tok;
char *sptr = NULL;
int ret = 1;
bgpstream_debug("Parsing filter string - %s", fstring);
bgpstream_filter_item_t *filteritem;
fp_state_t state = TERM;
tok = strtok_r((char *)fstring, (char *)" ", &sptr);
filteritem =
(bgpstream_filter_item_t *)calloc(1, sizeof(bgpstream_filter_item_t));
while (tok != NULL) {
switch (state) {
case TERM:
if (bgpstream_parse_filter_term(tok, &state, filteritem) == FAIL) {
ret = 0;
goto endparsing;
}
break;
case PREFIXEXT:
if (bgpstream_parse_prefixext(tok, &state, filteritem) == FAIL) {
ret = 0;
goto endparsing;
}
if (state == ENDVALUE) {
instantiate_filter(bs, filteritem);
}
break;
case VALUE:
if (bgpstream_parse_value(tok, &state, filteritem) == FAIL) {
ret = 0;
goto endparsing;
}
instantiate_filter(bs, filteritem);
break;
case QUOTEDVALUE:
if (bgpstream_parse_quotedvalue(tok, &state, filteritem) == FAIL) {
ret = 0;
goto endparsing;
}
if (state == ENDVALUE) {
instantiate_filter(bs, filteritem);
}
break;
case ENDVALUE:
if (bgpstream_parse_endvalue(tok, &state, &filteritem) == FAIL) {
ret = 0;
goto endparsing;
}
break;
default:
bgpstream_log_err("Unexpected BGPStream filter string state: %d", state);
ret = 0;
goto endparsing;
}
tok = strtok_r(NULL, " ", &sptr);
}
endparsing:
if (filteritem) {
if (filteritem->value) {
free(filteritem->value);
}
free(filteritem);
}
bgpstream_debug("Finished parsing filter string");
return ret;
}
bgpstream_broker_datasource_t *
bgpstream_broker_datasource_create(bgpstream_filter_mgr_t *filter_mgr,
char *broker_url, char **params,
int params_cnt)
{
int i;
bgpstream_debug("\t\tBSDS_BROKER: create broker_ds start");
bgpstream_broker_datasource_t *broker_ds;
if ((broker_ds = malloc_zero(sizeof(bgpstream_broker_datasource_t))) ==
NULL) {
bgpstream_log_err(
"\t\tBSDS_BROKER: create broker_ds can't allocate memory");
goto err;
}
if (broker_url == NULL) {
bgpstream_log_err("\t\tBSDS_BROKER: create broker_ds no file provided");
goto err;
}
broker_ds->filter_mgr = filter_mgr;
broker_ds->first_param = 1;
broker_ds->query_url_remaining = URL_BUFLEN;
broker_ds->query_url_buf[0] = '\0';
// http://bgpstream.caida.org/broker (e.g.)
APPEND_STR(broker_url);
// http://bgpstream.caida.org/broker/data?
APPEND_STR("/data");
// projects, collectors, bgp_types, and time_intervals are used as filters
// only if they are provided by the user
bgpstream_interval_filter_t *tif;
// projects
char *f;
if (filter_mgr->projects != NULL) {
bgpstream_str_set_rewind(filter_mgr->projects);
while ((f = bgpstream_str_set_next(filter_mgr->projects)) != NULL) {
AMPORQ;
APPEND_STR("projects[]=");
APPEND_STR(f);
}
}
// collectors
if (filter_mgr->collectors != NULL) {
bgpstream_str_set_rewind(filter_mgr->collectors);
while ((f = bgpstream_str_set_next(filter_mgr->collectors)) != NULL) {
AMPORQ;
APPEND_STR("collectors[]=");
APPEND_STR(f);
}
}
// bgp_types
if (filter_mgr->bgp_types != NULL) {
bgpstream_str_set_rewind(filter_mgr->bgp_types);
while ((f = bgpstream_str_set_next(filter_mgr->bgp_types)) != NULL) {
AMPORQ;
APPEND_STR("types[]=");
APPEND_STR(f);
}
}
// user-provided params
for (i = 0; i < params_cnt; i++) {
assert(params[i] != NULL);
AMPORQ;
APPEND_STR(params[i]);
}
// time_intervals
#define BUFLEN 20
char int_buf[BUFLEN];
if (filter_mgr->time_intervals != NULL) {
tif = filter_mgr->time_intervals;
while (tif != NULL) {
AMPORQ;
APPEND_STR("intervals[]=");
// BEGIN TIME
if (snprintf(int_buf, BUFLEN, "%" PRIu32, tif->begin_time) >= BUFLEN) {
goto err;
}
APPEND_STR(int_buf);
APPEND_STR(",");
// END TIME
if (snprintf(int_buf, BUFLEN, "%" PRIu32, tif->end_time) >= BUFLEN) {
goto err;
}
APPEND_STR(int_buf);
tif = tif->next;
}
}
// grab pointer to the end of the current string to simplify modifying the
// query later
broker_ds->query_url_end =
broker_ds->query_url_buf + strlen(broker_ds->query_url_buf);
assert((*broker_ds->query_url_end) == '\0');
bgpstream_debug("\t\tBSDS_BROKER: create broker_ds end");
return broker_ds;
err:
bgpstream_broker_datasource_destroy(broker_ds);
return NULL;
}
static int bgpstream_elem_check_filters(bgpstream_filter_mgr_t *filter_mgr,
bgpstream_elem_t *elem)
{
int pass = 0;
/* First up, check if this element is the right type */
if (filter_mgr->elemtype_mask) {
if (elem->type == BGPSTREAM_ELEM_TYPE_PEERSTATE &&
!(filter_mgr->elemtype_mask & BGPSTREAM_FILTER_ELEM_TYPE_PEERSTATE)) {
return 0;
}
if (elem->type == BGPSTREAM_ELEM_TYPE_RIB &&
!(filter_mgr->elemtype_mask & BGPSTREAM_FILTER_ELEM_TYPE_RIB)) {
return 0;
}
if (elem->type == BGPSTREAM_ELEM_TYPE_ANNOUNCEMENT &&
!(filter_mgr->elemtype_mask &
BGPSTREAM_FILTER_ELEM_TYPE_ANNOUNCEMENT)) {
return 0;
}
if (elem->type == BGPSTREAM_ELEM_TYPE_WITHDRAWAL &&
!(filter_mgr->elemtype_mask & BGPSTREAM_FILTER_ELEM_TYPE_WITHDRAWAL)) {
return 0;
}
}
/* Checking peer ASNs: if the filter is on and the peer asn is not in the
* set, return 0 */
if (filter_mgr->peer_asns &&
bgpstream_id_set_exists(filter_mgr->peer_asns, elem->peer_asnumber) ==
0) {
return 0;
}
if (filter_mgr->ipversion) {
/* Determine address version for the element prefix */
if (elem->type == BGPSTREAM_ELEM_TYPE_PEERSTATE) {
return 0;
}
bgpstream_ip_addr_t *addr = &(((bgpstream_pfx_t *)&elem->prefix)->address);
if (addr->version != filter_mgr->ipversion)
return 0;
}
if (filter_mgr->prefixes) {
if (elem->type == BGPSTREAM_ELEM_TYPE_PEERSTATE) {
return 0;
}
return bgpstream_elem_prefix_match(filter_mgr->prefixes,
(bgpstream_pfx_t *)&elem->prefix);
}
/* Checking AS Path expressions */
if (filter_mgr->aspath_exprs) {
char aspath[65536];
char *regexstr;
int pathlen;
regex_t re;
int result;
int negatives = 0;
int positives = 0;
int totalpositives = 0;
if (elem->type == BGPSTREAM_ELEM_TYPE_WITHDRAWAL ||
elem->type == BGPSTREAM_ELEM_TYPE_PEERSTATE) {
return 0;
}
pathlen = bgpstream_as_path_get_filterable(aspath, 65535, elem->aspath);
if (pathlen == 65535) {
bgpstream_log_warn("AS Path is too long? Filter may not work well.");
}
if (pathlen == 0) {
return 0;
}
bgpstream_str_set_rewind(filter_mgr->aspath_exprs);
while ((regexstr = bgpstream_str_set_next(filter_mgr->aspath_exprs)) !=
NULL) {
int negate = 0;
if (strlen(regexstr) == 0)
continue;
if (*regexstr == '!') {
negate = 1;
regexstr++;
} else {
totalpositives += 1;
}
if (regcomp(&re, regexstr, 0) < 0) {
/* XXX should really use regerror here for proper error reporting */
bgpstream_log_err("Failed to compile AS path regex");
break;
}
result = regexec(&re, aspath, 0, NULL, 0);
if (result == 0) {
if (!negate) {
positives++;
}
if (negate) {
negatives++;
}
}
regfree(&re);
if (result != REG_NOMATCH && result != 0) {
bgpstream_log_err("Error while matching AS path regex");
break;
}
}
if (positives == totalpositives && negatives == 0) {
return 1;
} else {
return 0;
}
}
/* Checking communities (unless it is a withdrawal message) */
pass = (filter_mgr->communities != NULL) ? 0 : 1;
if (filter_mgr->communities) {
if (elem->type == BGPSTREAM_ELEM_TYPE_WITHDRAWAL ||
elem->type == BGPSTREAM_ELEM_TYPE_PEERSTATE) {
return 0;
}
bgpstream_community_t *c;
khiter_t k;
for (k = kh_begin(filter_mgr->communities);
k != kh_end(filter_mgr->communities); ++k) {
if (kh_exist(filter_mgr->communities, k)) {
c = &(kh_key(filter_mgr->communities, k));
if (bgpstream_community_set_match(
elem->communities, c, kh_value(filter_mgr->communities, k))) {
pass = 1;
break;
}
}
}
if (pass == 0) {
return 0;
}
}
return 1;
}
