int highlight_node(const char *filename, struct buffer *buf)
{
int ret;
int length = 0;
int lasttype = -1;
struct ibuf *ibuf = ibuf_init();
struct tokenizer *t = tokenizer_init();
if (tokenizer_set_file(t, filename, buf->language) == -1) {
if_print_message("%s:%d tokenizer_set_file error", __FILE__, __LINE__);
return -1;
}
while ((ret = tokenizer_get_token(t)) > 0) {
enum tokenizer_type e = tokenizer_get_packet_type(t);
/*if_print_message ( "TOKEN(%d:%s)\n", e, tokenizer_get_printable_enum ( e ) ); */
if (e == TOKENIZER_NEWLINE) {
sbpush(buf->tlines, strdup(ibuf_get(ibuf)));
if (length > buf->max_width)
buf->max_width = length;
length = 0;
lasttype = -1;
ibuf_clear(ibuf);
} else {
const char *tok_data = tokenizer_get_data(t);
enum hl_group_kind hlg = hlg_from_tokenizer_type(e, tok_data);
if (hlg == HLG_LAST) {
logger_write_pos(logger, __FILE__, __LINE__, "Bad hlg_type for '%s', e==%d\n", tok_data, e);
hlg = HLG_TEXT;
}
/* Set the highlight group type */
add_type(ibuf, &lasttype, hlg);
/* Add the text and bump our length */
length += ibuf_add(ibuf, tok_data);
}
}
ibuf_free(ibuf);
tokenizer_destroy(t);
return 0;
}
int main ( int argc, char * argv[] )
{
static char* testinp = "First sentence. Second nice sentence. Is this all? No, this is the end!";
char* output;
int outlen;
t_tokenizer mytokenizer;
int i;
printf("Input='%s'\n", testinp);
printf("Initializing... ");
i = tokenizer_init(&mytokenizer);
printf("Done(%d)\n", i);
tokenizer_tokenize(mytokenizer, testinp, &output, &outlen);
printf("Output(%d)='%s'\n", outlen, output);
free(output);
tokenizer_destroy(mytokenizer);
return 0;
}
static int highlight_node(struct list_node *node)
{
int i;
int ret;
int line = 0;
int length = 0;
int lasttype = -1;
struct token_data tok_data;
struct tokenizer *t = tokenizer_init();
struct buffer *buf = &node->file_buf;
for (i = 0; i < sbcount(buf->lines); i++) {
sbfree(buf->lines[i].attrs);
buf->lines[i].attrs = NULL;
}
if (!buf->file_data) {
for (line = 0; line < sbcount(buf->lines); line++) {
struct source_line *sline = &buf->lines[line];
tokenizer_set_buffer(t, sline->line, buf->language);
length = 0;
lasttype = -1;
while ((ret = tokenizer_get_token(t, &tok_data)) > 0) {
if (tok_data.e == TOKENIZER_NEWLINE)
break;
enum hl_group_kind hlg = hlg_from_tokenizer_type(tok_data.e, tok_data.data);
/* Add attribute if highlight group has changed */
if (lasttype != hlg) {
sbpush(buf->lines[line].attrs, hl_line_attr(length, hlg));
lasttype = hlg;
}
/* Add the text and bump our length */
length += strlen(tok_data.data);
}
}
} else {
if (tokenizer_set_buffer(t, buf->file_data, buf->language) == -1) {
if_print_message("%s:%d tokenizer_set_buffer error", __FILE__, __LINE__);
return -1;
}
while ((ret = tokenizer_get_token(t, &tok_data)) > 0) {
if (tok_data.e == TOKENIZER_NEWLINE) {
if (length > buf->max_width)
buf->max_width = length;
length = 0;
lasttype = -1;
line++;
} else {
enum hl_group_kind hlg = hlg_from_tokenizer_type(tok_data.e, tok_data.data);
if (hlg == HLG_LAST) {
clog_error(CLOG_CGDB, "Bad hlg_type for '%s', e==%d\n", tok_data.data, tok_data.e);
hlg = HLG_TEXT;
}
/* Add attribute if highlight group has changed */
if (lasttype != hlg) {
sbpush(buf->lines[line].attrs, hl_line_attr(length, hlg));
lasttype = hlg;
}
/* Add the text and bump our length */
length += strlen(tok_data.data);
}
}
}
tokenizer_destroy(t);
return 0;
}
/**
* Load an AS-level topology in the Subramanian et al format. The
* file format is as follows. Each line describes a directed
* relationship between a two ASes:
* <AS1-number> <AS2-number> <peering-type>
* where peering type can be
* 0 for a peer-to-peer relationship
* 1 for a provider (AS1) to customer (AS2) relationship
* A relationship among two ASes is described in one direction only.
*/
int rexford_parser(FILE * file, as_level_topo_t * topo,
int * line_number)
{
char line[80];
asn_t asn1, asn2;
unsigned int relation;
int error= 0;
net_link_delay_t delay;
gds_tokenizer_t * tokenizer;
const gds_tokens_t * tokens;
as_level_domain_t * domain1, * domain2;
peer_type_t peer_type;
*line_number= 0;
// Parse input file
tokenizer= tokenizer_create(" \t", NULL, NULL);
while ((!feof(file)) && (!error)) {
if (fgets(line, sizeof(line), file) == NULL)
break;
(*line_number)++;
// Skip comments starting with '#'
if (line[0] == '#')
continue;
if (tokenizer_run(tokenizer, line) != TOKENIZER_SUCCESS) {
error= ASLEVEL_ERROR_UNEXPECTED;
break;
}
tokens= tokenizer_get_tokens(tokenizer);
// Set default value for optional parameters
delay= 0;
// Get and check mandatory parameters
if (tokens_get_num(tokens) < 3) {
error= ASLEVEL_ERROR_NUM_PARAMS;
break;
}
// Get and check ASNs
if (str2asn(tokens_get_string_at(tokens, 0), &asn1) ||
str2asn(tokens_get_string_at(tokens, 1), &asn2)) {
error= ASLEVEL_ERROR_INVALID_ASNUM;
break;
}
// Get and check business relationship
if ((tokens_get_uint_at(tokens, 2, &relation) != 0) ||
(_rexford_relation_to_peer_type(relation, &peer_type) != 0)) {
error= ASLEVEL_ERROR_INVALID_RELATION;
break;
}
// Get optional parameters
if (tokens_get_num(tokens) > 3) {
if (str2delay(tokens_get_string_at(tokens, 3), &delay)) {
error= ASLEVEL_ERROR_INVALID_DELAY;
break;
}
}
// Limit number of parameters
if (tokens_get_num(tokens) > 4) {
STREAM_ERR(STREAM_LEVEL_SEVERE,
"Error: too many arguments in topology, line %u\n",
*line_number);
error= ASLEVEL_ERROR_NUM_PARAMS;
break;
}
// Add/get domain 1
if (!(domain1= aslevel_topo_get_as(topo, asn1)) &&
!(domain1= aslevel_topo_add_as(topo, asn1))) {
error= ASLEVEL_ERROR_UNEXPECTED;
break;
}
// Add/get domain 2
if (!(domain2= aslevel_topo_get_as(topo, asn2)) &&
!(domain2= aslevel_topo_add_as(topo, asn2))) {
error= ASLEVEL_ERROR_UNEXPECTED;
break;
}
// Add link in both directions
error= aslevel_as_add_link(domain1, domain2, peer_type, NULL);
if (error != ASLEVEL_SUCCESS)
break;
peer_type= aslevel_reverse_relation(peer_type);
error= aslevel_as_add_link(domain2, domain1, peer_type, NULL);
if (error != ASLEVEL_SUCCESS)
break;
}
tokenizer_destroy(&tokenizer);
return error;
}
/* reverse polish https://en.wikipedia.org/wiki/Shunting-yard_algorithm */
queue *syard_run(const char *in) {
/* init */
stack *s, *arity;
queue *q;
int len;
tokenizer_ctx *tkc;
char *tok, *op, *newstr;
tokenizer_type tok_last = TOKEN_LBRACKET;
char comma = ',', mul = '*';
int *arn;
s = stack_new();
arity = stack_new();
q = queue_new();
tkc = tokenizer_new();
newstr = strdup(in);
newstr[strcspn(newstr, "\r\n")] = 0; /* strip newlines */
tokenizer_reset(tkc, newstr);
/* while there are tokens to be read, read a token. */
while ((tok = tokenizer_next(tkc)) != NULL) {
switch (tkc->type) {
/* if the token is a number, then push it to the output queue. */
case TOKEN_NUMBER:
/* special case: last token was a rbracket and we have number now */
if (tok_last == TOKEN_RBRACKET) {
/* push a * sign with high precendence */
stack_push(s, (void *)&mul);
}
queue_enqueue(q, syard_create_double(tok));
break;
/* if the token is an operator, then: */
case TOKEN_OPERATOR:
/* special case: last token was left bracket or operator and we have a minus sign now */
if ((tok_last == TOKEN_LBRACKET || tok_last == TOKEN_OPERATOR) && (*tok == '-')) {
/* change the operator to the special 'm' operator that we'll deal with in rpn_calc */
*tok = 'm';
}
/* while there is an operator at the top of the operator stack with
greater than or equal to precedence: */
while (((op = stack_top(s)) != NULL) && operator_is_preceding(*op, *tok)) {
/* pop operators from the operator stack, onto the output queue; */
queue_enqueue(q, create_char_data(*(char *)stack_pop(s)));
}
/* push the read operator onto the operator stack. */
stack_push(s, (void *)tok);
break;
/* if the token is a left bracket (i.e. "("), then: */
case TOKEN_LBRACKET:
/* special case: last token was a number or rbracket and we have lbracket now */
if (tok_last == TOKEN_NUMBER || tok_last == TOKEN_RBRACKET) {
/* push a * sign with high precendence */
stack_push(s, (void *)&mul);
}
/* push it onto the operator stack */
stack_push(s, tok);
break;
/* if the token is a right bracket (i.e. ")"), then: */
case TOKEN_RBRACKET:
/* while the operator at the top of the operator stack is not a left bracket: */
while (((op = stack_top(s)) != NULL) && *op != '(') {
/* pop operators from the operator stack onto the output queue. */
queue_enqueue(q, create_char_data(*(char *)stack_pop(s)));
}
/* if the stack runs out without finding a left bracket, then there are
mismatched parentheses. */
if (op == NULL || (op != NULL && *op != '(')) {
/* mismatched parentheses */
printf("! mismatched parentheses; extra )\n");
goto err_cleanup;
}
/* pop the left bracket from the stack. */
stack_pop(s);
/* check if stack top is a function and if so, pop it */
if (stack_top(s) != NULL && *((char *)stack_top(s)) == '\0') {
/* this was a function */
char *ps;
void *p;
/* pop item from stack */
ps = (char *)stack_pop(s);
/* remove leading null */
memmove(ps, ps+1, strlen(ps+1)+1);
/* fetch function arity */
arn = stack_pop(arity);
/* create function data */
p = create_function_data(*arn, ps);
free(ps);
free(arn);
/* enqueue */
queue_enqueue(q, p);
}
break;
case TOKEN_FUNCTION:
/* special case: last token was a number or rbracket and we have variable now */
if (tok_last == TOKEN_NUMBER || tok_last == TOKEN_RBRACKET) {
/* push a * sign with high precendence */
stack_push(s, (void *)&mul);
}
len = strlen(tok);
op = calloc(len + 2, sizeof(char));
op[0] = '\0';
op[len] = '\0';
memcpy(op+1, tok, len);
stack_push(s, op);
arn = malloc(sizeof(int));
*arn = 1;
stack_push(arity, arn);
break;
case TOKEN_COMMA:
while (((op = stack_top(s)) != NULL) && *op != ',' && *op != '(') {
/* pop operators from the operator stack onto the output queue. */
queue_enqueue(q, create_char_data(*(char *)stack_pop(s)));
}
if (*op == ',') stack_pop(s);
stack_push(s, &comma);
arn = stack_pop(arity);
(*arn)++;
stack_push(arity, arn);
break;
case TOKEN_VARIABLE:
/* special case: last token was a number or rbracket and we have variable now */
if (tok_last == TOKEN_NUMBER || tok_last == TOKEN_RBRACKET) {
/* push a * sign with high precendence */
stack_push(s, (void *)&mul);
}
queue_enqueue(q, create_var_data(tok));
break;
default:
break;
}
tok_last = tkc->type;
}
/* if there are no more tokens to read: */
if (tkc->type == TOKEN_END) {
/* while there are still operator tokens on the stack: */
while (((op = stack_top(s)) != NULL) && *op != '(') {
/* pop the operator onto the output queue. */
queue_enqueue(q, create_char_data(*(char *)stack_pop(s)));
}
/* if the operator token on the top of the stack is a bracket, then
there are mismatched parentheses. */
if (op != NULL && *op == '(') {
printf("! mismatched parentheses; extra (\n");
goto err_cleanup;
}
} else {
printf("! unknown character `%c` in equation\n", *(tkc->pos));
goto err_cleanup;
}
stack_destroy(s);
stack_destroy(arity);
tokenizer_destroy(tkc);
free(newstr);
return q;
err_cleanup:
stack_foreach(s, syard_string_cleanup, NULL);
stack_destroy(s);
stack_foreach(arity, syard_queue_cleanup, NULL);
stack_destroy(arity);
tokenizer_destroy(tkc);
queue_foreach(q, syard_queue_cleanup, NULL);
queue_destroy(q);
free(newstr);
return NULL;
}