static enum event_type read_token(char **tok)
{
enum event_type type;
char *token = NULL;
do {
free_token(token);
type = pevent_read_token(&token);
} while (type == EVENT_NEWLINE || type == EVENT_SPACE);
/* If token is = or ! check to see if the next char is ~ */
if (token &&
(strcmp(token, "=") == 0 || strcmp(token, "!") == 0) &&
pevent_peek_char() == '~') {
/* append it */
*tok = malloc_or_die(3);
sprintf(*tok, "%c%c", *token, '~');
free_token(token);
/* Now remove the '~' from the buffer */
pevent_read_token(&token);
free_token(token);
} else
*tok = token;
return type;
}
int main(int argc, char **argv) {
FILE *file = NULL;
FILE *out = NULL;
if (argc < 2) {
fprintf(stderr, "Pass filename as argument.\n");
return 1;
}
file = fopen(argv[1], "r");
if (argc == 3) {
out = fopen(argv[2], "w");
} else {
out = stdout;
}
while (1) {
struct token_t *token = get_token(file);
print_token(out, token);
if (token->type == EOF_T || token->type == ERROR) {
free_token(token);
break;
} else {
free_token(token);
}
}
fclose(file);
fclose(out);
return 0;
}
void
accept_request(int client){
extern char *buffer2send; /* extern from fileHandle.c */
char buf[0x400];
char *path;
int numchars;
token query_string;
token request_token;
struct stat st;
#ifdef DEBUGME
unsigned int k;
#endif
/* get the input from the client */
numchars = get_line(client, buf, sizeof(buf));
if(numchars >0){
/* change the input string into several tokens */
get_token(&request_token, buf, " \n\r");
#ifdef DEBUGME
PRINT_TOKEN(k, request_token);
#endif
/* check if the method is supported by our server */
if (strcasecmp(request_token.token[0], "GET")){
getHTML(http_501, "test.html");
}else{ /* if the method is a GET method */
get_token(&query_string, request_token.token[1], "?");
#ifdef DEBUGME
printf("query_string: %s\n",query_string.token[0]);
#endif
path = setPath("./", query_string.token[0]);
if(stat(path, &st) == 0 ){
if(!getHTML(http_200, path)){
printf("file open error\n");
close(client);
return;
}
}else{
if(!getHTML(http_404, "notfound.html")){
printf("file open error\n");
close(client);
return;
}
}
free(path);
free_token(&query_string);
}
free_token(&request_token);
SENDBUFFER2SEND(client);
/* free the malloc of the token */
}
}
static void
x_php_lex (token_ty *tp)
{
phase4_get (tp);
if (tp->type == token_type_string_literal
&& !(phase5_last == token_type_dot
|| phase5_last == token_type_operator1
|| phase5_last == token_type_operator2
|| phase5_last == token_type_rparen))
{
char *sum = tp->string;
size_t sum_len = strlen (sum);
for (;;)
{
token_ty token2;
phase4_get (&token2);
if (token2.type == token_type_dot)
{
token_ty token3;
phase4_get (&token3);
if (token3.type == token_type_string_literal)
{
token_ty token_after;
phase4_get (&token_after);
if (token_after.type != token_type_operator1)
{
char *addend = token3.string;
size_t addend_len = strlen (addend);
sum = (char *) xrealloc (sum, sum_len + addend_len + 1);
memcpy (sum + sum_len, addend, addend_len + 1);
sum_len += addend_len;
phase4_unget (&token_after);
free_token (&token3);
free_token (&token2);
continue;
}
phase4_unget (&token_after);
}
phase4_unget (&token3);
}
phase4_unget (&token2);
break;
}
tp->string = sum;
}
phase5_last = tp->type;
}
int token_endstatement(struct token *token)
{
printf("hello from token_endstatement\n");
if (token->op_type == EOE){
stack_pop();
input_consume();
free_token(token);
}
else{
log_error(TOKEN_ENDSTATEMENT);
free_token(token);
return -1;
}
return 0;
}
int assignment_operator(struct token *token)
{
printf("Hello from assignment operator!\n");//DEBUG
if(token->op_type == ASSIGN){
printf("found the assignment!\n");//DEBUG
tree_mknode(ASSIGNMENT_OPERATOR);
stack_pop();
struct hashentry *he = hash_retrieve(kwtable, token->lexeme);
if (he == NULL){
log_error(ASSIGNMENT_OPERATOR);
printf("disconnect between scanner's %s and parser!\n", token->lexeme);
return -1;
}
tree_add_attr(he->data, 1);
input_consume();
free_token(token);
return 0;
}
return -1;
}
int reference_operator(struct token *token)
{
if ((token->op_type == STAR) ||
(token->op_type == AND) ||
(token->op_type == MULTIPLY) ||
(token->op_type == BWAND)){
tree_mknode(REFERENCE_OPERATOR);
struct hashentry *he = hash_retrieve(kwtable, token->lexeme);
if (he == NULL){
log_error(ASSIGNMENT_OPERATOR);
printf("disconnect between scanner's %s and parser!\n", token->lexeme);
return -1;
}
tree_add_attr(he->data, 0);
input_consume();
stack_pop();
free_token(token);
return 0;
}
else {
log_error(REFERENCE_OPERATOR);
return -1;
}
}
int lpostfix_expression(struct token *token)
{
if (token->type == ID_KW){
struct hashentry *ifkw = hash_retrieve(kwtable, token->lexeme);
if (ifkw != NULL){ //is a keyword
log_error(LPOSTFIX_EXPRESSION);
return -1;
}
else{
struct hashentry *entry = hash_retrieve(symboltable, token->lexeme);
if (entry == NULL){ //not defined
log_error(POSTFIX_EXPRESSION);
printf("%s has not been defined!\n", token->lexeme);
}
else{
tree_mknode(LPOSTFIX_EXPRESSION);
tree_add_attr(entry->data, 1);
input_consume();
free_token(token);
stack_pop();
stack_push(PRIMARY_EXPRESSION_MODS);
return 0;
}
}
}
else{
log_error(LPOSTFIX_EXPRESSION);
return -1;
}
return -1;
}
void check_valid_list(t_list *list)
{
t_token *tmp;
if (list->level != 0)
{
free_list(list);
my_error(SYNTAXE_ERROR_MSG);
}
if (list->first->type == 3)
{
tmp = list->first;
list->first = list->first->next;
free_token(tmp);
}
else if (list->first->type > 1)
{
free_list(list);
my_error(SYNTAXE_ERROR_MSG);
}
if (list->last->type > 0 && list->last->type != list->ops[1])
{
free_list(list);
my_error(SYNTAXE_ERROR_MSG);
}
}
int main (int argc, char **argv) {
program_name = basename (argv[0]);
scan_options (argc, argv);
stack *stack = new_stack ();
token *scanner = new_token (stdin);
for (;;) {
int token = scan_token (scanner);
if (token == EOF) break;
switch (token) {
case NUMBER: do_push (stack, peek_token (scanner)); break;
case '+': do_binop (stack, add_bigint); break;
case '-': do_binop (stack, sub_bigint); break;
case '*': do_binop (stack, mul_bigint); break;
case 'c': do_clear (stack); break;
case 'f': do_print_all (stack); break;
case 'p': do_print (stack); break;
default: unimplemented (token); break;
}
}
do_clear(stack);
free_stack(stack);
free_token(scanner);
DEBUGF ('m', "EXIT %d\n", exit_status);
return EXIT_SUCCESS;
}
/*
* Print the first pending token truncated then print ellipsis.
* Free all tokens and empty the vector.
* - p->pending_col = column where printing should start
*/
static void print_pending_truncated(printer_t *p) {
pvector_t *v;
void *tk;
uint32_t i, n;
p->col = p->pending_col;
v = &p->pending_tokens;
n = v->size;
assert(n > 0);
tk = v->data[0];
switch (ptr_tag(tk)) {
case PP_TOKEN_OPEN_TAG:
print_label_truncated(p, untag_open(tk));
break;
case PP_TOKEN_ATOMIC_TAG:
print_atomic_truncated(p, untag_atomic(tk));
break;
case PP_TOKEN_CLOSE_TAG:
pp_space(p);
pp_ellipsis(p);
free_close_token(p, untag_close(tk));
break;
case PP_TOKEN_SEPARATOR_TAG:
print_atomic_truncated(p, untag_separator(tk));
break;
}
for (i=1; i<n; i++) {
free_token(p, v->data[i]);
}
pvector_reset(v);
}
void free_tokens(cvar_token_t *list_head)
{
cvar_token_t *curr;
if (!list_head)
return;
while ((curr = list_head->next) != NULL) {
list_head->next = curr->next;
free_token(curr);
}
free_token(list_head);
return;
}
static void
phase5_get (token_ty *tp)
{
phase4_get (tp);
/* Combine symbol1 . ... . symbolN to a single strings, so that
we can recognize function calls like
gettext.gettext. The information present for
symbolI.....symbolN has precedence over the information for
symbolJ.....symbolN with J > I. */
if (tp->type == token_type_symbol)
{
char *sum = tp->string;
size_t sum_len = strlen (sum);
for (;;)
{
token_ty token2;
phase4_get (&token2);
if (token2.type == token_type_dot)
{
token_ty token3;
phase4_get (&token3);
if (token3.type == token_type_symbol)
{
char *addend = token3.string;
size_t addend_len = strlen (addend);
sum = (char *) xrealloc (sum, sum_len + 1 + addend_len + 1);
sum[sum_len] = '.';
memcpy (sum + sum_len + 1, addend, addend_len + 1);
sum_len += 1 + addend_len;
free_token (&token2);
free_token (&token3);
continue;
}
phase4_unget (&token3);
}
phase4_unget (&token2);
break;
}
tp->string = sum;
}
}
/*
* Delete the formatter:
* - call free_token for every tk in the queue.
* - the printer should not be deleted before the formatter
*/
static void delete_formatter(formatter_t *f) {
while (! ptr_queue_is_empty(&f->token_queue)) {
free_token(f->printer, ptr_queue_pop(&f->token_queue));
}
delete_ptr_queue(&f->token_queue);
delete_block_queue(&f->block_queue);
f->last_atom = NULL;
}
/*
5 * 2 / 3
*/
int factor(struct token_t **current_token, char* input, int* position) {
struct token_t* next_t = eat(*current_token, INT, input, position);
int ret_val = atoi((*current_token)->value);
free_token(*current_token);
printf("in factor next_token POINTER: %p\n", next_t);
*current_token = next_t;
print_token(*current_token);
printf("in factor current_token POINTER: %p\n", *current_token);
return ret_val;
}
/*
* Parse an infix expression from a token stream
* Returns a parse tree
* See https://en.wikipedia.org/wiki/Shunting-yard_algorithm
*/
Token parse_infix_expression(Token tokens){
Token operators = NULL;
unsigned operatorsLen = 0;
Token output = NULL;
unsigned outputLen = 0;
Token nextToken;
while(tokens != NULL){
nextToken = tokens->next;
tokens->next = NULL;
// if it's not a builtin, stick it in output
if(tokens->type != BUILTIN){
push_token_stack(tokens, &output, &outputLen);
} else { // if it's a builtin, do magic
// check if it's allowed to be here
ERROR_UNLESS(!can_start_line(tokens->builtin), "statement operators not allowed in expressions")
if(operatorsLen == 0){
push_token_stack(tokens, &operators, &operatorsLen);
} else if(tokens->builtin == L_PAREN){
push_token_stack(tokens, &operators, &operatorsLen);
} else if(tokens->builtin == R_PAREN){
// pop operators until L_PAREN
free_token(&tokens); // we don't need the R_PAREN
while(operatorsLen > 0 && operators->builtin != L_PAREN)
pop_operator(&operators, &operatorsLen, &output, &outputLen);
if(operatorsLen == 0){ // there was no L_PAREN, so error
ERROR("mismatched parentheses")
} else {
// discard the L_PAREN
Token discard = pop_token_stack(&operators, &operatorsLen);
free_token(&discard);
}
} else if(get_precedence(tokens->builtin) > get_precedence(operators->builtin)){
push_token_stack(tokens, &operators, &operatorsLen);
} else {
// we are not a parenthesis and we're lower precedence than
// the operator on the stack, so pop it and put us there
pop_operator(&operators, &operatorsLen, &output, &outputLen);
push_token_stack(tokens, &operators, &operatorsLen);
}
}
//like read, without the return
void
token_stream_free_head (token_stream_t tst)
{
if (tst->head)
{
token_node_t temp_node = tst->head;
tst->head = tst->head->next;
free_token(temp_node->token);
free(temp_node);
}
}
/*
* Free all memory used by the printer
* - also call free on all the pending_tokens
*/
static void delete_printer(printer_t *p) {
pvector_t *v;
uint32_t i, n;
delete_pp_stack(&p->stack);
v = &p->pending_tokens;
n = v->size;
for (i=0; i<n; i++) {
free_token(p, v->data[i]);
}
delete_pvector(v);
}
void free_tokens(int n, ...)
{
TOKEN * t;
va_list ap;
va_start(ap, n);
while (n--) {
t = va_arg(ap, TOKEN *);
free_token(t);
}
va_end(ap);
}
int expr(char* input) {
printf("Entering expr\n");
int position = 0;
struct token_t* current_token = get_next_token(input, &position);
print_token(current_token);
int result = factor(¤t_token, input, &position);
printf("got result from factor: %d\n", result);
print_token(current_token);
enum token current_type = current_token->type;
struct token_t* next_token;
while (current_type == MUL || current_type == DIV) {
printf("\nbeginning of while\n");
if (current_type == MUL) {
printf("GOT MUL\n");
next_token = eat(current_token, MUL, input, &position);
free_token(current_token);
result *= factor(&next_token, input, &position);
print_token(next_token);
print_token_enum(next_token->type);
current_type = next_token->type;
} else if (current_type == DIV) {
printf("GOT DIV\n");
next_token = eat(current_token, DIV, input, &position);
free_token(current_token);
result /= factor(&next_token, input, &position);
print_token(current_token);
print_token_enum(next_token->type);
current_type = next_token->type;
}
current_token = next_token;
}
printf("\n\nAfter while loop current position is: %d\n", position);
printf("current_token 22222 POINTER: %p\n", current_token);
if (current_token) {
printf("Freeing\n");
free_token(current_token);
}
return result;
}
int token_openparen(struct token *token)
{
printf("hello from token_openparen\n");
if(token->op_type == OPENPAREN){
input_consume();
free_token(token);
stack_pop();
return 0;
}
else{
log_error(TOKEN_OPENPAREN);
return -1;
}
}
int free_grammar()
{
GRAMMAR *ptr;
//ptr = G_start;
while( G_start != NULL )
{
ptr = G_start;
G_start = G_start->next;
free_token( &(ptr->rhs_string) );
free_set( &(ptr->set) );
free(ptr);
}
}
int inclusive_or_expression(struct token *token)
{
if (token->op_type == BWOR){
tree_mknode(INCLUSIVE_OR_EXPRESSION);
stack_pop();
input_consume();//don't really need to save it
free_token(token);
return 0;
}
else{
log_error(INCLUSIVE_OR_EXPRESSION);
return -1;
}
}
int logical_and_expression(struct token *token)
{
if (token->op_type == LOGAND){
tree_mknode(LOGICAL_AND_EXPRESSION);
stack_pop();
input_consume();//don't really need to save it
free_token(token);
return 0;
}
else{
log_error(LOGICAL_AND_EXPRESSION);
return -1;
}
}
static struct lnode *liberar_no_atual(struct lnode *no)
{
struct token *tk;
if (list_node_old == NULL)
tk = list_remove_front(*tokens);
else
tk = list_remove_node(*tokens, no, list_node_old);
free_token(tk);
if (list_node_old == NULL)
return (*tokens)->front;
else
return list_node_old->next;
}
int token_closescope(struct token *token)
{
printf("hello from token_closescope\n");
if(token->op_type == CLOSESCOPE){
input_consume();
free_token(token);
stack_pop();
return 0;
}
else{
log_error(TOKEN_CLOSESCOPE);
return -1;
}
}
int token_openscope(struct token *token)
{
printf("hello from token_openscope\n");
if(token->op_type == OPENSCOPE){
input_consume();
free_token(token);
stack_pop();
return 0;
}
else{
log_error(OPENSCOPE);
return -1;
}
}
int postfix_expression(struct token *token)
{
if ((token->type >= ID_KW) && (token->type <= S_C)){
if(token->type == ID_KW){
struct hashentry *ifkw = hash_retrieve(kwtable, token->lexeme);
if (ifkw != NULL){ //is a keyword
log_error(POSTFIX_EXPRESSION);
return -1;
}
}
//if identifier
if (token->type == ID_KW){ //already checked if kw
struct hashentry *entry = hash_retrieve(symboltable, token->lexeme);
if (entry == NULL){ //not defined
log_error(POSTFIX_EXPRESSION);
printf("%s has not been defined!\n", token->lexeme);
return -1;
}
else{
tree_mknode(POSTFIX_EXPRESSION);
tree_add_attr(entry->data, 1);
input_consume();
free_token(token);
stack_pop();
stack_push(PRIMARY_EXPRESSION_MODS);
return 0;
}
}
else{ //should be some sort of constant
struct identifier *c = malloc(sizeof(struct identifier));
type_const(c->type);
c->lexeme = token->lexeme;
tree_mknode(POSTFIX_EXPRESSION);
tree_add_attr(c, 1);
input_consume();
free(token);
stack_pop();
return 0;
}
}
else{
log_error(POSTFIX_EXPRESSION);
return -1;
}
}
void free_token_list(token ** tokens)
{
int i;
token *this_token;
if (tokens != NULL)
{
for (i = 0; (this_token = tokens[i]) != NULL; i++)
{
free_token(this_token);
}
free(tokens);
}
}
void cli_js_destroy(struct parser_state *state)
{
size_t i;
if(!state)
return;
scope_free_all(state->list);
for(i=0;i<state->tokens.cnt;i++) {
free_token(&state->tokens.data[i]);
}
free(state->tokens.data);
/* detect use after free */
if(state->scanner)
yylex_destroy(state->scanner);
memset(state, 0x55, sizeof(*state));
free(state);
cli_dbgmsg(MODULE "cli_js_destroy() done\n");
}
