static enum event_type read_token(char **tok) { enum event_type type; char *token = NULL; do { free_token(token); type = pevent_read_token(&token); } while (type == EVENT_NEWLINE || type == EVENT_SPACE); /* If token is = or ! check to see if the next char is ~ */ if (token && (strcmp(token, "=") == 0 || strcmp(token, "!") == 0) && pevent_peek_char() == '~') { /* append it */ *tok = malloc_or_die(3); sprintf(*tok, "%c%c", *token, '~'); free_token(token); /* Now remove the '~' from the buffer */ pevent_read_token(&token); free_token(token); } else *tok = token; return type; }
int main(int argc, char **argv) { FILE *file = NULL; FILE *out = NULL; if (argc < 2) { fprintf(stderr, "Pass filename as argument.\n"); return 1; } file = fopen(argv[1], "r"); if (argc == 3) { out = fopen(argv[2], "w"); } else { out = stdout; } while (1) { struct token_t *token = get_token(file); print_token(out, token); if (token->type == EOF_T || token->type == ERROR) { free_token(token); break; } else { free_token(token); } } fclose(file); fclose(out); return 0; }
void accept_request(int client){ extern char *buffer2send; /* extern from fileHandle.c */ char buf[0x400]; char *path; int numchars; token query_string; token request_token; struct stat st; #ifdef DEBUGME unsigned int k; #endif /* get the input from the client */ numchars = get_line(client, buf, sizeof(buf)); if(numchars >0){ /* change the input string into several tokens */ get_token(&request_token, buf, " \n\r"); #ifdef DEBUGME PRINT_TOKEN(k, request_token); #endif /* check if the method is supported by our server */ if (strcasecmp(request_token.token[0], "GET")){ getHTML(http_501, "test.html"); }else{ /* if the method is a GET method */ get_token(&query_string, request_token.token[1], "?"); #ifdef DEBUGME printf("query_string: %s\n",query_string.token[0]); #endif path = setPath("./", query_string.token[0]); if(stat(path, &st) == 0 ){ if(!getHTML(http_200, path)){ printf("file open error\n"); close(client); return; } }else{ if(!getHTML(http_404, "notfound.html")){ printf("file open error\n"); close(client); return; } } free(path); free_token(&query_string); } free_token(&request_token); SENDBUFFER2SEND(client); /* free the malloc of the token */ } }
static void x_php_lex (token_ty *tp) { phase4_get (tp); if (tp->type == token_type_string_literal && !(phase5_last == token_type_dot || phase5_last == token_type_operator1 || phase5_last == token_type_operator2 || phase5_last == token_type_rparen)) { char *sum = tp->string; size_t sum_len = strlen (sum); for (;;) { token_ty token2; phase4_get (&token2); if (token2.type == token_type_dot) { token_ty token3; phase4_get (&token3); if (token3.type == token_type_string_literal) { token_ty token_after; phase4_get (&token_after); if (token_after.type != token_type_operator1) { char *addend = token3.string; size_t addend_len = strlen (addend); sum = (char *) xrealloc (sum, sum_len + addend_len + 1); memcpy (sum + sum_len, addend, addend_len + 1); sum_len += addend_len; phase4_unget (&token_after); free_token (&token3); free_token (&token2); continue; } phase4_unget (&token_after); } phase4_unget (&token3); } phase4_unget (&token2); break; } tp->string = sum; } phase5_last = tp->type; }
int token_endstatement(struct token *token) { printf("hello from token_endstatement\n"); if (token->op_type == EOE){ stack_pop(); input_consume(); free_token(token); } else{ log_error(TOKEN_ENDSTATEMENT); free_token(token); return -1; } return 0; }
int assignment_operator(struct token *token) { printf("Hello from assignment operator!\n");//DEBUG if(token->op_type == ASSIGN){ printf("found the assignment!\n");//DEBUG tree_mknode(ASSIGNMENT_OPERATOR); stack_pop(); struct hashentry *he = hash_retrieve(kwtable, token->lexeme); if (he == NULL){ log_error(ASSIGNMENT_OPERATOR); printf("disconnect between scanner's %s and parser!\n", token->lexeme); return -1; } tree_add_attr(he->data, 1); input_consume(); free_token(token); return 0; } return -1; }
int reference_operator(struct token *token) { if ((token->op_type == STAR) || (token->op_type == AND) || (token->op_type == MULTIPLY) || (token->op_type == BWAND)){ tree_mknode(REFERENCE_OPERATOR); struct hashentry *he = hash_retrieve(kwtable, token->lexeme); if (he == NULL){ log_error(ASSIGNMENT_OPERATOR); printf("disconnect between scanner's %s and parser!\n", token->lexeme); return -1; } tree_add_attr(he->data, 0); input_consume(); stack_pop(); free_token(token); return 0; } else { log_error(REFERENCE_OPERATOR); return -1; } }
int lpostfix_expression(struct token *token) { if (token->type == ID_KW){ struct hashentry *ifkw = hash_retrieve(kwtable, token->lexeme); if (ifkw != NULL){ //is a keyword log_error(LPOSTFIX_EXPRESSION); return -1; } else{ struct hashentry *entry = hash_retrieve(symboltable, token->lexeme); if (entry == NULL){ //not defined log_error(POSTFIX_EXPRESSION); printf("%s has not been defined!\n", token->lexeme); } else{ tree_mknode(LPOSTFIX_EXPRESSION); tree_add_attr(entry->data, 1); input_consume(); free_token(token); stack_pop(); stack_push(PRIMARY_EXPRESSION_MODS); return 0; } } } else{ log_error(LPOSTFIX_EXPRESSION); return -1; } return -1; }
void check_valid_list(t_list *list) { t_token *tmp; if (list->level != 0) { free_list(list); my_error(SYNTAXE_ERROR_MSG); } if (list->first->type == 3) { tmp = list->first; list->first = list->first->next; free_token(tmp); } else if (list->first->type > 1) { free_list(list); my_error(SYNTAXE_ERROR_MSG); } if (list->last->type > 0 && list->last->type != list->ops[1]) { free_list(list); my_error(SYNTAXE_ERROR_MSG); } }
int main (int argc, char **argv) { program_name = basename (argv[0]); scan_options (argc, argv); stack *stack = new_stack (); token *scanner = new_token (stdin); for (;;) { int token = scan_token (scanner); if (token == EOF) break; switch (token) { case NUMBER: do_push (stack, peek_token (scanner)); break; case '+': do_binop (stack, add_bigint); break; case '-': do_binop (stack, sub_bigint); break; case '*': do_binop (stack, mul_bigint); break; case 'c': do_clear (stack); break; case 'f': do_print_all (stack); break; case 'p': do_print (stack); break; default: unimplemented (token); break; } } do_clear(stack); free_stack(stack); free_token(scanner); DEBUGF ('m', "EXIT %d\n", exit_status); return EXIT_SUCCESS; }
/* * Print the first pending token truncated then print ellipsis. * Free all tokens and empty the vector. * - p->pending_col = column where printing should start */ static void print_pending_truncated(printer_t *p) { pvector_t *v; void *tk; uint32_t i, n; p->col = p->pending_col; v = &p->pending_tokens; n = v->size; assert(n > 0); tk = v->data[0]; switch (ptr_tag(tk)) { case PP_TOKEN_OPEN_TAG: print_label_truncated(p, untag_open(tk)); break; case PP_TOKEN_ATOMIC_TAG: print_atomic_truncated(p, untag_atomic(tk)); break; case PP_TOKEN_CLOSE_TAG: pp_space(p); pp_ellipsis(p); free_close_token(p, untag_close(tk)); break; case PP_TOKEN_SEPARATOR_TAG: print_atomic_truncated(p, untag_separator(tk)); break; } for (i=1; i<n; i++) { free_token(p, v->data[i]); } pvector_reset(v); }
void free_tokens(cvar_token_t *list_head) { cvar_token_t *curr; if (!list_head) return; while ((curr = list_head->next) != NULL) { list_head->next = curr->next; free_token(curr); } free_token(list_head); return; }
static void phase5_get (token_ty *tp) { phase4_get (tp); /* Combine symbol1 . ... . symbolN to a single strings, so that we can recognize function calls like gettext.gettext. The information present for symbolI.....symbolN has precedence over the information for symbolJ.....symbolN with J > I. */ if (tp->type == token_type_symbol) { char *sum = tp->string; size_t sum_len = strlen (sum); for (;;) { token_ty token2; phase4_get (&token2); if (token2.type == token_type_dot) { token_ty token3; phase4_get (&token3); if (token3.type == token_type_symbol) { char *addend = token3.string; size_t addend_len = strlen (addend); sum = (char *) xrealloc (sum, sum_len + 1 + addend_len + 1); sum[sum_len] = '.'; memcpy (sum + sum_len + 1, addend, addend_len + 1); sum_len += 1 + addend_len; free_token (&token2); free_token (&token3); continue; } phase4_unget (&token3); } phase4_unget (&token2); break; } tp->string = sum; } }
/* * Delete the formatter: * - call free_token for every tk in the queue. * - the printer should not be deleted before the formatter */ static void delete_formatter(formatter_t *f) { while (! ptr_queue_is_empty(&f->token_queue)) { free_token(f->printer, ptr_queue_pop(&f->token_queue)); } delete_ptr_queue(&f->token_queue); delete_block_queue(&f->block_queue); f->last_atom = NULL; }
/* 5 * 2 / 3 */ int factor(struct token_t **current_token, char* input, int* position) { struct token_t* next_t = eat(*current_token, INT, input, position); int ret_val = atoi((*current_token)->value); free_token(*current_token); printf("in factor next_token POINTER: %p\n", next_t); *current_token = next_t; print_token(*current_token); printf("in factor current_token POINTER: %p\n", *current_token); return ret_val; }
/* * Parse an infix expression from a token stream * Returns a parse tree * See https://en.wikipedia.org/wiki/Shunting-yard_algorithm */ Token parse_infix_expression(Token tokens){ Token operators = NULL; unsigned operatorsLen = 0; Token output = NULL; unsigned outputLen = 0; Token nextToken; while(tokens != NULL){ nextToken = tokens->next; tokens->next = NULL; // if it's not a builtin, stick it in output if(tokens->type != BUILTIN){ push_token_stack(tokens, &output, &outputLen); } else { // if it's a builtin, do magic // check if it's allowed to be here ERROR_UNLESS(!can_start_line(tokens->builtin), "statement operators not allowed in expressions") if(operatorsLen == 0){ push_token_stack(tokens, &operators, &operatorsLen); } else if(tokens->builtin == L_PAREN){ push_token_stack(tokens, &operators, &operatorsLen); } else if(tokens->builtin == R_PAREN){ // pop operators until L_PAREN free_token(&tokens); // we don't need the R_PAREN while(operatorsLen > 0 && operators->builtin != L_PAREN) pop_operator(&operators, &operatorsLen, &output, &outputLen); if(operatorsLen == 0){ // there was no L_PAREN, so error ERROR("mismatched parentheses") } else { // discard the L_PAREN Token discard = pop_token_stack(&operators, &operatorsLen); free_token(&discard); } } else if(get_precedence(tokens->builtin) > get_precedence(operators->builtin)){ push_token_stack(tokens, &operators, &operatorsLen); } else { // we are not a parenthesis and we're lower precedence than // the operator on the stack, so pop it and put us there pop_operator(&operators, &operatorsLen, &output, &outputLen); push_token_stack(tokens, &operators, &operatorsLen); } }
//like read, without the return void token_stream_free_head (token_stream_t tst) { if (tst->head) { token_node_t temp_node = tst->head; tst->head = tst->head->next; free_token(temp_node->token); free(temp_node); } }
/* * Free all memory used by the printer * - also call free on all the pending_tokens */ static void delete_printer(printer_t *p) { pvector_t *v; uint32_t i, n; delete_pp_stack(&p->stack); v = &p->pending_tokens; n = v->size; for (i=0; i<n; i++) { free_token(p, v->data[i]); } delete_pvector(v); }
void free_tokens(int n, ...) { TOKEN * t; va_list ap; va_start(ap, n); while (n--) { t = va_arg(ap, TOKEN *); free_token(t); } va_end(ap); }
int expr(char* input) { printf("Entering expr\n"); int position = 0; struct token_t* current_token = get_next_token(input, &position); print_token(current_token); int result = factor(¤t_token, input, &position); printf("got result from factor: %d\n", result); print_token(current_token); enum token current_type = current_token->type; struct token_t* next_token; while (current_type == MUL || current_type == DIV) { printf("\nbeginning of while\n"); if (current_type == MUL) { printf("GOT MUL\n"); next_token = eat(current_token, MUL, input, &position); free_token(current_token); result *= factor(&next_token, input, &position); print_token(next_token); print_token_enum(next_token->type); current_type = next_token->type; } else if (current_type == DIV) { printf("GOT DIV\n"); next_token = eat(current_token, DIV, input, &position); free_token(current_token); result /= factor(&next_token, input, &position); print_token(current_token); print_token_enum(next_token->type); current_type = next_token->type; } current_token = next_token; } printf("\n\nAfter while loop current position is: %d\n", position); printf("current_token 22222 POINTER: %p\n", current_token); if (current_token) { printf("Freeing\n"); free_token(current_token); } return result; }
int token_openparen(struct token *token) { printf("hello from token_openparen\n"); if(token->op_type == OPENPAREN){ input_consume(); free_token(token); stack_pop(); return 0; } else{ log_error(TOKEN_OPENPAREN); return -1; } }
int free_grammar() { GRAMMAR *ptr; //ptr = G_start; while( G_start != NULL ) { ptr = G_start; G_start = G_start->next; free_token( &(ptr->rhs_string) ); free_set( &(ptr->set) ); free(ptr); } }
int inclusive_or_expression(struct token *token) { if (token->op_type == BWOR){ tree_mknode(INCLUSIVE_OR_EXPRESSION); stack_pop(); input_consume();//don't really need to save it free_token(token); return 0; } else{ log_error(INCLUSIVE_OR_EXPRESSION); return -1; } }
int logical_and_expression(struct token *token) { if (token->op_type == LOGAND){ tree_mknode(LOGICAL_AND_EXPRESSION); stack_pop(); input_consume();//don't really need to save it free_token(token); return 0; } else{ log_error(LOGICAL_AND_EXPRESSION); return -1; } }
static struct lnode *liberar_no_atual(struct lnode *no) { struct token *tk; if (list_node_old == NULL) tk = list_remove_front(*tokens); else tk = list_remove_node(*tokens, no, list_node_old); free_token(tk); if (list_node_old == NULL) return (*tokens)->front; else return list_node_old->next; }
int token_closescope(struct token *token) { printf("hello from token_closescope\n"); if(token->op_type == CLOSESCOPE){ input_consume(); free_token(token); stack_pop(); return 0; } else{ log_error(TOKEN_CLOSESCOPE); return -1; } }
int token_openscope(struct token *token) { printf("hello from token_openscope\n"); if(token->op_type == OPENSCOPE){ input_consume(); free_token(token); stack_pop(); return 0; } else{ log_error(OPENSCOPE); return -1; } }
int postfix_expression(struct token *token) { if ((token->type >= ID_KW) && (token->type <= S_C)){ if(token->type == ID_KW){ struct hashentry *ifkw = hash_retrieve(kwtable, token->lexeme); if (ifkw != NULL){ //is a keyword log_error(POSTFIX_EXPRESSION); return -1; } } //if identifier if (token->type == ID_KW){ //already checked if kw struct hashentry *entry = hash_retrieve(symboltable, token->lexeme); if (entry == NULL){ //not defined log_error(POSTFIX_EXPRESSION); printf("%s has not been defined!\n", token->lexeme); return -1; } else{ tree_mknode(POSTFIX_EXPRESSION); tree_add_attr(entry->data, 1); input_consume(); free_token(token); stack_pop(); stack_push(PRIMARY_EXPRESSION_MODS); return 0; } } else{ //should be some sort of constant struct identifier *c = malloc(sizeof(struct identifier)); type_const(c->type); c->lexeme = token->lexeme; tree_mknode(POSTFIX_EXPRESSION); tree_add_attr(c, 1); input_consume(); free(token); stack_pop(); return 0; } } else{ log_error(POSTFIX_EXPRESSION); return -1; } }
void free_token_list(token ** tokens) { int i; token *this_token; if (tokens != NULL) { for (i = 0; (this_token = tokens[i]) != NULL; i++) { free_token(this_token); } free(tokens); } }
void cli_js_destroy(struct parser_state *state) { size_t i; if(!state) return; scope_free_all(state->list); for(i=0;i<state->tokens.cnt;i++) { free_token(&state->tokens.data[i]); } free(state->tokens.data); /* detect use after free */ if(state->scanner) yylex_destroy(state->scanner); memset(state, 0x55, sizeof(*state)); free(state); cli_dbgmsg(MODULE "cli_js_destroy() done\n"); }