token create_binary_operator_token(char c) { switch(c) { case '+': return create_token(binary_operator, plus); case '-': return create_token(binary_operator, minus); case '*': return create_token(binary_operator, multiply); case '/': return create_token(binary_operator, divide); } error_message("wrong argument passed to create_binary_operator_token\n"); exit(1); }
//-------------------------------------------------------- //static int8_t input0 (func_cb_ptr p, void *data, uint16_t length) { static int8_t input0 (func_cb_ptr p, token_type_t *t) { update_background_state_t *s = (update_background_state_t *)sys_get_state(); // Get token from port. if (s->state == UPD_STATE_INIT) { s->backMat = (CYCLOPS_Matrix *)capture_token_data(t, s->pid); if (s->backMat == NULL) return -ENOMEM; s->state = UPD_STATE_PROCESS; } else { CYCLOPS_Matrix *M = (CYCLOPS_Matrix *)get_token_data(t); //check that input matrix's depth is 1 byte if( (M->depth != CYCLOPS_1BYTE) || (s->backMat->depth != CYCLOPS_1BYTE) ) return -EINVAL; token_type_t *my_token = create_token(s->backMat, sizeof(CYCLOPS_Matrix), s->pid); if (my_token == NULL) return -EINVAL; set_token_type(my_token, CYCLOPS_MATRIX); //SOS_CALL(s->put_token, put_token_func_t, s->output0, my_token); dispatch(s->output0, my_token); destroy_token(my_token); } return SOS_OK; }
//-------------------------------------------------------- static int8_t process_input(over_thresh_state_t *s) { uint8_t overTheThresh; objectInfo newObject; if (s->state < 4) { return -EBUSY; } s->state = 0; overTheThresh = over_thresh(s->A, s->row, s->col, RANGE, s->thresh); newObject.objectPosition.x = s->col; newObject.objectPosition.y = s->row; newObject.objectSize.x = ROWS; newObject.objectSize.y = COLS; newObject.actionFlag = NONE; if (overTheThresh > DETECT_THRESH) { LED_DBG(LED_RED_TOGGLE); newObject.actionFlag = OBJ_DETECT; } token_type_t *my_token = create_token(&newObject, sizeof(objectInfo), s->pid); if (my_token == NULL) return -ENOMEM; //SOS_CALL(s->put_token, put_token_func_t, s->output0, my_token); dispatch(s->output0, my_token); destroy_token(my_token); //destroy_cyclops_matrix(s->A); destroy_token_data(s->A, CYCLOPS_MATRIX, sizeof(CYCLOPS_Matrix)); s->A = NULL; return SOS_OK; }
static PyObject* Preprocessor_tokenize(Preprocessor* self, PyObject *args) { const char *s = NULL; Py_ssize_t len; if (!PyArg_ParseTuple(args, "s#:tokenize", &s, &len)) return NULL; try { std::vector<cmonster::core::Token> result = self->preprocessor->tokenize(s, len); ScopedPyObject tuple(PyTuple_New(result.size())); if (!tuple) return NULL; for (Py_ssize_t i = 0; i < (Py_ssize_t)result.size(); ++i) { Token *token = create_token(self, result[i]); if (!token) return NULL; PyTuple_SetItem(tuple, i, (PyObject*)token); } return tuple.release(); } catch (...) { set_python_exception(); return NULL; } Py_INCREF(Py_None); return Py_None; }
static PyObject* Preprocessor_next(Preprocessor* self, PyObject *args) { PyObject *expand = Py_True; if (!PyArg_ParseTuple(args, "|O:next", &expand)) return NULL; try { std::auto_ptr<cmonster::core::Token> token( self->preprocessor->next(PyObject_IsTrue(expand))); if (token.get()) { return (PyObject*)create_token(self, *token); } else { PyErr_SetString(PyExc_RuntimeError, "Internal error: Preprocessor returned NULL"); return NULL; } } catch (...) { set_python_exception(); return NULL; } }
static int8_t element_module(void *state, Message *msg) { element_state_t *s = (element_state_t *)state; switch (msg->type) { case MSG_INIT: { // Default value = 9 seconds. s->computation_time = 9; // Default: 3 LSBits s->bit_mask = 0x07; s->pid = msg->did; break; } case MSG_TIMER_TIMEOUT: { DEBUG("Truncate_long timer fired. Output the old token.\n"); token_type_t *my_token = create_token(&s->value, sizeof(uint8_t), s->pid); if (my_token == NULL) { SOS_CALL(s->signal_error, signal_error_func_t, -ENOMEM); return -EINVAL; } //SOS_CALL(s->put_token, put_token_func_t, s->output0, my_token); dispatch(s->output0, my_token); destroy_token(my_token); break; } case MSG_FINAL: { break; } default: return -EINVAL; } return SOS_OK; }
// The salt here is owner-supplied random bits, not the address hash. bool create_token(encrypted_token& out_token, const std::string& passphrase, const ek_entropy& entropy) { // BIP38: If lot and sequence numbers are not being included, then // owner_salt is 8 random bytes instead of 4, lot_sequence is omitted and // owner_entropy becomes an alias for owner_salt. const auto prefix = parse_encrypted_token::prefix_factory(false); return create_token(out_token, passphrase, entropy, entropy, prefix); }
static void cmdline_tokenize(Cmdline *cmdline) { char ch[] = {0,'\0'}; int st, ed, pos; char *str = cmdline->line; bool esc = false; st = ed = pos = 0; for (;;) { ch[0] = str[pos++]; if (ch[0] == '\0') { create_token(cmdline->tokens, str, st, ed, false); break; } else if (ch[0] == '\\' && !esc) { esc = true; continue; } else if ((ch[0] == '\"' || ch[0] == '\'') && !esc) { char *closech = strchrnul(&str[pos], ch[0]); if (closech[-1] != '\\') { int end = (closech - &str[pos]) + pos; bool quote = str[end] == '\0'; pos = quote ? pos-1 : pos; create_token(cmdline->tokens, str, pos, end, !quote); st = pos = quote ? end : ++end; } } else if (strpbrk(ch, TOKENCHARS) && !esc) { create_token(cmdline->tokens, str, st, ed, false); if (*ch != ' ') { create_token(cmdline->tokens, str, pos-1, pos, false); ed = pos; } st = pos; } else ed = pos; esc = false; } cmdline->cont = esc; }
t_result create_result_from_instru_token(t_file_reader *file, t_position pos, t_instruction_type inst) { t_token token; token = create_token(file, pos, TOKEN_TYPE_INSTRUCTION); token.content.instruction = inst; return (create_result_from_token(token)); }
void UUID::generate() { if( !gUUIDStateInitialized ) { create_uuid_state( &gUUIDState ); gUUIDStateInitialized = true; } create_token( &gUUIDState, ( xuuid_t* ) this ); }
int main(int argc, char **argv) { uuid_state state; uuid_t uuid; char output[1024]; create_uuid_state(&state); create_token(&state, &uuid); format_token(output, &uuid); printf("%s\n", output); }
// tokenizes the string str Ttoken *tokenize(char *str) { Ttoken *tok_list, *tok_tmp = NULL; char word[256]; int a, b, c; int i = 0; tok_list = create_token("head"); while(str[i] != '\0') { // skip leading whitespace while(str[i] == ' ' || str[i] == '\t') i++; if (str[i] == '"') { // find end of string part i ++; a = i; while(str[i] != '"') i++; b = i; i ++; } else { // find end of word a = i; while(str[i] != ' ' && str[i] != '\t' && str[i] != '\0') i++; b = i; } // copy string to word for(c=0;c<b-a;c++) word[c] = str[a+c]; word[c] = '\0'; // make token and put it in the list tok_tmp = create_token(word); insert_token(tok_list, tok_tmp); } tok_tmp->word[strlen(tok_tmp->word) - 1] = '\0'; tok_tmp = (Ttoken *)tok_list->next; destroy_token(tok_list); return tok_tmp; }
// The salt here is owner-supplied random bits, not the address hash. bool create_token(encrypted_token& out_token, const std::string& passphrase, const ek_salt& salt, uint32_t lot, uint32_t sequence) { if (lot > ek_max_lot || sequence > ek_max_sequence) return false; static constexpr size_t max_sequence_bits = 12; const uint32_t lot_sequence = (lot << max_sequence_bits) || sequence; const auto entropy = splice(salt, to_big_endian(lot_sequence)); const auto prefix = parse_encrypted_token::prefix_factory(true); create_token(out_token, passphrase, salt, entropy, prefix); return true; }
static int8_t input0 (func_cb_ptr p, token_type_t *t) { element_state_t *s = (element_state_t *)sys_get_state(); // Get token from port. // If want to break the chain of calls here, then copy the token into a private // data structure(global), and return appropriate value (SOS_OK). // Process input: Extract 3 LSB's and pass it on to the next function. // We need a separate place to hold the output as we are modifying the input. // Remember, this module does not own the input token, so should not // overwrite it. uint8_t out_value = (*((uint8_t *)get_token_data(t))) & s->bit_mask; token_type_t *my_token = create_token(&out_value, sizeof(uint8_t), s->pid); if (my_token == NULL) return -ENOMEM; //SOS_CALL(s->put_token, put_token_func_t, s->output0, my_token); dispatch(s->output0, my_token); destroy_token(my_token); return SOS_OK; }
static int8_t process_input(element_state_t *s, uint8_t data) { uint8_t out_value; // Get token from port. // If want to break the chain of calls here, then copy the token into a private // data structure(global), and return appropriate value (-EBUSY). switch (s->status) { case IDLE: { s->stored_input[0] = data; s->status = WAITING_FOR_1; return -EBUSY; } case WAITING_FOR_1: { s->stored_input[1] = data; s->status = WAITING_FOR_2; return -EBUSY; } case WAITING_FOR_2: { // Process input: Combine (OR) the three inputs and pass the result on to the next function. // We need a separate place to hold the output as we are modifying the input. // Remember, this module does not own the input token, so should not // overwrite it. out_value = data | s->stored_input[0] | s->stored_input[1]; DEBUG("COMBINE: Output - token %d.\n", out_value); token_type_t *my_token = create_token(&out_value, sizeof(uint8_t), s->pid); if (my_token == NULL) return -ENOMEM; //SOS_CALL(s->put_token, put_token_func_t, s->output0, my_token); dispatch(s->output0, my_token); s->status = IDLE; destroy_token(my_token); return SOS_OK; } default: return SOS_OK; } return SOS_OK; }
static int8_t element_module(void *state, Message *msg) { element_state_t *s = (element_state_t *)state; switch (msg->type) { case MSG_INIT: { LED_DBG(LED_RED_ON); s->cnt = 0; // Default parameter value = 5 sec. s->sample_rate_in_sec = 5; s->pid = msg->did; sys_timer_start(TIMER_TID, ((uint32_t)s->sample_rate_in_sec) * 1024L, TIMER_ONE_SHOT); break; } case MSG_TIMER_TIMEOUT: { LED_DBG(LED_RED_TOGGLE); s->cnt++; DEBUG("Timer fired. Put token %d on output port. Function CB output = 0x%x.\n", s->cnt, s->output0); token_type_t *my_token = create_token(&s->cnt, sizeof(uint8_t), s->pid); if (my_token == NULL) return -ENOMEM; //SOS_CALL(s->put_token, put_token_func_t, s->output0, my_token); dispatch(s->output0, my_token); destroy_token(my_token); sys_timer_start(TIMER_TID, ((uint32_t)s->sample_rate_in_sec) * 1024L, TIMER_ONE_SHOT); break; } case MSG_FINAL: { break; } default: return -EINVAL; } return SOS_OK; }
/** * Parse and construct lexer token * * Note: * Currently, lexer token doesn't fully correspond to Token, defined in ECMA-262, v5, 7.5. * For example, there is no new-line token type in the token definition of ECMA-262 v5. * * Note: * For Lexer alone, it is hard to find out a token is whether a regexp or a division. * Parser must set maybe_regexp to true if a regexp is expected. * Otherwise, a division is expected. * * @return constructed token */ static token lexer_parse_token (bool maybe_regexp, /**< read '/' as regexp? */ bool *out_is_preceed_by_new_lines_p, /**< out: is constructed token preceded by newlines? */ bool is_strict) /**< flag, indicating whether current code is in strict mode code */ { JERRY_ASSERT (is_token_parse_in_progress == false); *out_is_preceed_by_new_lines_p = lexer_skip_whitespace_and_comments (); ecma_char_t c = LA (0); /* ECMA-262 v5, 7.6, Identifier */ if (lexer_is_char_can_be_identifier_start (c)) { return lexer_parse_identifier_or_keyword (is_strict); } /* ECMA-262 v5, 7.8.3, Numeric literal */ if (lit_char_is_decimal_digit (c) || (c == LIT_CHAR_DOT && lit_char_is_decimal_digit (LA (1)))) { return lexer_parse_number (is_strict); } if (c == LIT_CHAR_NULL) { return create_token (TOK_EOF, 0); } if (c == LIT_CHAR_SINGLE_QUOTE || c == LIT_CHAR_DOUBLE_QUOTE) { return lexer_parse_string (); } if (c == LIT_CHAR_SLASH && maybe_regexp) { return lexer_parse_regexp (); } /* ECMA-262 v5, 7.7, Punctuator */ switch (c) { case LIT_CHAR_LEFT_BRACE: { RETURN_PUNC (TOK_OPEN_BRACE); break; } case LIT_CHAR_RIGHT_BRACE: { RETURN_PUNC (TOK_CLOSE_BRACE); break; } case LIT_CHAR_LEFT_PAREN: { RETURN_PUNC (TOK_OPEN_PAREN); break; } case LIT_CHAR_RIGHT_PAREN: { RETURN_PUNC (TOK_CLOSE_PAREN); break; } case LIT_CHAR_LEFT_SQUARE: { RETURN_PUNC (TOK_OPEN_SQUARE); break; } case LIT_CHAR_RIGHT_SQUARE: { RETURN_PUNC (TOK_CLOSE_SQUARE); break; } case LIT_CHAR_DOT: { RETURN_PUNC (TOK_DOT); break; } case LIT_CHAR_SEMICOLON: { RETURN_PUNC (TOK_SEMICOLON); break; } case LIT_CHAR_COMMA: { RETURN_PUNC (TOK_COMMA); break; } case LIT_CHAR_TILDE: { RETURN_PUNC (TOK_COMPL); break; } case LIT_CHAR_COLON: { RETURN_PUNC (TOK_COLON); break; } case LIT_CHAR_QUESTION: { RETURN_PUNC (TOK_QUERY); break; } case LIT_CHAR_ASTERISK: { IF_LA_IS (LIT_CHAR_EQUALS, TOK_MULT_EQ, TOK_MULT); break; } case LIT_CHAR_SLASH: { IF_LA_IS (LIT_CHAR_EQUALS, TOK_DIV_EQ, TOK_DIV); break; } case LIT_CHAR_CIRCUMFLEX: { IF_LA_IS (LIT_CHAR_EQUALS, TOK_XOR_EQ, TOK_XOR); break; } case LIT_CHAR_PERCENT: { IF_LA_IS (LIT_CHAR_EQUALS, TOK_MOD_EQ, TOK_MOD); break; } case LIT_CHAR_PLUS: { IF_LA_IS_OR (LIT_CHAR_PLUS, TOK_DOUBLE_PLUS, LIT_CHAR_EQUALS, TOK_PLUS_EQ, TOK_PLUS); break; } case LIT_CHAR_MINUS: { IF_LA_IS_OR (LIT_CHAR_MINUS, TOK_DOUBLE_MINUS, LIT_CHAR_EQUALS, TOK_MINUS_EQ, TOK_MINUS); break; } case LIT_CHAR_AMPERSAND: { IF_LA_IS_OR (LIT_CHAR_AMPERSAND, TOK_DOUBLE_AND, LIT_CHAR_EQUALS, TOK_AND_EQ, TOK_AND); break; } case LIT_CHAR_VLINE: { IF_LA_IS_OR (LIT_CHAR_VLINE, TOK_DOUBLE_OR, LIT_CHAR_EQUALS, TOK_OR_EQ, TOK_OR); break; } case LIT_CHAR_LESS_THAN: { switch (LA (1)) { case LIT_CHAR_LESS_THAN: IF_LA_N_IS (LIT_CHAR_EQUALS, TOK_LSHIFT_EQ, TOK_LSHIFT, 2); break; case LIT_CHAR_EQUALS: RETURN_PUNC_EX (TOK_LESS_EQ, 2); break; default: RETURN_PUNC (TOK_LESS); } break; } case LIT_CHAR_GREATER_THAN: { switch (LA (1)) { case LIT_CHAR_GREATER_THAN: { switch (LA (2)) { case LIT_CHAR_GREATER_THAN: IF_LA_N_IS (LIT_CHAR_EQUALS, TOK_RSHIFT_EX_EQ, TOK_RSHIFT_EX, 3); break; case LIT_CHAR_EQUALS: RETURN_PUNC_EX (TOK_RSHIFT_EQ, 3); break; default: RETURN_PUNC_EX (TOK_RSHIFT, 2); } break; } case LIT_CHAR_EQUALS: RETURN_PUNC_EX (TOK_GREATER_EQ, 2); break; default: RETURN_PUNC (TOK_GREATER); } break; } case LIT_CHAR_EQUALS: { if (LA (1) == LIT_CHAR_EQUALS) { IF_LA_N_IS (LIT_CHAR_EQUALS, TOK_TRIPLE_EQ, TOK_DOUBLE_EQ, 2); } else { RETURN_PUNC (TOK_EQ); } break; } case LIT_CHAR_EXCLAMATION: { if (LA (1) == LIT_CHAR_EQUALS) { IF_LA_N_IS (LIT_CHAR_EQUALS, TOK_NOT_DOUBLE_EQ, TOK_NOT_EQ, 2); } else { RETURN_PUNC (TOK_NOT); } break; } } PARSE_ERROR (JSP_EARLY_ERROR_SYNTAX, "Illegal character", lit_utf8_iterator_get_pos (&src_iter)); } /* lexer_parse_token */
void Tokenizer::handle_char(char new_ch) { // Checks for token ending conditions and buffer tasks for each possible state. switch (current_state) { case 0: // ----- DEFAULT STATE // Buffer is now ready for character buffer_ready = true; return; case 1: // ----- WORD // Close word token on any non-alphanumeric character if (!isalpha(new_ch) && !isnumber(new_ch)) { // Determine token type and create token std::string type = get_word_type(token_buffer); create_token(type); // Open buffer to read incoming character buffer_ready = true; } else { // Add incoming character to token buffer token_buffer+=new_ch; } break; case 2: // ----- NUMBER // Close number token on anything non-numerical if (!isnumber(new_ch)) { if (isalpha(new_ch)) { // ERROR // !! } else { // Create NUMBER token create_token("NUMBER"); // Open buffer to read incoming character buffer_ready = true; } } else { // Add incoming character to token buffer token_buffer+=new_ch; } break; case 3: // ----- STRING // Close string when a " is reached if (new_ch == '\"') { // Add last quotation mark to token token_buffer+=new_ch; // Keep buffer closed from handling incoming character current_state = 0; // Create STRING token create_token("STRING"); } else { // Add incoming character to token buffer token_buffer+=new_ch; } break; case 4: // ----- COMMENT // Close comment on a newline if (new_ch == '\r' || new_ch == '\n') { // Ready to read tokens again buffer_ready = true; } // Otherwise, do nothing until comment ends break; case 5: // ----- ! EXC // Could be a NOT or a RELOP if (new_ch == '=') { // Add incoming character to token buffer token_buffer+=new_ch; // Keep buffer closed from handling incoming character current_state = 0; // Create a RELOP token create_token("RELOP"); } else { // Create a NOT token create_token("NOT"); // Open buffer to read incoming character buffer_ready = true; } break; case 6: // ----- = EQUALS // Check for second '=' if (new_ch == '=') { // Add incoming character to token buffer token_buffer+=new_ch; // Keep buffer closed from handling incoming character current_state = 0; // Create RELOP token create_token("RELOP"); } else { // Without the second '=', it's an error create_token("ERROR"); // Open buffer to read incoming character buffer_ready = true; } break; case 7: // ----- > GT // Could be a > or a >= if (new_ch == '=') { // Add incoming character to token buffer token_buffer+=new_ch; // Keep buffer closed from handling incoming character current_state = 0; // Create RELOP token create_token("RELOP"); } else { // Create '>' token create_token("RELOP"); // Open buffer to read incoming character buffer_ready = true; } break; case 8: // ----- < LT // Could be a < or a <= or an ASSIGNOP if (new_ch == '-') { // Add incoming character to token buffer token_buffer+=new_ch; // Keep buffer closed from handling incoming character current_state = 0; // Create ASSIGNOP token create_token("ASSIGNOP"); } else if (new_ch == '=') { // Add incoming character to token buffer token_buffer+=new_ch; // Keep buffer closed from handling incoming character current_state = 0; // Create RELOP token create_token("RELOP"); } else { // Create '<' token create_token("RELOP"); // Open buffer to read incoming character buffer_ready = true; } break; case 10: // ----- Unambiguous single character // Create token immediately std::string type = get_op_type(token_buffer[0]); create_token(type); // Open buffer to read incoming character buffer_ready = true; break; } }
/** * Parse and construct lexer token * * Note: * Currently, lexer token doesn't fully correspond to Token, defined in ECMA-262, v5, 7.5. * For example, there is no new-line token type in the token definition of ECMA-262 v5. * * @return constructed token */ static token lexer_parse_token (void) { ecma_char_t c = LA (0); if (lit_char_is_white_space (c)) { while (lit_char_is_white_space (c)) { consume_char (); c = LA (0); } } if (lit_char_is_line_terminator (c)) { while (lit_char_is_line_terminator (c)) { consume_char (); c = LA (0); } return create_token (TOK_NEWLINE, 0); } JERRY_ASSERT (is_token_parse_in_progress == false); /* ECMA-262 v5, 7.6, Identifier */ if (lexer_is_char_can_be_identifier_start (c)) { return lexer_parse_identifier_or_keyword (); } /* ECMA-262 v5, 7.8.3, Numeric literal */ if (lit_char_is_decimal_digit (c) || (c == LIT_CHAR_DOT && lit_char_is_decimal_digit (LA (1)))) { return lexer_parse_number (); } if (c == LIT_CHAR_LF) { consume_char (); return create_token (TOK_NEWLINE, 0); } if (c == LIT_CHAR_NULL) { return create_token (TOK_EOF, 0); } if (c == LIT_CHAR_SINGLE_QUOTE || c == LIT_CHAR_DOUBLE_QUOTE) { return lexer_parse_string (); } /* ECMA-262 v5, 7.4, SingleLineComment or MultiLineComment */ if (c == LIT_CHAR_SLASH && (LA (1) == LIT_CHAR_SLASH || LA (1) == LIT_CHAR_ASTERISK)) { if (lexer_parse_comment ()) { return create_token (TOK_NEWLINE, 0); } else { return lexer_parse_token (); } } if (c == LIT_CHAR_SLASH && !(prev_non_lf_token.type == TOK_NAME || prev_non_lf_token.type == TOK_NULL || prev_non_lf_token.type == TOK_BOOL || prev_non_lf_token.type == TOK_CLOSE_BRACE || prev_non_lf_token.type == TOK_CLOSE_SQUARE || prev_non_lf_token.type == TOK_CLOSE_PAREN || prev_non_lf_token.type == TOK_SMALL_INT || prev_non_lf_token.type == TOK_NUMBER || prev_non_lf_token.type == TOK_STRING || prev_non_lf_token.type == TOK_REGEXP)) { return lexer_parse_regexp (); } /* ECMA-262 v5, 7.7, Punctuator */ switch (c) { case LIT_CHAR_LEFT_BRACE: { RETURN_PUNC (TOK_OPEN_BRACE); break; } case LIT_CHAR_RIGHT_BRACE: { RETURN_PUNC (TOK_CLOSE_BRACE); break; } case LIT_CHAR_LEFT_PAREN: { RETURN_PUNC (TOK_OPEN_PAREN); break; } case LIT_CHAR_RIGHT_PAREN: { RETURN_PUNC (TOK_CLOSE_PAREN); break; } case LIT_CHAR_LEFT_SQUARE: { RETURN_PUNC (TOK_OPEN_SQUARE); break; } case LIT_CHAR_RIGHT_SQUARE: { RETURN_PUNC (TOK_CLOSE_SQUARE); break; } case LIT_CHAR_DOT: { RETURN_PUNC (TOK_DOT); break; } case LIT_CHAR_SEMICOLON: { RETURN_PUNC (TOK_SEMICOLON); break; } case LIT_CHAR_COMMA: { RETURN_PUNC (TOK_COMMA); break; } case LIT_CHAR_TILDE: { RETURN_PUNC (TOK_COMPL); break; } case LIT_CHAR_COLON: { RETURN_PUNC (TOK_COLON); break; } case LIT_CHAR_QUESTION: { RETURN_PUNC (TOK_QUERY); break; } case LIT_CHAR_ASTERISK: { IF_LA_IS (LIT_CHAR_EQUALS, TOK_MULT_EQ, TOK_MULT); break; } case LIT_CHAR_SLASH: { IF_LA_IS (LIT_CHAR_EQUALS, TOK_DIV_EQ, TOK_DIV); break; } case LIT_CHAR_CIRCUMFLEX: { IF_LA_IS (LIT_CHAR_EQUALS, TOK_XOR_EQ, TOK_XOR); break; } case LIT_CHAR_PERCENT: { IF_LA_IS (LIT_CHAR_EQUALS, TOK_MOD_EQ, TOK_MOD); break; } case LIT_CHAR_PLUS: { IF_LA_IS_OR (LIT_CHAR_PLUS, TOK_DOUBLE_PLUS, LIT_CHAR_EQUALS, TOK_PLUS_EQ, TOK_PLUS); break; } case LIT_CHAR_MINUS: { IF_LA_IS_OR (LIT_CHAR_MINUS, TOK_DOUBLE_MINUS, LIT_CHAR_EQUALS, TOK_MINUS_EQ, TOK_MINUS); break; } case LIT_CHAR_AMPERSAND: { IF_LA_IS_OR (LIT_CHAR_AMPERSAND, TOK_DOUBLE_AND, LIT_CHAR_EQUALS, TOK_AND_EQ, TOK_AND); break; } case LIT_CHAR_VLINE: { IF_LA_IS_OR (LIT_CHAR_VLINE, TOK_DOUBLE_OR, LIT_CHAR_EQUALS, TOK_OR_EQ, TOK_OR); break; } case LIT_CHAR_LESS_THAN: { switch (LA (1)) { case LIT_CHAR_LESS_THAN: IF_LA_N_IS (LIT_CHAR_EQUALS, TOK_LSHIFT_EQ, TOK_LSHIFT, 2); break; case LIT_CHAR_EQUALS: RETURN_PUNC_EX (TOK_LESS_EQ, 2); break; default: RETURN_PUNC (TOK_LESS); } break; } case LIT_CHAR_GREATER_THAN: { switch (LA (1)) { case LIT_CHAR_GREATER_THAN: { switch (LA (2)) { case LIT_CHAR_GREATER_THAN: IF_LA_N_IS (LIT_CHAR_EQUALS, TOK_RSHIFT_EX_EQ, TOK_RSHIFT_EX, 3); break; case LIT_CHAR_EQUALS: RETURN_PUNC_EX (TOK_RSHIFT_EQ, 3); break; default: RETURN_PUNC_EX (TOK_RSHIFT, 2); } break; } case LIT_CHAR_EQUALS: RETURN_PUNC_EX (TOK_GREATER_EQ, 2); break; default: RETURN_PUNC (TOK_GREATER); } break; } case LIT_CHAR_EQUALS: { if (LA (1) == LIT_CHAR_EQUALS) { IF_LA_N_IS (LIT_CHAR_EQUALS, TOK_TRIPLE_EQ, TOK_DOUBLE_EQ, 2); } else { RETURN_PUNC (TOK_EQ); } break; } case LIT_CHAR_EXCLAMATION: { if (LA (1) == LIT_CHAR_EQUALS) { IF_LA_N_IS (LIT_CHAR_EQUALS, TOK_NOT_DOUBLE_EQ, TOK_NOT_EQ, 2); } else { RETURN_PUNC (TOK_NOT); } break; } } PARSE_ERROR (JSP_EARLY_ERROR_SYNTAX, "Illegal character", lit_utf8_iterator_get_pos (&src_iter)); } /* lexer_parse_token */
/* In this function we cannot use strtol function since there is no octal literals in ECMAscript. */ static token parse_number (void) { ecma_char_t c = LA (0); bool is_hex = false; bool is_fp = false; bool is_exp = false; bool is_overflow = false; ecma_number_t fp_res = .0; size_t tok_length = 0, i; uint32_t res = 0; token known_token; JERRY_ASSERT (isdigit (c) || c == '.'); if (c == '0') { if (LA (1) == 'x' || LA (1) == 'X') { is_hex = true; } } if (c == '.') { JERRY_ASSERT (!isalpha (LA (1))); is_fp = true; } if (is_hex) { // Eat up '0x' consume_char (); consume_char (); new_token (); while (true) { c = LA (0); if (!isxdigit (c)) { break; } consume_char (); } if (isalpha (c) || c == '_' || c == '$') { PARSE_ERROR ("Integer literal shall not contain non-digit characters", buffer - buffer_start); } tok_length = (size_t) (buffer - token_start); for (i = 0; i < tok_length; i++) { if (!is_overflow) { res = (res << 4) + ecma_char_hex_to_int (token_start[i]); } else { fp_res = fp_res * 16 + (ecma_number_t) ecma_char_hex_to_int (token_start[i]); } if (res > 255) { fp_res = (ecma_number_t) res; is_overflow = true; res = 0; } } if (is_overflow) { known_token = convert_seen_num_to_token (fp_res); token_start = NULL; return known_token; } else { known_token = create_token (TOK_SMALL_INT, (uint8_t) res); token_start = NULL; return known_token; } } JERRY_ASSERT (!is_hex && !is_exp); new_token (); // Eat up '.' if (is_fp) { consume_char (); } while (true) { c = LA (0); if (is_fp && c == '.') { FIXME (/* This is wrong: 1..toString (). */) PARSE_ERROR ("Integer literal shall not contain more than one dot character", buffer - buffer_start); } if (is_exp && (c == 'e' || c == 'E')) { PARSE_ERROR ("Integer literal shall not contain more than exponential marker ('e' or 'E')", buffer - buffer_start); } if (c == '.') { if (isalpha (LA (1)) || LA (1) == '_' || LA (1) == '$') { PARSE_ERROR ("Integer literal shall not contain non-digit character after got character", buffer - buffer_start); } is_fp = true; consume_char (); continue; } if (c == 'e' || c == 'E') { if (LA (1) == '-' || LA (1) == '+') { consume_char (); } if (!isdigit (LA (1))) { PARSE_ERROR ("Integer literal shall not contain non-digit character after exponential marker ('e' or 'E')", buffer - buffer_start); } is_exp = true; consume_char (); continue; } if (isalpha (c) || c == '_' || c == '$') { PARSE_ERROR ("Integer literal shall not contain non-digit characters", buffer - buffer_start); } if (!isdigit (c)) { break; } consume_char (); } tok_length = (size_t) (buffer - token_start); if (is_fp || is_exp) { ecma_number_t res = ecma_utf8_string_to_number (token_start, (jerry_api_size_t) tok_length); JERRY_ASSERT (!ecma_number_is_nan (res)); known_token = convert_seen_num_to_token (res); token_start = NULL; return known_token; } if (*token_start == '0' && tok_length != 1) { if (strict_mode) { PARSE_ERROR ("Octal tnteger literals are not allowed in strict mode", token_start - buffer_start); } for (i = 0; i < tok_length; i++) { if (!is_overflow) { res = res * 8 + ecma_char_hex_to_int (token_start[i]); } else { fp_res = fp_res * 8 + (ecma_number_t) ecma_char_hex_to_int (token_start[i]); } if (res > 255) { fp_res = (ecma_number_t) res; is_overflow = true; res = 0; } } } else { for (i = 0; i < tok_length; i++) { if (!is_overflow) { res = res * 10 + ecma_char_hex_to_int (token_start[i]); } else { fp_res = fp_res * 10 + (ecma_number_t) ecma_char_hex_to_int (token_start[i]); } if (res > 255) { fp_res = (ecma_number_t) res; is_overflow = true; res = 0; } } } if (is_overflow) { known_token = convert_seen_num_to_token (fp_res); token_start = NULL; return known_token; } else { known_token = create_token (TOK_SMALL_INT, (uint8_t) res); token_start = NULL; return known_token; } }
int tokenize(struct token_list* tk_list, char* file_buffer) { enum Status status; line_num; size_t token_begin, token_end; token_begin = 0, token_end = 0; status = STATUS_INVALID; str_toupper(file_buffer); /* * Careful: it seems an error to let "i <= len", * but we need one more execution to flush the last token into token list. */ size_t line_num = 1; for (size_t i = 0; ; ++i) { struct token_node* tok_node; switch (status) { case STATUS_LETTER: if (!IS_LETTER(file_buffer[i]) && !IS_DIGIT(file_buffer[i])) { token_end = i; tok_node = create_token(TOKEN_LABEL, file_buffer + token_begin, token_end - token_begin); tok_node->type = letter_type(tok_node->liter, tok_node->len); token_append(tk_list, tok_node); token_begin = i; status = next_status(status, file_buffer[i]); } break; case STATUS_PRAGMA: if (!IS_LETTER(file_buffer[i])) { int type; token_end = i; type = pragma_type(file_buffer + token_begin, token_end - token_begin); if (type < 0) { error("invalid pragma ad line %d\n", line_num); return -4; } tok_node = create_token(type, file_buffer + token_begin, token_end - token_begin); token_append(tk_list, tok_node); token_begin = i; status = next_status(status, file_buffer[i]); } break; case STATUS_PUNCTUATION: token_end = i; tok_node = create_token(file_buffer[token_begin], file_buffer + token_begin, token_end - token_begin); token_append(tk_list, tok_node); token_begin = i; status = next_status(status, file_buffer[i]); break; case STATUS_NUMBER: if (!IS_NUMBER(file_buffer[i])) { token_end = i; if (!check_number(file_buffer + token_begin, token_end - token_begin)) { error("invalid number format at line %d\n", line_num); return -2; } tok_node = create_token(TOKEN_NUMBER, file_buffer + token_begin, token_end - token_begin); tok_node->data = parse_number(tok_node->liter); token_append(tk_list, tok_node); token_begin = i; status = next_status(status, file_buffer[i]); } break; case STATUS_BLANK: if (!IS_BLANK(file_buffer[i])) { token_begin = i; status = next_status(status, file_buffer[i]); } break; case STATUS_COMMENTS: //once status is in comments, it will always be in comments if ('\n' == file_buffer[i]) { token_begin = i; status = next_status(status, file_buffer[i]); } break; case STATUS_INVALID: token_begin = i; status = next_status(status, file_buffer[i]); if (STATUS_INVALID == status && 0 != file_buffer[i]) { error("invalid format at line %d\n", line_num); return -3; } break; } if (0 == file_buffer[i]) break; else if ('\n' == file_buffer[i]) ++line_num; } return 0; }
token get_token() { char c = nextchar(); while (c == ' ' || c == '\n') c = nextchar(); switch(c) { case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': case 'y': case 'z': case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': case '_': { char temp_string[32]; int i = 0; while (is_string_character(c)) { temp_string[i] = c; i += 1; c = nextchar(); } backchar(); temp_string[i] = 0; char * s = strdup(temp_string); return create_string_token(s, 0); } case '\"': { char temp_string[32]; c = nextchar(); int pos = 0; while (pos < 31 && c != '\"') { temp_string[pos] = c; pos += 1; c = nextchar(); } temp_string[pos] = 0; if (pos == 31) { error_message("max string length is 31 characters\n"); exit(1); } char * s = strdup(temp_string); return create_string_token(s, 1); } case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { int value = c - '0'; c = nextchar(); while (c >= '0' && c <= '9') { value = value * 10 + c - '0'; c = nextchar(); } backchar(); return create_token(integer_constant, value); } case '+': case '-': case '*': return create_binary_operator_token(c); case '/': { c = nextchar(); if (c == ' ') return create_binary_operator_token(c); if (c == '/') { current_line->character_number = current_line->length; return get_token(); } else { error_message("unrecognized use of '/' character\n"); exit(1); } } case '=': { c = nextchar(); if (c == ' ') { error_message("for initialization and assignment use :="); exit(1); } else if (c == '=') return create_token(comparison_operator, equal); } case '!': { c = nextchar(); if (c == '=') return create_token(comparison_operator, not_equal); else { error_message("undefined use of '!' character"); exit(1); } } case '<': { c = nextchar(); if (c == '=') return create_token(comparison_operator, less_or_equal); backchar(); return create_token(comparison_operator, less_than); } case '>': { c = nextchar(); if (c == '=') return create_token(comparison_operator, greater_or_equal); backchar(); return create_token(comparison_operator, greater_than); } case ':': { c = nextchar(); if (c != '=') { error_message("did you mean :=?"); exit(1); } else return create_token(assignment, 0); } case '(': return create_token(open_parenthesis, 0); case ')': return create_token(close_parenthesis, 0); case '{': return create_token(open_curly_bracket, 0); case '}': return create_token(close_curly_bracket, 0); case ';': return create_token(semi_colon, 0); case(0): return create_token(end_of_file, 0); //todo: return error token default: { error_message(""); printf("character %c unrecognized\n", c); exit(1); } } }
CK_RV parse_readers(ykpiv_state *state, const CK_BYTE_PTR readers, const CK_ULONG len, ykcs11_slot_t *slots, CK_ULONG_PTR n_slots, CK_ULONG_PTR n_with_token) { CK_BYTE i; CK_BYTE_PTR p; CK_BYTE_PTR s; CK_ULONG l; slot_vendor_t slot; *n_slots = 0; *n_with_token = 0; p = readers; /* * According to pcsc-lite, the format of a reader name is: * name [interface] (serial) index slot * http://ludovicrousseau.blogspot.se/2010/05/what-is-in-pcsc-reader-name.html */ for (i = 0; i < len; i++) if (readers[i] == '\0' && i != len - 1) { slots[*n_slots].vid = get_vendor_id(p); if (slots[*n_slots].vid == UNKNOWN) { // TODO: distinguish between tokenless and unsupported? // Unknown slot, just save what info we have memset(&slots[*n_slots].info, 0, sizeof(CK_SLOT_INFO)); memset(slots[*n_slots].info.slotDescription, ' ', sizeof(slots[*n_slots].info.slotDescription)); if (strlen(p) <= sizeof(slots[*n_slots].info.slotDescription)) strncpy(slots[*n_slots].info.slotDescription, p, strlen(p)); else strncpy(slots[*n_slots].info.slotDescription, p, sizeof(slots[*n_slots].info.slotDescription)); } else { // Supported slot slot = get_slot_vendor(slots[*n_slots].vid); // Values must NOT be null terminated and ' ' padded memset(slots[*n_slots].info.slotDescription, ' ', sizeof(slots[*n_slots].info.slotDescription)); s = slots[*n_slots].info.slotDescription; l = sizeof(slots[*n_slots].info.slotDescription); strncpy((char *)s, (char*)p, l); memset(slots[*n_slots].info.manufacturerID, ' ', sizeof(slots[*n_slots].info.manufacturerID)); s = slots[*n_slots].info.manufacturerID; l = sizeof(slots[*n_slots].info.manufacturerID); if(slot.get_slot_manufacturer(s, l) != CKR_OK) goto failure; if (slot.get_slot_flags(&slots[*n_slots].info.flags) != CKR_OK) goto failure; // Treating hw and fw version the same if (slot.get_slot_version(&slots[*n_slots].info.hardwareVersion) != CKR_OK) goto failure; if (slot.get_slot_version(&slots[*n_slots].info.firmwareVersion) != CKR_OK) goto failure; if (has_token(slots + *n_slots)) { // Save token information (*n_with_token)++; if (create_token(state, p, slots + *n_slots) != CKR_OK) goto failure; } } (*n_slots)++; p += i + 1; } return CKR_OK; failure: // TODO: destroy all token objects for (i = 0; i < *n_slots; i++) if (has_token(slots + i)) destroy_token(slots + i); return CKR_FUNCTION_FAILED; }
/** * Parse numeric literal (ECMA-262, v5, 7.8.3) * * @return token of TOK_SMALL_INT or TOK_NUMBER types */ static token lexer_parse_number (void) { ecma_char_t c = LA (0); bool is_hex = false; bool is_fp = false; ecma_number_t fp_res = .0; size_t tok_length = 0, i; token known_token; JERRY_ASSERT (lit_char_is_decimal_digit (c) || c == LIT_CHAR_DOT); if (c == LIT_CHAR_0) { if (LA (1) == LIT_CHAR_LOWERCASE_X || LA (1) == LIT_CHAR_UPPERCASE_X) { is_hex = true; } } else if (c == LIT_CHAR_DOT) { JERRY_ASSERT (lit_char_is_decimal_digit (LA (1))); is_fp = true; } if (is_hex) { // Eat up '0x' consume_char (); consume_char (); new_token (); c = LA (0); if (!lit_char_is_hex_digit (c)) { PARSE_ERROR (JSP_EARLY_ERROR_SYNTAX, "Invalid HexIntegerLiteral", lit_utf8_iterator_get_pos (&src_iter)); } do { consume_char (); c = LA (0); } while (lit_char_is_hex_digit (c)); if (lexer_is_char_can_be_identifier_start (c)) { PARSE_ERROR (JSP_EARLY_ERROR_SYNTAX, "Identifier just after integer literal", lit_utf8_iterator_get_pos (&src_iter)); } tok_length = (size_t) (TOK_SIZE ()); const lit_utf8_byte_t *fp_buf_p = TOK_START (); /* token is constructed at end of function */ for (i = 0; i < tok_length; i++) { fp_res = fp_res * 16 + (ecma_number_t) lit_char_hex_to_int (fp_buf_p[i]); } } else { bool is_exp = false; new_token (); // Eat up '.' if (is_fp) { consume_char (); } while (true) { c = LA (0); if (c == LIT_CHAR_DOT) { if (is_fp) { /* token is constructed at end of function */ break; } else { is_fp = true; consume_char (); continue; } } else if (c == LIT_CHAR_LOWERCASE_E || c == LIT_CHAR_UPPERCASE_E) { if (is_exp) { PARSE_ERROR (JSP_EARLY_ERROR_SYNTAX, "Numeric literal shall not contain more than exponential marker ('e' or 'E')", lit_utf8_iterator_get_pos (&src_iter)); } else { is_exp = true; consume_char (); if (LA (0) == LIT_CHAR_MINUS || LA (0) == LIT_CHAR_PLUS) { consume_char (); } continue; } } else if (!lit_char_is_decimal_digit (c)) { if (lexer_is_char_can_be_identifier_start (c)) { PARSE_ERROR (JSP_EARLY_ERROR_SYNTAX, "Numeric literal shall not contain non-numeric characters", lit_utf8_iterator_get_pos (&src_iter)); } /* token is constructed at end of function */ break; } consume_char (); } tok_length = (size_t) (TOK_SIZE ()); if (is_fp || is_exp) { ecma_number_t res = ecma_utf8_string_to_number (TOK_START (), (jerry_api_size_t) tok_length); JERRY_ASSERT (!ecma_number_is_nan (res)); known_token = convert_seen_num_to_token (res); is_token_parse_in_progress = NULL; return known_token; } else if (*TOK_START () == LIT_CHAR_0 && tok_length != 1) { /* Octal integer literals */ if (strict_mode) { PARSE_ERROR (JSP_EARLY_ERROR_SYNTAX, "Octal integer literals are not allowed in strict mode", token_start_pos); } else { /* token is constructed at end of function */ const lit_utf8_byte_t *fp_buf_p = TOK_START (); for (i = 0; i < tok_length; i++) { fp_res = fp_res * 8 + (ecma_number_t) lit_char_hex_to_int (fp_buf_p[i]); } } } else { const lit_utf8_byte_t *fp_buf_p = TOK_START (); /* token is constructed at end of function */ ecma_number_t mult = 1.0f; for (i = tok_length; i > 0; i--, mult *= 10) { fp_res += (ecma_number_t) lit_char_hex_to_int (fp_buf_p[i - 1]) * mult; } } } if (fp_res >= 0 && fp_res <= 255 && (uint8_t) fp_res == fp_res) { known_token = create_token (TOK_SMALL_INT, (uint8_t) fp_res); is_token_parse_in_progress = NULL; return known_token; } else { known_token = convert_seen_num_to_token (fp_res); is_token_parse_in_progress = NULL; return known_token; } } /* lexer_parse_number */
void st_crtToken(State s, register Ptt type) {st_setToken(s, create_token(s->flux, type));}
Token *Lexer::scan() { #define YYCTYPE char #define YYCURSOR (this->cursor) #define YYLIMIT (this->lim) #define YYMARKER (this->marker) #define YYFILL(n) {fill();} for(;;) { capture_begin(); #line 22 "<stdout>" { YYCTYPE yych; if ((YYLIMIT - YYCURSOR) < 5) YYFILL(5); yych = *YYCURSOR; switch (yych) { case 0x00: goto yy26; case '\t': case '\f': case ' ': goto yy30; case '\n': case '\r': goto yy28; case '(': goto yy22; case ')': goto yy24; case '*': goto yy18; case '+': goto yy14; case ',': goto yy10; case '-': goto yy16; case '.': goto yy8; case '/': goto yy20; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': goto yy6; case ':': goto yy12; case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': case 'a': case 'b': case 'c': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': case 'y': case 'z': goto yy5; case 'd': goto yy2; case 'p': goto yy4; default: goto yy32; } yy2: ++YYCURSOR; switch ((yych = *YYCURSOR)) { case 'e': goto yy46; default: goto yy41; } yy3: #line 31 "LexerX.re" { return create_name_token(capture_string()); } #line 117 "<stdout>" yy4: yych = *++YYCURSOR; switch (yych) { case 'a': goto yy42; default: goto yy41; } yy5: yych = *++YYCURSOR; goto yy41; yy6: yych = *(YYMARKER = ++YYCURSOR); goto yy39; yy7: #line 33 "LexerX.re" { return create_int_token(capture_int()); } #line 133 "<stdout>" yy8: ++YYCURSOR; switch ((yych = *YYCURSOR)) { case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': goto yy33; default: goto yy9; } yy9: #line 59 "LexerX.re" { printf( "Unrecognized character: %c\n", *(this->ccursor) ); continue; } #line 155 "<stdout>" yy10: ++YYCURSOR; #line 37 "LexerX.re" { return create_token(TOKEN_COMMA); } #line 160 "<stdout>" yy12: ++YYCURSOR; #line 39 "LexerX.re" { return create_token(TOKEN_COLON); } #line 165 "<stdout>" yy14: ++YYCURSOR; #line 41 "LexerX.re" { return create_token(TOKEN_ADD); } #line 170 "<stdout>" yy16: ++YYCURSOR; #line 43 "LexerX.re" { return create_token(TOKEN_SUB); } #line 175 "<stdout>" yy18: ++YYCURSOR; #line 45 "LexerX.re" { return create_token(TOKEN_MUL); } #line 180 "<stdout>" yy20: ++YYCURSOR; #line 47 "LexerX.re" { return create_token(TOKEN_DIV); } #line 185 "<stdout>" yy22: ++YYCURSOR; #line 49 "LexerX.re" { return create_token(TOKEN_LPAREN); } #line 190 "<stdout>" yy24: ++YYCURSOR; #line 51 "LexerX.re" { return create_token(TOKEN_RPAREN); } #line 195 "<stdout>" yy26: ++YYCURSOR; #line 53 "LexerX.re" { return create_token(TOKEN_EOF); } #line 200 "<stdout>" yy28: ++YYCURSOR; #line 55 "LexerX.re" { return emit_newline(); } #line 205 "<stdout>" yy30: ++YYCURSOR; #line 57 "LexerX.re" { continue; } #line 210 "<stdout>" yy32: yych = *++YYCURSOR; goto yy9; yy33: ++YYCURSOR; if (YYLIMIT <= YYCURSOR) YYFILL(1); yych = *YYCURSOR; switch (yych) { case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': goto yy33; default: goto yy35; } yy35: #line 35 "LexerX.re" { return create_float_token(capture_float()); } #line 234 "<stdout>" yy36: yych = *++YYCURSOR; switch (yych) { case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': goto yy33; default: goto yy37; } yy37: YYCURSOR = YYMARKER; goto yy7; yy38: YYMARKER = ++YYCURSOR; if ((YYLIMIT - YYCURSOR) < 2) YYFILL(2); yych = *YYCURSOR; yy39: switch (yych) { case '.': goto yy36; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': goto yy38; default: goto yy7; } yy40: ++YYCURSOR; if (YYLIMIT <= YYCURSOR) YYFILL(1); yych = *YYCURSOR; yy41: switch (yych) { case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': case 'y': case 'z': goto yy40; default: goto yy3; } yy42: yych = *++YYCURSOR; switch (yych) { case 's': goto yy43; default: goto yy41; } yy43: yych = *++YYCURSOR; switch (yych) { case 's': goto yy44; default: goto yy41; } yy44: ++YYCURSOR; switch ((yych = *YYCURSOR)) { case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': case 'y': case 'z': goto yy40; default: goto yy45; } yy45: #line 29 "LexerX.re" { return create_token(TOKEN_PASS); } #line 404 "<stdout>" yy46: yych = *++YYCURSOR; switch (yych) { case 'f': goto yy47; default: goto yy41; } yy47: ++YYCURSOR; switch ((yych = *YYCURSOR)) { case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': case 'y': case 'z': goto yy40; default: goto yy48; } yy48: #line 27 "LexerX.re" { return create_token(TOKEN_DEF); } #line 471 "<stdout>" } #line 63 "LexerX.re" } }
t_point parse_pipe(Hash *h, int level) { char chars[MAX_NAME_LENGTH]; char *c = chars; int whitespaces = 1; // cetli jsme whitespace? int first = 1; int is_def = 0; int is_close_tag = 0; Cons all = {NIL, NIL}; Cons *l = &all; t_point s = NIL; while ((c - chars) < MAX_NAME_LENGTH && (*c = read_char()) != EOF) { if (is_whitespace(*c) || (is_close_tag = (*c == CLOSE_TAG))) { if (whitespaces && !is_close_tag) continue; *c = '\0'; whitespaces = 1; if (first && ((is_def = (strcmp(chars, "def") == 0)) || (strcmp(chars, "lambda") == 0))) { if (is_close_tag) ERROR(SYNTAX_ERROR); if (is_def) { // is def if (!read_word(chars, 0)) ERROR(SYNTAX_ERROR); init_def(h, chars, level); } else { // is lambda return kontext_params(init_def(h, NULL, level), level); } break; } // c is not empy if (c != chars) { l->b = create_token(h, chars, level); if (first) { l->a = l->b; l->b = NIL; } else { l->b = pnew_List(l->b); l = next(l); } } if (is_close_tag) break; c = chars; first = 0; continue; } // nacitame dal if (!whitespaces) { c++; continue; } switch (*c) { case '\'': s = parse_char(); break; case '"': s = parse_string(); break; case OPEN_TAG: s = parse_pipe(h, level); break; default: c++; whitespaces = 0; continue; } if (s != NIL) { if (first) { l->a = s; l->b = NIL; first = 0; } else { l->b = pnew_List(s); l = next(l); } s = NIL; } } if ((c - chars) >= MAX_NAME_LENGTH) ERROR(SYNTAX_ERROR); if (all.b == NIL && !is_Func(all.a)) return all.a; else return pnew_Thunk(all.a, get_Cons(all.b)); }
static token lexer_next_token_private (void) { ecma_char_t c = LA (0); JERRY_ASSERT (token_start == NULL); if (isalpha (c) || c == '$' || c == '_') { return parse_name (); } if (isdigit (c) || (c == '.' && isdigit (LA (1)))) { return parse_number (); } if (c == '\n') { consume_char (); return create_token (TOK_NEWLINE, 0); } if (c == '\0') { return create_token (TOK_EOF, 0); } if (c == '\'' || c == '"') { return parse_string (); } if (isspace (c)) { grobble_whitespaces (); return lexer_next_token_private (); } if (c == '/' && LA (1) == '*') { if (replace_comment_by_newline ()) { token ret; ret.type = TOK_NEWLINE; ret.uid = 0; return ret; } else { return lexer_next_token_private (); } } if (c == '/') { if (LA (1) == '/') { replace_comment_by_newline (); return lexer_next_token_private (); } else if (!(sent_token.type == TOK_NAME || sent_token.type == TOK_NULL || sent_token.type == TOK_BOOL || sent_token.type == TOK_CLOSE_BRACE || sent_token.type == TOK_CLOSE_SQUARE || sent_token.type == TOK_CLOSE_PAREN || sent_token.type == TOK_SMALL_INT || sent_token.type == TOK_NUMBER || sent_token.type == TOK_STRING || sent_token.type == TOK_REGEXP)) { return parse_regexp (); } } switch (c) { case '{': RETURN_PUNC (TOK_OPEN_BRACE); break; case '}': RETURN_PUNC (TOK_CLOSE_BRACE); break; case '(': RETURN_PUNC (TOK_OPEN_PAREN); break; case ')': RETURN_PUNC (TOK_CLOSE_PAREN); break; case '[': RETURN_PUNC (TOK_OPEN_SQUARE); break; case ']': RETURN_PUNC (TOK_CLOSE_SQUARE); break; case '.': RETURN_PUNC (TOK_DOT); break; case ';': RETURN_PUNC (TOK_SEMICOLON); break; case ',': RETURN_PUNC (TOK_COMMA); break; case '~': RETURN_PUNC (TOK_COMPL); break; case ':': RETURN_PUNC (TOK_COLON); break; case '?': RETURN_PUNC (TOK_QUERY); break; case '*': IF_LA_IS ('=', TOK_MULT_EQ, TOK_MULT); break; case '/': IF_LA_IS ('=', TOK_DIV_EQ, TOK_DIV); break; case '^': IF_LA_IS ('=', TOK_XOR_EQ, TOK_XOR); break; case '%': IF_LA_IS ('=', TOK_MOD_EQ, TOK_MOD); break; case '+': IF_LA_IS_OR ('+', TOK_DOUBLE_PLUS, '=', TOK_PLUS_EQ, TOK_PLUS); break; case '-': IF_LA_IS_OR ('-', TOK_DOUBLE_MINUS, '=', TOK_MINUS_EQ, TOK_MINUS); break; case '&': IF_LA_IS_OR ('&', TOK_DOUBLE_AND, '=', TOK_AND_EQ, TOK_AND); break; case '|': IF_LA_IS_OR ('|', TOK_DOUBLE_OR, '=', TOK_OR_EQ, TOK_OR); break; case '<': { switch (LA (1)) { case '<': IF_LA_N_IS ('=', TOK_LSHIFT_EQ, TOK_LSHIFT, 2); break; case '=': RETURN_PUNC_EX (TOK_LESS_EQ, 2); break; default: RETURN_PUNC (TOK_LESS); } break; } case '>': { switch (LA (1)) { case '>': { switch (LA (2)) { case '>': IF_LA_N_IS ('=', TOK_RSHIFT_EX_EQ, TOK_RSHIFT_EX, 3); break; case '=': RETURN_PUNC_EX (TOK_RSHIFT_EQ, 3); break; default: RETURN_PUNC_EX (TOK_RSHIFT, 2); } break; } case '=': RETURN_PUNC_EX (TOK_GREATER_EQ, 2); break; default: RETURN_PUNC (TOK_GREATER); } break; } case '=': { if (LA (1) == '=') { IF_LA_N_IS ('=', TOK_TRIPLE_EQ, TOK_DOUBLE_EQ, 2); } else { RETURN_PUNC (TOK_EQ); } break; } case '!': { if (LA (1) == '=') { IF_LA_N_IS ('=', TOK_NOT_DOUBLE_EQ, TOK_NOT_EQ, 2); } else { RETURN_PUNC (TOK_NOT); } break; } default: PARSE_SORRY ("Unknown character", buffer - buffer_start); } PARSE_SORRY ("Unknown character", buffer - buffer_start); }
/** * Try to decode specified string as keyword * * @return if specified string represents a keyword, return corresponding keyword token, * else if it is 'null' - return TOK_NULL token, * else if it is 'true' or 'false' - return TOK_BOOL with corresponding boolean value, * else - return empty_token. */ static token decode_keyword (const lit_utf8_byte_t *str_p, /**< characters buffer */ lit_utf8_size_t str_size) /**< string's length */ { typedef struct { const char *keyword_p; keyword keyword_id; } kw_descr_t; const kw_descr_t keywords[] = { #define KW_DESCR(literal, keyword_id) { literal, keyword_id } KW_DESCR ("break", KW_BREAK), KW_DESCR ("case", KW_CASE), KW_DESCR ("catch", KW_CATCH), KW_DESCR ("class", KW_CLASS), KW_DESCR ("const", KW_CONST), KW_DESCR ("continue", KW_CONTINUE), KW_DESCR ("debugger", KW_DEBUGGER), KW_DESCR ("default", KW_DEFAULT), KW_DESCR ("delete", KW_DELETE), KW_DESCR ("do", KW_DO), KW_DESCR ("else", KW_ELSE), KW_DESCR ("enum", KW_ENUM), KW_DESCR ("export", KW_EXPORT), KW_DESCR ("extends", KW_EXTENDS), KW_DESCR ("finally", KW_FINALLY), KW_DESCR ("for", KW_FOR), KW_DESCR ("function", KW_FUNCTION), KW_DESCR ("if", KW_IF), KW_DESCR ("in", KW_IN), KW_DESCR ("instanceof", KW_INSTANCEOF), KW_DESCR ("interface", KW_INTERFACE), KW_DESCR ("import", KW_IMPORT), KW_DESCR ("implements", KW_IMPLEMENTS), KW_DESCR ("let", KW_LET), KW_DESCR ("new", KW_NEW), KW_DESCR ("package", KW_PACKAGE), KW_DESCR ("private", KW_PRIVATE), KW_DESCR ("protected", KW_PROTECTED), KW_DESCR ("public", KW_PUBLIC), KW_DESCR ("return", KW_RETURN), KW_DESCR ("static", KW_STATIC), KW_DESCR ("super", KW_SUPER), KW_DESCR ("switch", KW_SWITCH), KW_DESCR ("this", KW_THIS), KW_DESCR ("throw", KW_THROW), KW_DESCR ("try", KW_TRY), KW_DESCR ("typeof", KW_TYPEOF), KW_DESCR ("var", KW_VAR), KW_DESCR ("void", KW_VOID), KW_DESCR ("while", KW_WHILE), KW_DESCR ("with", KW_WITH), KW_DESCR ("yield", KW_YIELD) #undef KW_DESCR }; keyword kw = KW_NONE; for (uint32_t i = 0; i < sizeof (keywords) / sizeof (kw_descr_t); i++) { if (lit_compare_utf8_strings (str_p, str_size, (lit_utf8_byte_t *) keywords[i].keyword_p, (lit_utf8_size_t) strlen (keywords[i].keyword_p))) { kw = keywords[i].keyword_id; break; } } if (!strict_mode) { switch (kw) { case KW_INTERFACE: case KW_IMPLEMENTS: case KW_LET: case KW_PACKAGE: case KW_PRIVATE: case KW_PROTECTED: case KW_PUBLIC: case KW_STATIC: case KW_YIELD: { return convert_string_to_token (TOK_NAME, str_p, (ecma_length_t) str_size); } default: { break; } } } if (kw != KW_NONE) { return create_token (TOK_KEYWORD, kw); } else { if (lit_compare_utf8_string_and_magic_string (str_p, str_size, LIT_MAGIC_STRING_FALSE)) { return create_token (TOK_BOOL, false); } else if (lit_compare_utf8_string_and_magic_string (str_p, str_size, LIT_MAGIC_STRING_TRUE)) { return create_token (TOK_BOOL, true); } else if (lit_compare_utf8_string_and_magic_string (str_p, str_size, LIT_MAGIC_STRING_NULL)) { return create_token (TOK_NULL, 0); } else { return empty_token; } } } /* decode_keyword */
struct token_t *get_token(FILE *file) { static char c = 0; static char next = ' '; static unsigned line = 0, column = -1; struct token_t *token = NULL; int i = 0; while (isspace(next)) { if (next == '\n') { ++line; column = 0; } else { ++column; } next = fgetc(file); } c = next; next = fgetc(file); if (isalpha(c)) { /* ident */ token = create_token(IDENT); set_line_column(token, line, column); token->ident[0] = c; ++column; while (isalnum(next)) { c = next; next = fgetc(file); ++column; token->ident[++i] = c; } } else if (isdigit(c)) { /* number */ char buffer[BUFFER_SIZE]; token = create_token(NUMBER); set_line_column(token, line, column); buffer[0] = c; ++column; while (isdigit(next)) { c = next; next = fgetc(file); ++column; buffer[++i] = c; } token->number = atoi(buffer); } else if (c == '"') { /* text */ token = create_token(TEXT); ++column; set_line_column(token, line, column); do { c = next; next = fgetc(file); ++column; token->ident[i++] = c; if (feof(file)) { free_token(token); token = create_token(ERROR); set_line_column(token, line, column); token->error.type = BAD_EOF; return token; } else if (next == '\n') { free_token(token); token = create_token(ERROR); set_line_column(token, line, column); token->error.type = BAD_EOL; return token; } } while (next != '"'); next = fgetc(file); ++column; } else if (c == '#') { /* comment */ token = create_token(COMMENT); set_line_column(token, line, column); do { c = next; next = fgetc(file); token->ident[i++] = c; ++column; } while (next != '\n' && !feof(file)); } else if (c == '[') { token = create_token(LEFTBR); set_line_column(token, line, column++); } else if (c == ']') { token = create_token(RIGHTBR); set_line_column(token, line, column++); } else if (c == '=') { token = create_token(EQ); set_line_column(token, line, column++); } else if (feof(file)) { token = create_token(EOF_T); set_line_column(token, line, column++); } else { token = create_token(ERROR); set_line_column(token, line, column++); token->error.type = BAD_CHAR; token->error.c = c; } return token; }