/*---------------------------------------------------------------------------*/ static int get_next_token(void) { struct keyword_token const *kt; int i; DEBUG_PRINTF("get_next_token(): '%s'\n", ptr); if(*ptr == 0) { return TOKENIZER_ENDOFINPUT; } if(isdigit(*ptr)) { for(i = 0; i < MAX_NUMLEN; ++i) { if(!isdigit(ptr[i])) { if(i > 0) { nextptr = ptr + i; return TOKENIZER_NUMBER; } else { DEBUG_PRINTF("get_next_token: error due to too short number\n"); return TOKENIZER_ERROR; } } if(!isdigit(ptr[i])) { DEBUG_PRINTF("get_next_token: error due to malformed number\n"); return TOKENIZER_ERROR; } } DEBUG_PRINTF("get_next_token: error due to too long number\n"); return TOKENIZER_ERROR; } else if(singlechar()) { nextptr = ptr + 1; return singlechar(); } else if(*ptr == '"') { nextptr = ptr; do { ++nextptr; } while(*nextptr != '"'); ++nextptr; return TOKENIZER_STRING; } else { for(kt = keywords; kt->keyword != NULL; ++kt) { if(strncmp(ptr, kt->keyword, strlen(kt->keyword)) == 0) { nextptr = ptr + strlen(kt->keyword); return kt->token; } } } if(*ptr >= 'a' && *ptr <= 'z') { nextptr = ptr + 1; return TOKENIZER_VARIABLE; } return TOKENIZER_ERROR; }
/*---------------------------------------------------------------------------*/ static int get_next_token(void) { struct keyword_token const *kt; int i; DEBUG_PRINTF("get_next_token(): '%s'\n", ptr); // eat all whitespace while(*ptr == ' ' || *ptr == '\t' || *ptr == '\r') ptr++; if(*ptr == 0) { return TOKENIZER_ENDOFINPUT; } // UnknStatement should have size, otherwise hanging-up in ubasic.c possible for some cases nextptr = ptr + 1; if(isdigit(*ptr)) { for(i = 0; i < (MAX_NUMLEN+1); ++i) { if(!isdigit(ptr[i])) { if(i > 0) { nextptr = ptr + i; return TOKENIZER_NUMBER; } else { DEBUG_PRINTF("get_next_token: error due to too short number\n"); return TOKENIZER_ERROR; } } if(!isdigit(ptr[i])) { DEBUG_PRINTF("get_next_token: error due to malformed number\n"); return TOKENIZER_ERROR; } } DEBUG_PRINTF("get_next_token: error due to too long number\n"); return TOKENIZER_ERROR; } else if(*ptr == ':') { // label nextptr = ptr; do { ++nextptr; } while(*nextptr != ' ' && *nextptr != '\r' && *nextptr != '\n' && *nextptr != '\t'); return TOKENIZER_LABEL; } else if((i=singlechar()) != 0) { if (i == TOKENIZER_CR){ // move to next line, and skip all following empty lines as well while (singlechar() == TOKENIZER_CR) { current_line++; ptr++; // eat all whitespace while(*ptr == ' ' || *ptr == '\t' || *ptr == '\r') ptr++; }; ptr--; // dangelo: now the last char might point to a whitespace instead of // a CR. I hope that doesn't break anything. } nextptr = ptr + 1; return i; } else if(*ptr == '"') { nextptr = ptr; do { ++nextptr; } while(*nextptr != '"'); ++nextptr; return TOKENIZER_STRING; } else { for(kt = keywords; kt->keyword != NULL; ++kt) { if(strncmp(ptr, kt->keyword, strlen(kt->keyword)) == 0) { nextptr = ptr + strlen(kt->keyword); if (kt->token == TOKENIZER_REM) { while(*nextptr != 0 && *nextptr != '\r' && *nextptr != '\n') ++nextptr; } return kt->token; } } } if((*ptr >= 'a' && *ptr <= 'z') || (*ptr >= 'A' && *ptr <= 'Z')) { nextptr = ptr + 1; return TOKENIZER_VARIABLE; } return TOKENIZER_ERROR; }
/** * Get the next token. */ CTokenizer::token_t CTokenizer::get_next_token() { token_t token=TOKEN_ERROR; if( static_cast<uint8_t>(*m_ptr) & (uint8_t)0x80 ) // compiled instruction?.... { m_nextptr = m_ptr; token = (token_t)static_cast<uint8_t>(*m_ptr); ++m_nextptr; return token; } if(*m_ptr == 0) { return TOKEN_EOF; } if(isdigit(*m_ptr)) { uint8_t decimal_point=0; for( m_nextptr = m_ptr; (isdigit(*m_nextptr) || *m_nextptr == '.') && decimal_point<=1; ++m_nextptr) { if ( *m_nextptr == '.' ) { ++decimal_point; } } return TOKEN_NUMBER; } else if((token=singlechar()) != TOKEN_ERROR ) { m_nextptr = m_ptr + 1; switch(token) { case TOKEN_GT: if ( *m_nextptr == '=' ) { token = TOKEN_GE; ++m_nextptr; } else if ( *m_nextptr == '>' ) { token = TOKEN_SHR; ++m_nextptr; } break; case TOKEN_LT: if ( *m_nextptr == '=' ) { token = TOKEN_LE; ++m_nextptr; } else if ( *m_nextptr == '>' ) { token = TOKEN_NE; ++m_nextptr; } else if ( *m_nextptr == '<' ) { token = TOKEN_SHL; ++m_nextptr; } break; case TOKEN_BITWISEAND: if ( *m_nextptr == '&' ) { token = TOKEN_LOGICAND; ++m_nextptr; } break; case TOKEN_BITWISEOR: if ( *m_nextptr == '|' ) { token = TOKEN_LOGICAND; ++m_nextptr; } break; } return token; } else if(*m_ptr == '"') { m_nextptr = m_ptr; do { ++m_nextptr; } while(valid_ptr(m_nextptr) && *m_nextptr != '"'); if (valid_ptr(m_nextptr)) { ++m_nextptr; return TOKEN_STRING; } else { set_runtime_error( RUNTIME_OPEN_QUOTE ); return TOKEN_ERROR; } } else if ( (token=keywordtok()) != TOKEN_ERROR ) { return token; } if ( CARIBOU::CString::isalpha(*m_ptr) || *m_ptr == '_' ) { /// Handle the case where it could be a variable or function... m_nextptr = m_ptr; while( valid_ptr(m_nextptr) && ( CARIBOU::CString::isalpha(*m_nextptr) || isnum(*m_nextptr) || *m_nextptr == '_' || *m_nextptr == '$' || *m_nextptr == '@' || *m_nextptr == '#' ) ) { m_nextptr++; } if ( *m_nextptr == '(' ) { token = functiontok(); if ( token == TOKEN_ERROR ) { set_runtime_error( RUNTIME_NO_SUCH_FUNCTION ); } return token; } return TOKEN_VARIABLE; } return TOKEN_ERROR; }