/*---------------------------------------------------------------------------*/
static int
get_next_token(void)
{
struct keyword_token const *kt;
int i;
DEBUG_PRINTF("get_next_token(): '%s'\n", ptr);
if(*ptr == 0) {
return TOKENIZER_ENDOFINPUT;
}
if(isdigit(*ptr)) {
for(i = 0; i < MAX_NUMLEN; ++i) {
if(!isdigit(ptr[i])) {
if(i > 0) {
nextptr = ptr + i;
return TOKENIZER_NUMBER;
} else {
DEBUG_PRINTF("get_next_token: error due to too short number\n");
return TOKENIZER_ERROR;
}
}
if(!isdigit(ptr[i])) {
DEBUG_PRINTF("get_next_token: error due to malformed number\n");
return TOKENIZER_ERROR;
}
}
DEBUG_PRINTF("get_next_token: error due to too long number\n");
return TOKENIZER_ERROR;
} else if(singlechar()) {
nextptr = ptr + 1;
return singlechar();
} else if(*ptr == '"') {
nextptr = ptr;
do {
++nextptr;
} while(*nextptr != '"');
++nextptr;
return TOKENIZER_STRING;
} else {
for(kt = keywords; kt->keyword != NULL; ++kt) {
if(strncmp(ptr, kt->keyword, strlen(kt->keyword)) == 0) {
nextptr = ptr + strlen(kt->keyword);
return kt->token;
}
}
}
if(*ptr >= 'a' && *ptr <= 'z') {
nextptr = ptr + 1;
return TOKENIZER_VARIABLE;
}
return TOKENIZER_ERROR;
}
/*---------------------------------------------------------------------------*/
static int
get_next_token(void)
{
struct keyword_token const *kt;
int i;
DEBUG_PRINTF("get_next_token(): '%s'\n", ptr);
// eat all whitespace
while(*ptr == ' ' || *ptr == '\t' || *ptr == '\r') ptr++;
if(*ptr == 0) {
return TOKENIZER_ENDOFINPUT;
}
// UnknStatement should have size, otherwise hanging-up in ubasic.c possible for some cases
nextptr = ptr + 1;
if(isdigit(*ptr)) {
for(i = 0; i < (MAX_NUMLEN+1); ++i) {
if(!isdigit(ptr[i])) {
if(i > 0) {
nextptr = ptr + i;
return TOKENIZER_NUMBER;
} else {
DEBUG_PRINTF("get_next_token: error due to too short number\n");
return TOKENIZER_ERROR;
}
}
if(!isdigit(ptr[i])) {
DEBUG_PRINTF("get_next_token: error due to malformed number\n");
return TOKENIZER_ERROR;
}
}
DEBUG_PRINTF("get_next_token: error due to too long number\n");
return TOKENIZER_ERROR;
} else if(*ptr == ':') {
// label
nextptr = ptr;
do {
++nextptr;
} while(*nextptr != ' ' && *nextptr != '\r' && *nextptr != '\n' && *nextptr != '\t');
return TOKENIZER_LABEL;
} else if((i=singlechar()) != 0) {
if (i == TOKENIZER_CR){
// move to next line, and skip all following empty lines as well
while (singlechar() == TOKENIZER_CR)
{
current_line++;
ptr++;
// eat all whitespace
while(*ptr == ' ' || *ptr == '\t' || *ptr == '\r') ptr++;
};
ptr--;
// dangelo: now the last char might point to a whitespace instead of
// a CR. I hope that doesn't break anything.
}
nextptr = ptr + 1;
return i;
} else if(*ptr == '"') {
nextptr = ptr;
do {
++nextptr;
} while(*nextptr != '"');
++nextptr;
return TOKENIZER_STRING;
} else {
for(kt = keywords; kt->keyword != NULL; ++kt) {
if(strncmp(ptr, kt->keyword, strlen(kt->keyword)) == 0) {
nextptr = ptr + strlen(kt->keyword);
if (kt->token == TOKENIZER_REM) {
while(*nextptr != 0 && *nextptr != '\r' && *nextptr != '\n') ++nextptr;
}
return kt->token;
}
}
}
if((*ptr >= 'a' && *ptr <= 'z') || (*ptr >= 'A' && *ptr <= 'Z')) {
nextptr = ptr + 1;
return TOKENIZER_VARIABLE;
}
return TOKENIZER_ERROR;
}
/**
* Get the next token.
*/
CTokenizer::token_t CTokenizer::get_next_token()
{
token_t token=TOKEN_ERROR;
if( static_cast<uint8_t>(*m_ptr) & (uint8_t)0x80 ) // compiled instruction?....
{
m_nextptr = m_ptr;
token = (token_t)static_cast<uint8_t>(*m_ptr);
++m_nextptr;
return token;
}
if(*m_ptr == 0)
{
return TOKEN_EOF;
}
if(isdigit(*m_ptr))
{
uint8_t decimal_point=0;
for( m_nextptr = m_ptr; (isdigit(*m_nextptr) || *m_nextptr == '.') && decimal_point<=1; ++m_nextptr)
{
if ( *m_nextptr == '.' )
{
++decimal_point;
}
}
return TOKEN_NUMBER;
}
else if((token=singlechar()) != TOKEN_ERROR )
{
m_nextptr = m_ptr + 1;
switch(token)
{
case TOKEN_GT:
if ( *m_nextptr == '=' )
{ token = TOKEN_GE; ++m_nextptr; }
else if ( *m_nextptr == '>' )
{ token = TOKEN_SHR; ++m_nextptr; }
break;
case TOKEN_LT:
if ( *m_nextptr == '=' )
{ token = TOKEN_LE; ++m_nextptr; }
else if ( *m_nextptr == '>' )
{ token = TOKEN_NE; ++m_nextptr; }
else if ( *m_nextptr == '<' )
{ token = TOKEN_SHL; ++m_nextptr; }
break;
case TOKEN_BITWISEAND:
if ( *m_nextptr == '&' )
{ token = TOKEN_LOGICAND; ++m_nextptr; }
break;
case TOKEN_BITWISEOR:
if ( *m_nextptr == '|' )
{ token = TOKEN_LOGICAND; ++m_nextptr; }
break;
}
return token;
}
else if(*m_ptr == '"')
{
m_nextptr = m_ptr;
do
{
++m_nextptr;
} while(valid_ptr(m_nextptr) && *m_nextptr != '"');
if (valid_ptr(m_nextptr))
{
++m_nextptr;
return TOKEN_STRING;
}
else
{
set_runtime_error( RUNTIME_OPEN_QUOTE );
return TOKEN_ERROR;
}
}
else if ( (token=keywordtok()) != TOKEN_ERROR )
{
return token;
}
if ( CARIBOU::CString::isalpha(*m_ptr) || *m_ptr == '_' )
{
/// Handle the case where it could be a variable or function...
m_nextptr = m_ptr;
while( valid_ptr(m_nextptr) && ( CARIBOU::CString::isalpha(*m_nextptr) || isnum(*m_nextptr) || *m_nextptr == '_' || *m_nextptr == '$' || *m_nextptr == '@' || *m_nextptr == '#' ) )
{
m_nextptr++;
}
if ( *m_nextptr == '(' )
{
token = functiontok();
if ( token == TOKEN_ERROR )
{
set_runtime_error( RUNTIME_NO_SUCH_FUNCTION );
}
return token;
}
return TOKEN_VARIABLE;
}
return TOKEN_ERROR;
}
