// Reads the next token available, and sets up the variables for that.
void Tokenizer::parse_next_token()
{
// If the current token is the end of a statement, read the next one.
if( current_token == END_STATEMENT)
{
if(m_file->has_next())
{
#ifdef DEBUG
std::cout << "Getting next" << std::endl;
#endif
current_line = m_file->read_line();
current_line_position = 0;
// Continue parsing on!
#ifdef DEBUG
std::cout << "Next is: " << current_line << std::endl;
#endif
}
else
{
current_token = END_PROGRAM;
return;
}
}
// Skip all whitespace.
while(is_whitespace(current_line[current_line_position]))
current_line_position++;
#ifdef DEBUG
std::cout << "Getting next token at pos: " << current_line_position << std::endl;
std::cout << current_line << std::endl;
for(int i = 0; i < current_line_position; ++i)
std::cout << " ";
std::cout << "^" << std::endl;
#endif
// Check to see if this is the end of the statement.
if(current_line[current_line_position] == 0)
{
current_token = END_STATEMENT;
return;
}
// Check to see if this is a number
if(is_digit(current_line[current_line_position]))
{
curr_int = 0;
while(is_digit(current_line[current_line_position]))
{
curr_int *= 10;
curr_int += to_digit(current_line[current_line_position]);
current_line_position++;
}
current_token = NUMBER;
return;
}
// Check to see if this is an operator
current_line_position ++; // Skip past this for now, decrement later if wrong.
switch(current_line[current_line_position - 1])
{
case ',':
current_token = COMMA;
return;
case '+':
current_token = PLUS;
return;
case '-':
current_token = MINUS;
return;
case '*':
current_token = MULTIPLY;
return;
case '/':
current_token = DIVIDE;
return;
case '=':
current_token = EQUALS;
return;
case '(':
current_token = LPAREN;
return;
case ')':
current_token = RPAREN;
return;
case '>':
current_token = GREATER_THAN;
return;
case '<':
current_token = LESS_THAN;
return;
case '\n':
current_token = END_STATEMENT;
return;
default:
current_line_position--;
break;
}
// Check to see if this is a var.
if(is_alpha(current_line[current_line_position]) &&
! is_alpha(current_line[current_line_position + 1]))
{
current_token = VARIABLE;
varname = current_line[current_line_position];
current_line_position++;
#ifdef DEBUG
std::cout << "Is a variable: " << varname << std::endl;
#endif
return;
}
// Check to see if this is a string
if(current_line[current_line_position] == '"')
{
int tmp = 1;
while(current_line[current_line_position + tmp] != '"' &&
current_line[current_line_position + tmp] != 0)
tmp ++;
tmp++; // advance past the final "
delete[] curr_str;
curr_str = 0;
curr_str = substring(current_line,current_line_position + 1, tmp - 2);
current_line_position += tmp;
current_token = STRING;
return;
}
// Check to see if this is a function
if(starts_with_ignore_case(current_line, current_line_position, (char*)"PRINT"))
{
current_token = PRINT;
current_line_position += 5;
return;
}
if(starts_with_ignore_case(current_line, current_line_position, (char*)"IF"))
{
current_token = IF;
current_line_position += 2;
return;
}
if(starts_with_ignore_case(current_line, current_line_position, (char*)"THEN"))
{
current_token = THEN;
current_line_position += 4;
return;
}
if(starts_with_ignore_case(current_line, current_line_position, (char*)"GOTO"))
{
current_token = GOTO;
current_line_position += 4;
return;
}
if(starts_with_ignore_case(current_line, current_line_position, (char*)"INPUT"))
{
current_token = INPUT;
current_line_position += 5;
return;
}
if(starts_with_ignore_case(current_line, current_line_position, (char*)"LET"))
{
current_token = LET;
current_line_position += 3;
return;
}
if(starts_with_ignore_case(current_line, current_line_position, (char*)"GOSUB"))
{
current_token = GOSUB;
current_line_position += 5;
return;
}
if(starts_with_ignore_case(current_line, current_line_position, (char*)"RETURN"))
{
current_token = RETURN;
current_line_position += 6;
return;
}
if(starts_with_ignore_case(current_line, current_line_position, (char*)"CLEAR"))
{
current_token = CLEAR;
current_line_position += 5;
return;
}
if(starts_with_ignore_case(current_line, current_line_position, (char*)"LIST"))
{
current_token = LIST;
current_line_position += 4;
return;
}
if(starts_with_ignore_case(current_line, current_line_position, (char*)"RUN"))
{
current_token = RUN;
return;
}
if(starts_with_ignore_case(current_line, current_line_position, (char*)"END"))
{
current_token = END;
current_line_position += 3;
return;
}
current_token = ERROR;
}
int Xcode_puny_decodeString( const UCHAR8 * pzInputString,
const int iInputSize,
UTF16CHAR * puzOutputString,
int * piOutputSize )
{
int status;
int offset = 0;
int input_offset = 0;
int output_offset = 0;
unsigned int punycode_input_length, punycode_output_length;
char punycode_input[MAX_LABEL_SIZE_8];
DWORD punycode_output[MAX_LABEL_SIZE_32];
if ( iInputSize < 1 ) {return XCODE_BAD_ARGUMENT_ERROR;}
/* Make sure we have a punycode encoded label here, otherwise, just
return the string untouched. */
if ( !starts_with_ignore_case( pzInputString, iInputSize, (const unsigned char *)ACE_PREFIX, strlen(ACE_PREFIX) ) )
{
//punycode_input_length = 0;
for( offset = 0; offset < iInputSize; offset++ )
{
if ( offset >= *piOutputSize ) return XCODE_BUFFER_OVERFLOW_ERROR;
*(puzOutputString + offset) = (UTF16CHAR)pzInputString[offset];
}
*piOutputSize = iInputSize;
return XCODE_SUCCESS;
}
/* copy the input to punycode input ignoring the prefix */
input_offset = strlen(ACE_PREFIX);
punycode_input_length = 0;
for(offset = 0; input_offset < iInputSize; offset++, input_offset++)
{
punycode_input[offset] = (char)pzInputString[input_offset];
punycode_input_length++;
}
/* lowercase it */
lower_case( (unsigned char *)punycode_input, punycode_input_length );
punycode_output_length = MAX_LABEL_SIZE_32;
/* decode the input */
status = punycode_decode(punycode_input_length, punycode_input,
&punycode_output_length, punycode_output);
/* check the status */
if (status != XCODE_SUCCESS) { return status; }
/* copy the punycode output to the output if there is room */
output_offset = 0;
if ((int)output_offset > *piOutputSize - (int)punycode_output_length)
{
return XCODE_BUFFER_OVERFLOW_ERROR;
}
/* Convert result to UTF16 */
status = Xcode_convert32BitToUTF16( punycode_output, punycode_output_length,
puzOutputString, piOutputSize );
if ( status != XCODE_SUCCESS ) return status;
/* terminate the string */
*(puzOutputString + *piOutputSize) = 0;
return XCODE_SUCCESS;
}
