void setup() {
s1.line = 0;
s1.column = 1;
s2.line = 2;
s2.column = 3;
t1 = token_new( value, length, type, s1, s2 );
t2 = token_new( value, length, type, s1, s2 );
}
Token *run_addop() {
char cur = next_char();
switch(cur) {
case '+':
return token_new(TOK_ADDOP, ADDOP_ADD);
case '-':
return token_new(TOK_ADDOP, ADDOP_SUBTRACT);
default:
return token_new(TOK_BLOCKED, 0);
}
}
Token *run_assignop() {
char cur = next_char();
if(cur == ':') {
cur = next_char();
if(cur == '=') {
return token_new(TOK_ASSIGNOP, 0);
}
}
return token_new(TOK_BLOCKED, 0);
}
PyContextToken *
PyContextVar_Set(PyContextVar *var, PyObject *val)
{
if (!PyContextVar_CheckExact(var)) {
PyErr_SetString(
PyExc_TypeError, "an instance of ContextVar was expected");
return NULL;
}
PyContext *ctx = context_get();
if (ctx == NULL) {
return NULL;
}
PyObject *old_val = NULL;
int found = _PyHamt_Find(ctx->ctx_vars, (PyObject *)var, &old_val);
if (found < 0) {
return NULL;
}
Py_XINCREF(old_val);
PyContextToken *tok = token_new(ctx, var, old_val);
Py_XDECREF(old_val);
if (contextvar_set(var, val)) {
Py_DECREF(tok);
return NULL;
}
return tok;
}
inline static grn_cell *
get_phrase(grn_ctx *ctx, grn_query *q)
{
char *start, *s, *d;
start = s = d = q->cur;
while (1) {
unsigned int len;
if (s >= q->str_end) {
q->cur = s;
break;
}
len = grn_charlen(ctx, s, q->str_end);
if (len == 0) {
/* invalid string containing malformed multibyte char */
return NULL;
} else if (len == 1) {
if (*s == GRN_QUERY_QUOTER) {
q->cur = s + 1;
break;
} else if (*s == GRN_QUERY_ESCAPE && s + 1 < q->str_end) {
s++;
len = grn_charlen(ctx, s, q->str_end);
}
}
while (len--) { *d++ = *s++; }
}
return token_new(q, start, d);
}
/* parser_get_token */
int parser_get_token(Parser * parser, Token ** token)
{
int ret = 0; /* XXX not sure */
size_t i;
ParserCallbackData * pcd;
if((*token = token_new(parser->filename, parser->line, parser->col))
== NULL)
return 1;
if(parser->last == EOF)
parser_scan_filter(parser);
#ifdef DEBUG
fprintf(stderr, "DEBUG: %s() %c\n", __func__, parser->last);
#endif
for(i = 0; i < parser->callbacks_cnt; i++)
{
pcd = &parser->callbacks[i];
if((ret = pcd->callback(parser, *token, parser->last,
pcd->data)) <= 0)
break;
}
if(ret == 0 && i != parser->callbacks_cnt)
return 0; /* there is a token and no error */
token_delete(*token);
*token = NULL;
return (ret >= 0 && parser->last == EOF) ? 0 : 1;
}
ssize_t tokenize(const char *string,
struct s_vec *queue)
{
size_t i;
size_t len;
struct s_token *token;
assert(string != NULL);
assert(queue != NULL);
if (vec_init(queue, 0, &free_token_queue) == -1)
return (-1);
len = strlen(string);
i = 0;
while (i < len)
{
if (string[i] == ' ' || string[i] == '\t')
{
++i;
continue;
}
if (token_new(&token) != 0 || token_init(token) != 0)
return (-1);
if (!call_tokenizers(string, queue, token, &i))
i += 1;
}
return (queue->size);
}
inline static grn_cell *
get_word(grn_ctx *ctx, grn_query *q, int *prefixp)
{
char *start = q->cur, *end;
unsigned int len;
for (end = q->cur;; ) {
/* null check and length check */
if (!(len = grn_charlen(ctx, end, q->str_end))) {
q->cur = q->str_end;
break;
}
if (grn_isspace(end, q->encoding) ||
*end == GRN_QUERY_PARENR) {
q->cur = end;
break;
}
if (*end == GRN_QUERY_PREFIX) {
*prefixp = 1;
q->cur = end + 1;
break;
}
end += len;
}
return token_new(q, start, end);
}
Token *run_ws() {
char c = next_char();
if(!match_ws(c)) {
Token *blocked_token = token_new(TOK_BLOCKED, 0);
return blocked_token;
}
while(match_ws(c)) {
c = next_char();
}
move_f_back();
Token *my_token = token_new(TOK_WS, 0);
return my_token;
}
END_TEST
START_TEST (test_token_new)
{
struct s_token *t;
ck_assert_int_eq(token_new(&t), 0);
}
END_TEST
START_TEST (test_token_new_malloc_fail)
{
struct s_token *t;
set_mock_malloc(&my_mock_malloc_null);
ck_assert_int_ne(token_new(&t), 0);
set_mock_malloc(&malloc);
}
// Process any deferred token we have
static void process_deferred_ast(parser_t* parser, rule_state_t* state)
{
assert(parser != NULL);
assert(state != NULL);
if(state->deferred)
{
token_t* deferred_token = token_new(state->deferred_id);
token_set_pos(deferred_token, parser->source, state->line, state->pos);
state->ast = ast_token(deferred_token);
state->deferred = false;
}
}
Token *parseComment( Tokenizer *tokenizer ) {
StatefulString *ss = tokenizer->ss_;
assert( isCommentStart( ss, 0 ) );
int start, length;
StatefulStringPosition pos1, pos2;
Token *token;
wchar_t *error = malloc( 201 * sizeof( wchar_t ) );
error[ 0 ] = L'\0';
start = ss->next_index;
pos1 = ss->next_position;
length = 2;
TokenType type = COMMENT;
ss_getchar( ss ); ss_getchar( ss ); // Throw away `/*`
while (
ss_peek( ss ) != WEOF &&
(
ss_peek( ss ) != L'*' ||
ss_peekx( ss, 1 ) != L'/'
)
) {
length++;
if ( ss_getchar( ss ) == L'\\' ) {
ss_getchar( ss );
length++;
}
}
if ( ss_peek( ss ) == WEOF ) {
swprintf( error, 200, L"Encountered end-of-file while parsing a comment. Probably a forgotten `*/`." );
} else {
ss_getchar( ss ); ss_getchar( ss ); // Throw away `*/`
length += 2;
}
// Return the token.
pos2 = ss->next_position;
token = token_new( ss_substr( ss, start, length ), length, type, pos1, pos2 );
if ( wcscmp( error, L"" ) != 0 ) {
tokenizer_error( tokenizer, error, token );
} else {
free( error );
}
return token;
}
Token* addtoken( Tokenizer *tokenizer, const wchar_t *value,
const unsigned int length, const TokenType type,
const StatefulStringPosition start,
const StatefulStringPosition end ) {
Token *token = token_new( value, length, type, start, end );
if ( tokenizer->numtokens >= tokenizer->maxtokens_ ) {
tokenizer->maxtokens_ *= 2;
tokenizer->tokens_ = realloc( tokenizer->tokens_, ( tokenizer->maxtokens_ + 1 ) * sizeof( Token* ) );
if ( tokenizer->tokens_ == NULL ) {
allocationerror( tokenizer->maxtokens_, L"Tokenizer::addtoken" );
exit( EXIT_FAILURE );
}
}
tokenizer->tokens_[ tokenizer->numtokens++ ] = token;
return token;
}
static Token *advance() {
char cur = next_char();
Token *result_token;
switch(state) {
case 0:
if(cur == '<') {
state = 1;
return advance();
}
if(cur == '>') {
state = 2;
return advance();
}
if(cur == '=') {
result_token = token_new(TOK_RELOP, RELOP_EQUAL);
return result_token;
}
result_token = token_new(TOK_BLOCKED, 0);
return result_token;
case 1:
if(cur == '>') {
result_token = token_new(TOK_RELOP, RELOP_NOT_EQUAL);
return result_token;
}
if(cur == '=') {
result_token = token_new(TOK_RELOP, RELOP_LESS_THAN_OR_EQUAL);
return result_token;
}
move_f_back();
result_token = token_new(TOK_RELOP, RELOP_LESS_THAN);
return result_token;
case 2:
if(cur == '=') {
result_token = token_new(TOK_RELOP, RELOP_GREATER_THAN_OR_EQUAL);
return result_token;
}
move_f_back();
result_token = token_new(TOK_RELOP, RELOP_GREATER_THAN);
return result_token;
default:
result_token = token_new(TOK_BLOCKED, 0);
return result_token;
}
}
///////////////////////////////////////////////////////////////////////////
//
// Parsing Functions
//
/////////////////////////////////////
//
// Whitespace
//
Token *parseWhitespace( Tokenizer *tokenizer ) {
StatefulString *ss = tokenizer->ss_;
assert( isWhitespaceStart( ss, 0 ) );
int start, length;
StatefulStringPosition pos1, pos2;
start = ss->next_index;
length = 0;
pos1 = ss->next_position;
while ( isWhitespace( ss_peek( ss ) ) ) {
ss_getchar( ss );
length++;
}
pos2 = ss->next_position;
return token_new( ss_substr( ss, start, length ), length, WHITESPACE, pos1, pos2 );
}
/////////////////////////////////////
//
// Strings
//
Token *parseString( Tokenizer *tokenizer ) {
StatefulString *ss = tokenizer->ss_;
assert( isStringStart( ss, 0 ) );
int start, length;
StatefulStringPosition pos1, pos2;
wchar_t temp;
Token *token;
length = 0;
pos1 = ss->next_position;
wchar_t quote = ss_getchar( ss );
start = ss->next_index; // Exclude the quotes
while (
( temp = ss_getchar( ss ) ) != WEOF &&
!isNewline( temp ) &&
temp != quote
) {
length++;
if ( temp == L'\\' ) {
ss_getchar( ss );
length++;
}
}
pos2 = ss->next_position;
token = token_new( ss_substr( ss, start, length ), length, STRING, pos1, pos2 );
wchar_t *error = malloc( 201 * sizeof( wchar_t ) );
error[ 0 ] = L'\0';
if ( temp == WEOF ) {
swprintf( error, 200, L"Encountered end-of-file while parsing a string. Looks like a `%C` is missing.", quote );
} else if ( isNewline( temp ) ) {
error = L"Encountered a newline while parsing a string. Strings can only contain newlines if they're properly escaped (e.g. `\\[\\n]`)";
}
if ( wcscmp( error, L"" ) != 0 ) {
tokenizer_error( tokenizer, error, token );
} else {
free( error );
}
return token;
}
Token *parseNumber( Tokenizer *tokenizer ) {
StatefulString *ss = tokenizer->ss_;
assert( isNumberStart( ss, 0 ) );
int start, length;
StatefulStringPosition pos1, pos2;
start = ss->next_index;
length = 0;
pos1 = ss->next_position;
TokenType type = NUMBER;
int isFloat = 0;
wchar_t temp;
while (
isNumeric( ss_peek( ss ) ) ||
( ss_peek( ss ) == L'-' && length == 0 ) ||
( ss_peek( ss ) == L'.' && !isFloat )
) {
temp = ss_getchar( ss );
if ( temp == L'.' ) {
isFloat = 1;
}
length++;
}
// Is the number followed by a percentage?
if ( ss_peek( ss ) == L'%' ) {
ss_getchar( ss );
length++;
type = PERCENTAGE;
}
// Is the number followed by a dimension?
else if ( isIdentifierStart( ss, 0 ) ) {
while ( isNameChar( ss_peek( ss ) ) ) {
ss_getchar( ss );
length++;
}
type = DIMENSION;
}
pos2 = ss->next_position;
return token_new( ss_substr( ss, start, length ), length, type, pos1, pos2 );
}
int assembler(struct asm_context_t *ctx, char *infile)
{
struct token_t *token;
int error;
token = token_new(&ctx->tokens);
token_prepare(token, infile);
while (1)
{
token_drop(token);
if (lang_eof(token) == 0)
break;
if (lang_comment(token) == 0)
continue;
if (lang_label(ctx, token) == 0)
continue;
if (ctx->pass)
{
if (lang_directive(ctx, token) == 0)
continue;
if (lang_instruction(ctx, token) == 0)
continue;
} else {
if (token_get(token, TOKEN_TYPE_LINE, TOKEN_CURRENT))
continue;
}
debug_emsg("Unknown program construction");
goto error;
}
error = 0;
goto noerror;
error:
error = -1;
token_print_rollback(token);
debug_emsgf("Error in file", "%s" NL, infile);
noerror:
token_remove(&ctx->tokens, token);
return error;
}
Token *parseName( Tokenizer *tokenizer ) {
StatefulString *ss = tokenizer->ss_;
assert( isNameChar( ss_peek( ss ) ) || isUnicodeSequenceStart( ss, 0 ) );
int start, length;
StatefulStringPosition pos1, pos2;
start = ss->next_index;
length = 0;
pos1 = ss->next_position;
while ( isNameChar( ss_peek( ss ) ) || isUnicodeSequenceStart( ss, 0 ) ) {
length += processChar( ss );
}
if ( ss_peek( ss ) == L'(' ) {
ss_getchar( ss );
length++;
}
pos2 = ss->next_position;
return token_new( ss_substr( ss, start, length ), length, NAME, pos1, pos2 );
}
Token *parseSGMLComment( Tokenizer *tokenizer ) {
StatefulString *ss = tokenizer->ss_;
assert( isCDOStart( ss, 0 ) || isCDCStart( ss, 0 ) );
int start, length;
StatefulStringPosition pos1, pos2;
start = ss->next_index;
pos1 = ss->next_position;
length = 4;
TokenType type = SGML_COMMENT_OPEN;
if ( isCDCStart( ss, 0 ) ) {
type = SGML_COMMENT_CLOSE;
length = 3;
}
for ( int i = 0; i<length; i++ ) {
ss_getchar( ss );
}
pos2 = ss->next_position;
return token_new( ss_substr( ss, start, length ), length, type, pos1, pos2 );
}
static int read_conllx(FILE* fp)
{
int num_sent = 0;
unsigned int line_num = 0;
char buff[BUF_SIZE];
const char* sep = "\t";
char *t, *line, *tofree;
size_t len;
const char** seq = xmalloc(sizeof(char *) * CONLLX_TOKEN_NUM_FIELDS);
struct blocks* blocks = new_blocks(256);
struct sentence* sent = sentence_new();
while (fgets(buff, sizeof(buff), fp)) {
line_num++;
if (buff[0] == '\n') {
latex_print_dep_tree(sent);
sentence_destroy(sent);
sent = sentence_new();
num_sent++;
continue;
}
len = strnlen(buff, BUF_SIZE);
tofree = line = strndup(buff, len-1);
blocks_add_elem(blocks, tofree);
int i = 0;
while ((t = strsep(&line, sep)) != NULL) {
seq[i] = t;
i++;
}
sentence_add_token(sent, token_new(seq, CONLLX_TOKEN_NUM_FIELDS));
}
destroy_blocks(blocks);
sentence_destroy(sent);
free(seq);
fprintf(stderr, "INFO: Number of sentences = %d\n", num_sent);
return 0;
}
/////////////////////////////////////
//
// Operators, et al.
//
Token *parseEverythingElse( Tokenizer *tokenizer ) {
StatefulString *ss = tokenizer->ss_;
int start, length;
StatefulStringPosition pos1, pos2;
start = ss->next_index;
pos1 = ss->next_position;
TokenType type = DELIM;
length = 1;
wchar_t temp = ss_getchar( ss );
if (
( temp== L'~' || temp == L'|' || temp == L'*' || temp == L'^' || temp == L'$' ) &&
ss_peek( ss ) == L'='
) {
ss_getchar( ss );
length++;
}
if ( length == 1 ) {
switch ( temp ) {
case L'{':
type = CURLY_BRACE_OPEN;
break;
case L'}':
type = CURLY_BRACE_CLOSE;
break;
case L'[':
type = SQUARE_BRACE_OPEN;
break;
case L']':
type = SQUARE_BRACE_CLOSE;
break;
case L'(':
type = PAREN_OPEN;
break;
case L')':
type = PAREN_CLOSE;
break;
case L':':
type = COLON;
break;
case L';':
type = SEMICOLON;
break;
case L'@':
type = AT;
break;
case L'#':
type = HASH;
break;
case L'%':
type = PERCENT;
break;
case L'.':
type = DOT;
break;
}
} else if ( length == 2 ) {
switch ( temp ) {
case L'~':
type = INCLUDES;
break;
case L'|':
type = DASHMATCH;
break;
case L'^':
type = PREFIXMATCH;
break;
case L'$':
type = SUFFIXMATCH;
break;
case L'*':
type = SUBSTRINGMATCH;
break;
}
}
pos2 = ss->next_position;
return token_new( ss_substr( ss, start, length ), length, type, pos1, pos2 );
}
struct token token_stream_read(struct token_stream * ts)
{
if(ts->end)
return token_new(TOKEN_TYPE_END);
enum states{
READ_NEXT = 0,
SCAN, SCAN_ERR,
BUILD_SECTION, BUILD_SECTION1, BUILD_SECTION_END,
BUILD_NAME, BUILD_NAME1, BUILD_NAME_END,
BUILD_ASSIGNMENT,
BUILD_STRING, BUILD_STRING1, BUILD_STRING_END,
BUILD_COMMENT,
BUILD_NEWLINE,
BUILD_END,
ERROR,
SKIP_LINE,
} state = ts->next_state;
struct varstr * buf = varstr_new();
struct char_stream * cs = ts->cs;
struct token tok;
int running = 1;
do switch(state){
case READ_NEXT:
next_char(cs);
state = SCAN;
case SCAN:
if(!cs->escaped)
{
if(cs->c == ' ' || cs->c == '\t')
state = READ_NEXT;
else if(cs->c == '[')
state = BUILD_SECTION;
else if(char_is_name(cs))
state = BUILD_NAME;
else if(cs->c == '=')
state = BUILD_ASSIGNMENT;
else if(cs->c == '"')
state = BUILD_STRING;
else if(cs->c == '#')
state = BUILD_COMMENT;
else if(cs->c == '\n')
state = BUILD_NEWLINE;
else if(cs->c == EOF)
state = BUILD_END;
else
state = SCAN_ERR;
}
else
state = SCAN_ERR;
break;
case SCAN_ERR:
fprintf(stderr,"next_token: "
"bad token at row %d, column %d\n",
cs->row, cs->col);
state = SKIP_LINE;
break;
case BUILD_SECTION:
// [section]
varstr_clear(buf);
tok = token_new(TOKEN_TYPE_SECTION);
state = BUILD_SECTION1;
case BUILD_SECTION1:
next_char(cs);
if(char_is_name(cs))
varstr_append(buf, cs->c);
else
state = BUILD_SECTION_END;
break;
case BUILD_SECTION_END:
if(cs->c == ']' && !cs->escaped)
{
ts->next_state = READ_NEXT;
tok.text = cstr_dup(varstr_view(buf));
running = 0;
}
else
{
fprintf(stderr, "next_token: "
"bad section name at row %d, column %d, "
"expected ']'\n",
cs->row, cs->col);
state = ERROR;
}
break;
case BUILD_NAME:
// name
tok = token_new(TOKEN_TYPE_NAME);
varstr_clear(buf);
varstr_append(buf, cs->c);
state = BUILD_NAME1;
case BUILD_NAME1:
next_char(cs);
if(char_is_name(cs))
varstr_append(buf, cs->c);
else
state = BUILD_NAME_END;
break;
case BUILD_NAME_END:
ts->next_state = SCAN;
tok.text = cstr_dup(varstr_view(buf));
running = 0;
break;
case BUILD_ASSIGNMENT:
// =
tok = token_new(TOKEN_TYPE_ASSIGNMENT);
ts->next_state = READ_NEXT;
running = 0;
break;
case BUILD_STRING:
/* "multi
line
string
with escape /"/n"
*/
tok = token_new(TOKEN_TYPE_STRING);
varstr_clear(buf);
state = BUILD_STRING1;
case BUILD_STRING1:
next_char(cs);
if(cs->c == EOF)
state = BUILD_STRING_END;
else if(cs->c == '"' && !cs->escaped)
state = BUILD_STRING_END;
else
varstr_append(buf, cs->c);
break;
case BUILD_STRING_END:
if(cs->c == '"' && !cs->escaped)
{
ts->next_state = READ_NEXT;
tok.text = cstr_dup(varstr_view(buf));
running = 0;
}
else if(cs->c == EOF)
{
fprintf(stderr, "next_token"
"bad string at row %d, column %d, "
"expected '\"'\n",
cs->row, cs->col);
state = ERROR;
}
break;
case BUILD_COMMENT:
// # comment
state = SKIP_LINE;
break;
case BUILD_NEWLINE:
tok = token_new(TOKEN_TYPE_NEWLINE);
ts->next_state = READ_NEXT;
running = 0;
break;
case BUILD_END:
tok = token_new(TOKEN_TYPE_NEWLINE);
ts->end = 1;
running = 0;
break;
case ERROR:
state = SKIP_LINE;
case SKIP_LINE:
if(cs->c == '\n' && !cs->escaped)
state = BUILD_NEWLINE;
else if(cs->c == EOF)
state = BUILD_END;
else
next_char(cs);
break;
}while(running);
varstr_del(buf);
return tok;
}
token_t *token_new_no_data(token_id_t id) {
return token_new(id, NULL, NULL, NULL, NULL);
}
static token_t *error_new(error_info_t *info) {
return token_new(TOKEN_ERROR, info, NULL, error_info_free_func,
error_info_to_string_func);
}
// Make a token with the specified ID and no token text
static token_t* make_token(lexer_t* lexer, token_id id)
{
token_t* t = token_new(id);
token_set_pos(t, lexer->source, lexer->token_line, lexer->token_pos);
return t;
}
/////////////////////////////////////
//
// Url
//
Token *parseUrl( Tokenizer *tokenizer ) {
StatefulString *ss = tokenizer->ss_;
assert( isUrlStart( ss, 0 ) );
int start, length;
StatefulStringPosition pos1, pos2;
Token *token;
wchar_t *error = malloc( 201 * sizeof( wchar_t ) );
error[ 0 ] = L'\0';
start = ss->next_index;
pos1 = ss->next_position;
TokenType type = URL;
// Throw away `url(`
for ( int i = 0; i<4; i++ ) {
ss_getchar( ss );
}
length = 4;
// Throw away leading whitespace
while ( isWhitespaceStart( ss, 0 ) ) {
ss_getchar( ss );
length++;
}
// Process the url
if ( isStringStart( ss, 0 ) ) {
token = parseString( tokenizer );
length += token->length + 2; // +2 for the quotes
if ( token->error != NULL ) {
length -= 1; // Error'd strings don't have a trailing quote
pos2 = ss->next_position;
Token *t = token_new( ss_substr( ss, start, length ), length, type, pos1, pos2 );
t->error = token->error;
( token->error )->token = t;
free( token );
return t;
} else {
free( token );
}
} else if ( isUrlChar( ss_peek( ss ) ) || isUnicodeSequenceStart( ss, 0 ) ) {
while ( isUrlChar( ss_peek( ss ) ) || isUnicodeSequenceStart( ss, 0 ) ) {
length += processChar( ss );
}
} else {
// ERROR:
if ( ss_peek( ss ) == WEOF ) {
swprintf( error, 200, L"Encountered end-of-file while parsing a URL." );
} else {
swprintf( error, 200, L"Encountered an invalid character (`%C`) while parsing a URL.", ss_peek( ss ) );
}
}
// Throw away trailing whitespace
while ( isWhitespaceStart( ss, 0 ) ) {
ss_getchar( ss );
length++;
}
// Grab the trailing `)`
if ( ss_peek( ss ) == L')' ) {
ss_getchar( ss );
length++;
} else {
// ERROR:
if ( ss_peek( ss ) == WEOF ) {
swprintf( error, 200, L"Encountered end-of-file while parsing a URL. Expected a `)`." );
} else {
swprintf( error, 200, L"Encountered an invalid character (`%C`) while parsing a URL. Expected a `)`. Perhaps a typo?", ss_peek( ss ) );
}
}
// Return the token.
pos2 = ss->next_position;
token = token_new( ss_substr( ss, start, length ), length, type, pos1, pos2 );
if ( wcscmp( error, L"" ) != 0 ) {
tokenizer_error( tokenizer, error, token );
} else {
free( error );
}
return token;
}
// returns a token on the heap, or a null pointer
Token *scan_string(char **buf) {
char *starting_buf = *buf;
static int in_multiline_comment = false;
char c = read_character(buf);
if (in_multiline_comment) {
if (c == '*') {
if (read_character(buf) == '/') {
in_multiline_comment = false;
return token_new(COMMENT, "/* ... */", 9);
}
unread_character(buf);
}
if (c != '\0') {
return scan_string(buf);
}
}
switch(c) {
case ':':
if (read_character(buf) == '=') {
return token_new(ASSIGN, ":=", 2);
}
unread_character(buf);
case '+':
return token_new(PLUS, "+", 1);
case '-':
return token_new(MINUS, "-", 1);
case '*':
return token_new(TIMES, "*", 1);
case '/':
c = read_character(buf);
if (c == '/') {
int len = 2;
while ((c = read_character(buf)) && c != '\n' && c != EOF) {
len++;
}
unread_character(buf);
return token_new(COMMENT, *buf - len, len);
} else if (c == '*' && !in_multiline_comment) {
in_multiline_comment = true;
return scan_string(buf);
}
unread_character(buf);
return token_new(DIV, "/", 1);
case '(':
return token_new(LPAREN, "(", 1);
case ')':
return token_new(RPAREN, ")", 1);
case ' ':
case '\t':
case '\n':
return scan_string(buf);
case '\0':
case EOF:
return NULL;
}
// try and parse an identifer or keyword
if (isalpha(c)) {
// check if keyword
char *keywords[] = {"read", "write"};
int keywords_len = 2;
for (int i = 0; i < keywords_len; i++) {
if (strncmp(*buf - 1, keywords[i], strlen(keywords[i])) == 0) {
*buf += strlen(keywords[i]) - 1;
return token_new(KEYWORD, keywords[i], strlen(keywords[i]));
}
}
// parse as identifer
int len = 1;
while ((c = read_character(buf)) && isalnum(c)) {
len++;
}
// unread the character that broke the while loop
unread_character(buf);
return token_new(IDENTIFIER, *buf - len, len);
}
if (isdigit(c) || c == '.') {
// parse as digit
int len = 1;
int dot_found = (c == '.');
while ((c = read_character(buf))) {
if (c == '.') {
if (dot_found) {
break;
}
dot_found = true;
} else if (!isdigit(c)) {
break;
}
len++;
}
// unread the character that broke the while loop
unread_character(buf);
return token_new(NUMBER, *buf - len, len);
}
char debug_str[2] = {c, '\0'};
return token_new(INVALID, debug_str, 1);
}
ast_t* ast_blank(token_id id)
{
return ast_token(token_new(id));
}
