void test_buffer_walk(void) {
const char *b = "this is a small test buffer";
scanner_t *s = scanner_init(b);
assert(scanner_peek(s) == 't');
assert(scanner_advance(s) == 't');
assert(scanner_advance(s) == 'h');
assert(scanner_advance(s) == 'i');
assert(scanner_advance(s) == 's');
assert(scanner_peek(s) == ' ');
assert(scanner_current(s) == 's');
assert(scanner_advance(s) == ' ');
assert(scanner_backup(s) == 's');
char *xthis = (char*)scanner_accept(s, (acceptfn)test_accept_fn);
assert(strcmp(xthis, "this") == 0);
free(xthis);
size_t i = 4;
char c;
while ((c = scanner_advance(s)))
assert(c == b[i++]);
assert(strlen(b) == i);
scanner_destroy(s);
}
/* get the next token out of a scanner */
token_t * lexer_nextitem(scanner_t *s) {
/* try to match longest tokens first */
static lexcomp_t (*tokenizers[])(scanner_t*) = {
tokenize_text,
tokenize_identifier,
tokenize_number,
tokenize_bitops,
tokenize_relops,
tokenize_mathops,
tokenize_miscops,
};
lexcomp_t lc;
size_t i;
/* consume all whitespace */
while (is_white(scanner_advance(s)));
scanner_backup(s);
scanner_ignore(s);
if (scanner_peek(s) == 0)
return token_init(tokStackEmpty, "");
for (i = 0; i < sizeof(tokenizers)/sizeof(tokenizers[0]); i++) {
if ((lc = tokenizers[i](s)) != tokNoMatch) {
token_t *t = (token_t*)scanner_accept(s, (acceptfn)tok_maker);
t->lexcomp = lc;
return t;
}
}
return token_init(tokNoMatch, "");
}
void scanner_init(Scanner *scn, FILE *f)
{
scn->f = f;
scn->line = 1;
scn->ch = BEG_OF_FILE;
scn->error = 0;
scn->tok.id = TOK_ERROR;
scn->tok.ival = 0;
scn->tok.fval = 0.0;
scn->tok.sval = 0;
scanner_accept(scn);
}
void test_file_walk(const char *file) {
char buf[3008];
FILE *f = fopen(file, "rb");
int i, j, n = fread(buf, sizeof(char), sizeof(buf), f);
fclose(f);
scanner_t *s = scanner_init_file(file);
for (j = 0; j < n; j+=16) {
for (i = 0; i < 16 && i+j < n; i++)
assert(scanner_advance(s) == buf[j+i]);
char *tok = (char*)scanner_accept(s, (acceptfn)test_accept_fn);
assert(strncmp(tok, buf+j, 16) == 0);
free(tok);
}
scanner_destroy(s);
}
/* accept current token */
static void accept(Parser *p)
{
scanner_accept(p->scn);
}