static token * _token_ctrl_char(char c) {
tok_type type;
switch(c) {
case '}':
type = TOK_CL_BC;
break;
case ']':
type = TOK_CL_BK;
break;
case ')':
type = TOK_CL_PA;
break;
case '[':
type = TOK_OP_BK;
break;
case '{':
type = TOK_OP_BC;
break;
case '(':
type = TOK_OP_PA;
break;
default:
type = TOK_ERR;
break;
}
return tok_new(type, 0);
}
struct tok_state *
PyTokenizer_FromFile(FILE *fp, const char* enc,
const char *ps1, const char *ps2)
{
struct tok_state *tok = tok_new();
if (tok == NULL)
return NULL;
if ((tok->buf = (char *)PyMem_MALLOC(BUFSIZ)) == NULL) {
PyTokenizer_Free(tok);
return NULL;
}
tok->cur = tok->inp = tok->buf;
tok->end = tok->buf + BUFSIZ;
tok->fp = fp;
tok->prompt = ps1;
tok->nextprompt = ps2;
if (enc != NULL) {
/* Must copy encoding declaration since it
gets copied into the parse tree. */
tok->encoding = PyMem_MALLOC(strlen(enc)+1);
if (!tok->encoding) {
PyTokenizer_Free(tok);
return NULL;
}
strcpy(tok->encoding, enc);
tok->decoding_state = STATE_NORMAL;
}
return tok;
}
struct tok_state *
PyTokenizer_FromUTF8(const char *str, int exec_input)
{
struct tok_state *tok = tok_new();
if (tok == NULL)
return NULL;
#ifndef PGEN
tok->input = str = translate_newlines(str, exec_input, tok);
#endif
if (str == NULL) {
PyTokenizer_Free(tok);
return NULL;
}
tok->decoding_state = STATE_RAW;
tok->read_coding_spec = 1;
tok->enc = NULL;
tok->str = str;
tok->encoding = (char *)PyMem_MALLOC(6);
if (!tok->encoding) {
PyTokenizer_Free(tok);
return NULL;
}
strcpy(tok->encoding, "utf-8");
/* XXX: constify members. */
tok->buf = tok->cur = tok->end = tok->inp = (char*)str;
return tok;
}
static token * _tok_new(lexer * lex, tok_type type, lex_state state) {
token * t;
if(!lex) { return 0; }
t = tok_new(type, lex->tok_buf);
lex->tok_buf = str_new(0);
_lex_set_state(lex, state);
return t;
}
struct tok_state *
PyTokenizer_FromString(const char *str, int exec_input)
{
struct tok_state *tok = tok_new();
if (tok == NULL)
return NULL;
str = decode_str(str, exec_input, tok);
if (str == NULL) {
PyTokenizer_Free(tok);
return NULL;
}
/* XXX: constify members. */
tok->buf = tok->cur = tok->end = tok->inp = (char*)str;
return tok;
}
struct tok_state *
PyTokenizer_FromFile(FILE *fp, char *ps1, char *ps2)
{
struct tok_state *tok = tok_new();
if (tok == NULL)
return NULL;
if ((tok->buf = (char *)PyMem_MALLOC(BUFSIZ)) == NULL) {
PyTokenizer_Free(tok);
return NULL;
}
tok->cur = tok->inp = tok->buf;
tok->end = tok->buf + BUFSIZ;
tok->fp = fp;
tok->prompt = ps1;
tok->nextprompt = ps2;
return tok;
}
token* tok_fromstring(enum TokenType type, const char *s, int lo, int hi) {
if (!s)
return NULL;
token *t = tok_new();
if (!t)
return NULL;
int slen = hi - lo + 1;
char *ss = malloc((slen + 1) * sizeof(char));
if (!ss) {
free(t);
return NULL;
}
memcpy(ss, s + lo, slen);
ss[slen] = '\0';
t->type = type;
t->s = ss;
return t;
}
