static token * _token_ctrl_char(char c) { tok_type type; switch(c) { case '}': type = TOK_CL_BC; break; case ']': type = TOK_CL_BK; break; case ')': type = TOK_CL_PA; break; case '[': type = TOK_OP_BK; break; case '{': type = TOK_OP_BC; break; case '(': type = TOK_OP_PA; break; default: type = TOK_ERR; break; } return tok_new(type, 0); }
struct tok_state * PyTokenizer_FromFile(FILE *fp, const char* enc, const char *ps1, const char *ps2) { struct tok_state *tok = tok_new(); if (tok == NULL) return NULL; if ((tok->buf = (char *)PyMem_MALLOC(BUFSIZ)) == NULL) { PyTokenizer_Free(tok); return NULL; } tok->cur = tok->inp = tok->buf; tok->end = tok->buf + BUFSIZ; tok->fp = fp; tok->prompt = ps1; tok->nextprompt = ps2; if (enc != NULL) { /* Must copy encoding declaration since it gets copied into the parse tree. */ tok->encoding = PyMem_MALLOC(strlen(enc)+1); if (!tok->encoding) { PyTokenizer_Free(tok); return NULL; } strcpy(tok->encoding, enc); tok->decoding_state = STATE_NORMAL; } return tok; }
struct tok_state * PyTokenizer_FromUTF8(const char *str, int exec_input) { struct tok_state *tok = tok_new(); if (tok == NULL) return NULL; #ifndef PGEN tok->input = str = translate_newlines(str, exec_input, tok); #endif if (str == NULL) { PyTokenizer_Free(tok); return NULL; } tok->decoding_state = STATE_RAW; tok->read_coding_spec = 1; tok->enc = NULL; tok->str = str; tok->encoding = (char *)PyMem_MALLOC(6); if (!tok->encoding) { PyTokenizer_Free(tok); return NULL; } strcpy(tok->encoding, "utf-8"); /* XXX: constify members. */ tok->buf = tok->cur = tok->end = tok->inp = (char*)str; return tok; }
static token * _tok_new(lexer * lex, tok_type type, lex_state state) { token * t; if(!lex) { return 0; } t = tok_new(type, lex->tok_buf); lex->tok_buf = str_new(0); _lex_set_state(lex, state); return t; }
struct tok_state * PyTokenizer_FromString(const char *str, int exec_input) { struct tok_state *tok = tok_new(); if (tok == NULL) return NULL; str = decode_str(str, exec_input, tok); if (str == NULL) { PyTokenizer_Free(tok); return NULL; } /* XXX: constify members. */ tok->buf = tok->cur = tok->end = tok->inp = (char*)str; return tok; }
struct tok_state * PyTokenizer_FromFile(FILE *fp, char *ps1, char *ps2) { struct tok_state *tok = tok_new(); if (tok == NULL) return NULL; if ((tok->buf = (char *)PyMem_MALLOC(BUFSIZ)) == NULL) { PyTokenizer_Free(tok); return NULL; } tok->cur = tok->inp = tok->buf; tok->end = tok->buf + BUFSIZ; tok->fp = fp; tok->prompt = ps1; tok->nextprompt = ps2; return tok; }
token* tok_fromstring(enum TokenType type, const char *s, int lo, int hi) { if (!s) return NULL; token *t = tok_new(); if (!t) return NULL; int slen = hi - lo + 1; char *ss = malloc((slen + 1) * sizeof(char)); if (!ss) { free(t); return NULL; } memcpy(ss, s + lo, slen); ss[slen] = '\0'; t->type = type; t->s = ss; return t; }