static int
check_bom(int get_char(struct tok_state *),
void unget_char(int, struct tok_state *),
int set_readline(struct tok_state *, const char *),
struct tok_state *tok)
{
int ch1, ch2, ch3;
ch1 = get_char(tok);
tok->decoding_state = 1;
if (ch1 == EOF) {
return 1;
} else if (ch1 == 0xEF) {
ch2 = get_char(tok);
if (ch2 != 0xBB) {
unget_char(ch2, tok);
unget_char(ch1, tok);
return 1;
}
ch3 = get_char(tok);
if (ch3 != 0xBF) {
unget_char(ch3, tok);
unget_char(ch2, tok);
unget_char(ch1, tok);
return 1;
}
#if 0
/* Disable support for UTF-16 BOMs until a decision
is made whether this needs to be supported. */
} else if (ch1 == 0xFE) {
ch2 = get_char(tok);
if (ch2 != 0xFF) {
unget_char(ch2, tok);
unget_char(ch1, tok);
return 1;
}
if (!set_readline(tok, "utf-16-be"))
return 0;
tok->decoding_state = -1;
} else if (ch1 == 0xFF) {
ch2 = get_char(tok);
if (ch2 != 0xFE) {
unget_char(ch2, tok);
unget_char(ch1, tok);
return 1;
}
if (!set_readline(tok, "utf-16-le"))
return 0;
tok->decoding_state = -1;
#endif
} else {
unget_char(ch1, tok);
return 1;
}
if (tok->encoding != NULL)
PyMem_FREE(tok->encoding);
tok->encoding = new_string("utf-8", 5); /* resulting is in utf-8 */
return 1;
}
static int
check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok,
int set_readline(struct tok_state *, const char *))
{
char *cs;
int r = 1;
if (tok->cont_line) {
/* It's a continuation line, so it can't be a coding spec. */
tok->read_coding_spec = 1;
return 1;
}
if (!get_coding_spec(line, &cs, size, tok))
return 0;
if (!cs) {
Py_ssize_t i;
for (i = 0; i < size; i++) {
if (line[i] == '#' || line[i] == '\n' || line[i] == '\r')
break;
if (line[i] != ' ' && line[i] != '\t' && line[i] != '\014') {
/* Stop checking coding spec after a line containing
* anything except a comment. */
tok->read_coding_spec = 1;
break;
}
}
return 1;
}
tok->read_coding_spec = 1;
if (tok->encoding == NULL) {
assert(tok->decoding_state == STATE_RAW);
if (strcmp(cs, "utf-8") == 0) {
tok->encoding = cs;
} else {
r = set_readline(tok, cs);
if (r) {
tok->encoding = cs;
tok->decoding_state = STATE_NORMAL;
}
else {
PyErr_Format(PyExc_SyntaxError,
"encoding problem: %s", cs);
PyMem_FREE(cs);
}
}
} else { /* then, compare cs with BOM */
r = (strcmp(tok->encoding, cs) == 0);
if (!r)
PyErr_Format(PyExc_SyntaxError,
"encoding problem: %s with BOM", cs);
PyMem_FREE(cs);
}
return r;
}
static int
check_bom(int get_char(struct tok_state *),
void unget_char(int, struct tok_state *),
int set_readline(struct tok_state *, const char *),
struct tok_state *tok)
{
int ch = get_char(tok);
tok->decoding_state = 1;
if (ch == EOF) {
return 1;
} else if (ch == 0xEF) {
ch = get_char(tok); if (ch != 0xBB) goto NON_BOM;
ch = get_char(tok); if (ch != 0xBF) goto NON_BOM;
#if 0
/* Disable support for UTF-16 BOMs until a decision
is made whether this needs to be supported. */
} else if (ch == 0xFE) {
ch = get_char(tok); if (ch != 0xFF) goto NON_BOM;
if (!set_readline(tok, "utf-16-be")) return 0;
tok->decoding_state = -1;
} else if (ch == 0xFF) {
ch = get_char(tok); if (ch != 0xFE) goto NON_BOM;
if (!set_readline(tok, "utf-16-le")) return 0;
tok->decoding_state = -1;
#endif
} else {
unget_char(ch, tok);
return 1;
}
if (tok->encoding != NULL)
PyMem_FREE(tok->encoding);
tok->encoding = new_string("utf-8", 5); /* resulting is in utf-8 */
return 1;
NON_BOM:
/* any token beginning with '\xEF', '\xFE', '\xFF' is a bad token */
unget_char(0xFF, tok); /* XXX this will cause a syntax error */
return 1;
}
static int
check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok,
int set_readline(struct tok_state *, const char *))
{
char * cs;
int r = 1;
if (tok->cont_line)
/* It's a continuation line, so it can't be a coding spec. */
return 1;
cs = get_coding_spec(line, size);
if (cs != NULL) {
tok->read_coding_spec = 1;
if (tok->encoding == NULL) {
assert(tok->decoding_state == 1); /* raw */
if (strcmp(cs, "utf-8") == 0 ||
strcmp(cs, "iso-8859-1") == 0) {
tok->encoding = cs;
} else {
#ifdef Py_USING_UNICODE
r = set_readline(tok, cs);
if (r) {
tok->encoding = cs;
tok->decoding_state = -1;
}
else
PyMem_FREE(cs);
#else
/* Without Unicode support, we cannot
process the coding spec. Since there
won't be any Unicode literals, that
won't matter. */
PyMem_FREE(cs);
#endif
}
} else { /* then, compare cs with BOM */
r = (strcmp(tok->encoding, cs) == 0);
PyMem_FREE(cs);
}
}
if (!r) {
cs = tok->encoding;
if (!cs)
cs = "with BOM";
PyErr_Format(PyExc_SyntaxError, "encoding problem: %s", cs);
}
return r;
}
static int
check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok,
int set_readline(struct tok_state *, const char *))
{
char * cs;
int r = 1;
if (tok->cont_line) {
/* It's a continuation line, so it can't be a coding spec. */
tok->read_coding_spec = 1;
return 1;
}
cs = get_coding_spec(line, size);
if (!cs) {
Py_ssize_t i;
for (i = 0; i < size; i++) {
if (line[i] == '#' || line[i] == '\n' || line[i] == '\r')
break;
if (line[i] != ' ' && line[i] != '\t' && line[i] != '\014') {
/* Stop checking coding spec after a line containing
* anything except a comment. */
tok->read_coding_spec = 1;
break;
}
}
} else {
tok->read_coding_spec = 1;
if (tok->encoding == NULL) {
assert(tok->decoding_state == 1); /* raw */
if (strcmp(cs, "utf-8") == 0 ||
strcmp(cs, "iso-8859-1") == 0) {
tok->encoding = cs;
} else {
#ifdef Py_USING_UNICODE
r = set_readline(tok, cs);
if (r) {
tok->encoding = cs;
tok->decoding_state = -1;
}
else {
PyErr_Format(PyExc_SyntaxError,
"encoding problem: %s", cs);
PyMem_FREE(cs);
}
#else
/* Without Unicode support, we cannot
process the coding spec. Since there
won't be any Unicode literals, that
won't matter. */
PyMem_FREE(cs);
#endif
}
} else { /* then, compare cs with BOM */
r = (strcmp(tok->encoding, cs) == 0);
if (!r)
PyErr_Format(PyExc_SyntaxError,
"encoding problem: %s with BOM", cs);
PyMem_FREE(cs);
}
}
return r;
}
