/*-------------------------------------------------------------------------
* The next token in the input stream is known to be a number; lex it.
*
* In JSON, a number consists of four parts:
*
* (1) An optional minus sign ('-').
*
* (2) Either a single '0', or a string of one or more digits that does not
* begin with a '0'.
*
* (3) An optional decimal part, consisting of a period ('.') followed by
* one or more digits. (Note: While this part can be omitted
* completely, it's not OK to have only the decimal point without
* any digits afterwards.)
*
* (4) An optional exponent part, consisting of 'e' or 'E', optionally
* followed by '+' or '-', followed by one or more digits. (Note:
* As with the decimal part, if 'e' or 'E' is present, it must be
* followed by at least one digit.)
*
* The 's' argument to this function points to the ostensible beginning
* of part 2 - i.e. the character after any optional minus sign, and the
* first character of the string if there is none.
*
*-------------------------------------------------------------------------
*/
static void
json_lex_number(JsonLexContext *lex, char *s)
{
bool error = false;
char *p;
/* Part (1): leading sign indicator. */
/* Caller already did this for us; so do nothing. */
/* Part (2): parse main digit string. */
if (*s == '0')
++s;
else if (*s >= '1' && *s <= '9')
{
do
{
++s;
} while (*s >= '0' && *s <= '9');
}
else
error = true;
/* Part (3): parse optional decimal portion. */
if (*s == '.')
{
++s;
if (*s < '0' && *s > '9')
error = true;
else
{
do
{
++s;
} while (*s >= '0' && *s <= '9');
}
}
/* Part (4): parse optional exponent. */
if (*s == 'e' || *s == 'E')
{
++s;
if (*s == '+' || *s == '-')
++s;
if (*s < '0' && *s > '9')
error = true;
else
{
do
{
++s;
} while (*s >= '0' && *s <= '9');
}
}
/* Check for trailing garbage. */
for (p = s; (*p >= 'a' && *p <= 'z') || (*p >= 'A' && *p <= 'Z')
|| (*p >= '0' && *p <= '9') || *p == '_' || IS_HIGHBIT_SET(*p); ++p)
;
lex->token_terminator = p;
if (p > s || error)
report_invalid_token(lex);
}
/*
* Lex one token from the input stream.
*/
static void
json_lex(JsonLexContext *lex)
{
char *s;
/* Skip leading whitespace. */
s = lex->token_terminator;
while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r')
{
if (*s == '\n')
++lex->line_number;
++s;
}
lex->token_start = s;
/* Determine token type. */
if (strchr("{}[],:", s[0]))
{
/* strchr() doesn't return false on a NUL input. */
if (s[0] == '\0')
{
/* End of string. */
lex->token_start = NULL;
lex->token_terminator = NULL;
}
else
{
/* Single-character token, some kind of punctuation mark. */
lex->token_terminator = s + 1;
}
lex->token_type = JSON_VALUE_INVALID;
}
else if (*s == '"')
{
/* String. */
json_lex_string(lex);
lex->token_type = JSON_VALUE_STRING;
}
else if (*s == '-')
{
/* Negative number. */
json_lex_number(lex, s + 1);
lex->token_type = JSON_VALUE_NUMBER;
}
else if (*s >= '0' && *s <= '9')
{
/* Positive number. */
json_lex_number(lex, s);
lex->token_type = JSON_VALUE_NUMBER;
}
else
{
char *p;
/*
* We're not dealing with a string, number, legal punctuation mark,
* or end of string. The only legal tokens we might find here are
* true, false, and null, but for error reporting purposes we scan
* until we see a non-alphanumeric character. That way, we can report
* the whole word as an unexpected token, rather than just some
* unintuitive prefix thereof.
*/
for (p = s; (*p >= 'a' && *p <= 'z') || (*p >= 'A' && *p <= 'Z')
|| (*p >= '0' && *p <= '9') || *p == '_' || IS_HIGHBIT_SET(*p);
++p)
;
/*
* We got some sort of unexpected punctuation or an otherwise
* unexpected character, so just complain about that one character.
*/
if (p == s)
{
lex->token_terminator = s + 1;
report_invalid_token(lex);
}
/*
* We've got a real alphanumeric token here. If it happens to be
* true, false, or null, all is well. If not, error out.
*/
lex->token_terminator = p;
if (p - s == 4)
{
if (memcmp(s, "true", 4) == 0)
lex->token_type = JSON_VALUE_TRUE;
else if (memcmp(s, "null", 4) == 0)
lex->token_type = JSON_VALUE_NULL;
else
report_invalid_token(lex);
}
else if (p - s == 5 && memcmp(s, "false", 5) == 0)
lex->token_type = JSON_VALUE_FALSE;
else
report_invalid_token(lex);
}
}
/*
* The next token in the input stream is known to be a string; lex it.
*/
static void
json_lex_string(JsonLexContext *lex)
{
char *s = lex->token_start + 1;
for (s = lex->token_start + 1; *s != '"'; ++s)
{
/* Per RFC4627, these characters MUST be escaped. */
if (*s < 32)
{
/* A NUL byte marks the (premature) end of the string. */
if (*s == '\0')
{
lex->token_terminator = s;
report_invalid_token(lex);
}
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type json"),
errdetail_internal("line %d: Character \"%c\" must be escaped.",
lex->line_number, *s)));
}
else if (*s == '\\')
{
/* OK, we have an escape character. */
++s;
if (*s == '\0')
{
lex->token_terminator = s;
report_invalid_token(lex);
}
else if (*s == 'u')
{
int i;
int ch = 0;
for (i = 1; i <= 4; ++i)
{
if (s[i] == '\0')
{
lex->token_terminator = s + i;
report_invalid_token(lex);
}
else if (s[i] >= '0' && s[i] <= '9')
ch = (ch * 16) + (s[i] - '0');
else if (s[i] >= 'a' && s[i] <= 'f')
ch = (ch * 16) + (s[i] - 'a') + 10;
else if (s[i] >= 'A' && s[i] <= 'F')
ch = (ch * 16) + (s[i] - 'A') + 10;
else
{
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type json"),
errdetail_internal("line %d: \"\\u\" must be followed by four hexadecimal digits.",
lex->line_number)));
}
}
/* Account for the four additional bytes we just parsed. */
s += 4;
}
else if (!strchr("\"\\/bfnrt", *s))
{
/* Error out. */
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type json"),
errdetail_internal("line %d: Invalid escape \"\\%s\".",
lex->line_number, extract_mb_char(s))));
}
}
}
/* Hooray, we found the end of the string! */
lex->token_terminator = s + 1;
}
/*-------------------------------------------------------------------------
* The next token in the input stream is known to be a number; lex it.
*
* In JSON, a number consists of four parts:
*
* (1) An optional minus sign ('-').
*
* (2) Either a single '0', or a string of one or more digits that does not
* begin with a '0'.
*
* (3) An optional decimal part, consisting of a period ('.') followed by
* one or more digits. (Note: While this part can be omitted
* completely, it's not OK to have only the decimal point without
* any digits afterwards.)
*
* (4) An optional exponent part, consisting of 'e' or 'E', optionally
* followed by '+' or '-', followed by one or more digits. (Note:
* As with the decimal part, if 'e' or 'E' is present, it must be
* followed by at least one digit.)
*
* The 's' argument to this function points to the ostensible beginning
* of part 2 - i.e. the character after any optional minus sign, and the
* first character of the string if there is none.
*
*-------------------------------------------------------------------------
*/
static inline void
json_lex_number(JsonLexContext *lex, char *s)
{
bool error = false;
char *p;
int len;
len = s - lex->input;
/* Part (1): leading sign indicator. */
/* Caller already did this for us; so do nothing. */
/* Part (2): parse main digit string. */
if (*s == '0')
{
s++;
len++;
}
else if (*s >= '1' && *s <= '9')
{
do
{
s++;
len++;
} while (len < lex->input_length && *s >= '0' && *s <= '9');
}
else
error = true;
/* Part (3): parse optional decimal portion. */
if (len < lex->input_length && *s == '.')
{
s++;
len++;
if (len == lex->input_length || *s < '0' || *s > '9')
error = true;
else
{
do
{
s++;
len++;
} while (len < lex->input_length && *s >= '0' && *s <= '9');
}
}
/* Part (4): parse optional exponent. */
if (len < lex->input_length && (*s == 'e' || *s == 'E'))
{
s++;
len++;
if (len < lex->input_length && (*s == '+' || *s == '-'))
{
s++;
len++;
}
if (len == lex->input_length || *s < '0' || *s > '9')
error = true;
else
{
do
{
s++;
len++;
} while (len < lex->input_length && *s >= '0' && *s <= '9');
}
}
/*
* Check for trailing garbage. As in json_lex(), any alphanumeric stuff
* here should be considered part of the token for error-reporting
* purposes.
*/
for (p = s; len < lex->input_length && JSON_ALPHANUMERIC_CHAR(*p); p++, len++)
error = true;
lex->prev_token_terminator = lex->token_terminator;
lex->token_terminator = p;
if (error)
report_invalid_token(lex);
}
/*
* The next token in the input stream is known to be a string; lex it.
*/
static inline void
json_lex_string(JsonLexContext *lex)
{
char *s;
int len;
int hi_surrogate = -1;
if (lex->strval != NULL)
resetStringInfo(lex->strval);
Assert(lex->input_length > 0);
s = lex->token_start;
len = lex->token_start - lex->input;
for (;;)
{
s++;
len++;
/* Premature end of the string. */
if (len >= lex->input_length)
{
lex->token_terminator = s;
report_invalid_token(lex);
}
else if (*s == '"')
break;
else if ((unsigned char) *s < 32)
{
/* Per RFC4627, these characters MUST be escaped. */
/* Since *s isn't printable, exclude it from the context string */
lex->token_terminator = s;
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type json"),
errdetail("Character with value 0x%02x must be escaped.",
(unsigned char) *s),
report_json_context(lex)));
}
else if (*s == '\\')
{
/* OK, we have an escape character. */
s++;
len++;
if (len >= lex->input_length)
{
lex->token_terminator = s;
report_invalid_token(lex);
}
else if (*s == 'u')
{
int i;
int ch = 0;
for (i = 1; i <= 4; i++)
{
s++;
len++;
if (len >= lex->input_length)
{
lex->token_terminator = s;
report_invalid_token(lex);
}
else if (*s >= '0' && *s <= '9')
ch = (ch * 16) + (*s - '0');
else if (*s >= 'a' && *s <= 'f')
ch = (ch * 16) + (*s - 'a') + 10;
else if (*s >= 'A' && *s <= 'F')
ch = (ch * 16) + (*s - 'A') + 10;
else
{
lex->token_terminator = s + pg_mblen(s);
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type json"),
errdetail("\"\\u\" must be followed by four hexadecimal digits."),
report_json_context(lex)));
}
}
if (lex->strval != NULL)
{
char utf8str[5];
int utf8len;
if (ch >= 0xd800 && ch <= 0xdbff)
{
if (hi_surrogate != -1)
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type json"),
errdetail("Unicode high surrogate must not follow a high surrogate."),
report_json_context(lex)));
hi_surrogate = (ch & 0x3ff) << 10;
continue;
}
else if (ch >= 0xdc00 && ch <= 0xdfff)
{
if (hi_surrogate == -1)
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type json"),
errdetail("Unicode low surrogate must follow a high surrogate."),
report_json_context(lex)));
ch = 0x10000 + hi_surrogate + (ch & 0x3ff);
hi_surrogate = -1;
}
if (hi_surrogate != -1)
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type json"),
errdetail("Unicode low surrogate must follow a high surrogate."),
report_json_context(lex)));
/*
* For UTF8, replace the escape sequence by the actual utf8
* character in lex->strval. Do this also for other encodings
* if the escape designates an ASCII character, otherwise
* raise an error. We don't ever unescape a \u0000, since that
* would result in an impermissible nul byte.
*/
if (ch == 0)
{
appendStringInfoString(lex->strval, "\\u0000");
}
else if (GetDatabaseEncoding() == PG_UTF8)
{
unicode_to_utf8(ch, (unsigned char *) utf8str);
utf8len = pg_utf_mblen((unsigned char *) utf8str);
appendBinaryStringInfo(lex->strval, utf8str, utf8len);
}
else if (ch <= 0x007f)
{
/*
* This is the only way to designate things like a form feed
* character in JSON, so it's useful in all encodings.
*/
appendStringInfoChar(lex->strval, (char) ch);
}
else
{
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type json"),
errdetail("Unicode escape values cannot be used for code point values above 007F when the server encoding is not UTF8."),
report_json_context(lex)));
}
}
}
else if (lex->strval != NULL)
{
if (hi_surrogate != -1)
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type json"),
errdetail("Unicode low surrogate must follow a high surrogate."),
report_json_context(lex)));
switch (*s)
{
case '"':
case '\\':
case '/':
appendStringInfoChar(lex->strval, *s);
break;
case 'b':
appendStringInfoChar(lex->strval, '\b');
break;
case 'f':
appendStringInfoChar(lex->strval, '\f');
break;
case 'n':
appendStringInfoChar(lex->strval, '\n');
break;
case 'r':
appendStringInfoChar(lex->strval, '\r');
break;
case 't':
appendStringInfoChar(lex->strval, '\t');
break;
default:
/* Not a valid string escape, so error out. */
lex->token_terminator = s + pg_mblen(s);
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type json"),
errdetail("Escape sequence \"\\%s\" is invalid.",
extract_mb_char(s)),
report_json_context(lex)));
}
}
else if (strchr("\"\\/bfnrt", *s) == NULL)
{
/*
* Simpler processing if we're not bothered about de-escaping
*
* It's very tempting to remove the strchr() call here and
* replace it with a switch statement, but testing so far has
* shown it's not a performance win.
*/
lex->token_terminator = s + pg_mblen(s);
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type json"),
errdetail("Escape sequence \"\\%s\" is invalid.",
extract_mb_char(s)),
report_json_context(lex)));
}
}
else if (lex->strval != NULL)
{
if (hi_surrogate != -1)
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type json"),
errdetail("Unicode low surrogate must follow a high surrogate."),
report_json_context(lex)));
appendStringInfoChar(lex->strval, *s);
}
}
if (hi_surrogate != -1)
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type json"),
errdetail("Unicode low surrogate must follow a high surrogate."),
report_json_context(lex)));
/* Hooray, we found the end of the string! */
lex->prev_token_terminator = lex->token_terminator;
lex->token_terminator = s + 1;
}
/*
* Lex one token from the input stream.
*/
static inline void
json_lex(JsonLexContext *lex)
{
char *s;
int len;
/* Skip leading whitespace. */
s = lex->token_terminator;
len = s - lex->input;
while (len < lex->input_length &&
(*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r'))
{
if (*s == '\n')
++lex->line_number;
++s;
++len;
}
lex->token_start = s;
/* Determine token type. */
if (len >= lex->input_length)
{
lex->token_start = NULL;
lex->prev_token_terminator = lex->token_terminator;
lex->token_terminator = s;
lex->token_type = JSON_TOKEN_END;
}
else
switch (*s)
{
/* Single-character token, some kind of punctuation mark. */
case '{':
lex->prev_token_terminator = lex->token_terminator;
lex->token_terminator = s + 1;
lex->token_type = JSON_TOKEN_OBJECT_START;
break;
case '}':
lex->prev_token_terminator = lex->token_terminator;
lex->token_terminator = s + 1;
lex->token_type = JSON_TOKEN_OBJECT_END;
break;
case '[':
lex->prev_token_terminator = lex->token_terminator;
lex->token_terminator = s + 1;
lex->token_type = JSON_TOKEN_ARRAY_START;
break;
case ']':
lex->prev_token_terminator = lex->token_terminator;
lex->token_terminator = s + 1;
lex->token_type = JSON_TOKEN_ARRAY_END;
break;
case ',':
lex->prev_token_terminator = lex->token_terminator;
lex->token_terminator = s + 1;
lex->token_type = JSON_TOKEN_COMMA;
break;
case ':':
lex->prev_token_terminator = lex->token_terminator;
lex->token_terminator = s + 1;
lex->token_type = JSON_TOKEN_COLON;
break;
case '"':
/* string */
json_lex_string(lex);
lex->token_type = JSON_TOKEN_STRING;
break;
case '-':
/* Negative number. */
json_lex_number(lex, s + 1);
lex->token_type = JSON_TOKEN_NUMBER;
break;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
/* Positive number. */
json_lex_number(lex, s);
lex->token_type = JSON_TOKEN_NUMBER;
break;
default:
{
char *p;
/*
* We're not dealing with a string, number, legal
* punctuation mark, or end of string. The only legal
* tokens we might find here are true, false, and null,
* but for error reporting purposes we scan until we see a
* non-alphanumeric character. That way, we can report
* the whole word as an unexpected token, rather than just
* some unintuitive prefix thereof.
*/
for (p = s; p - s < lex->input_length - len && JSON_ALPHANUMERIC_CHAR(*p); p++)
/* skip */ ;
/*
* We got some sort of unexpected punctuation or an
* otherwise unexpected character, so just complain about
* that one character.
*/
if (p == s)
{
lex->prev_token_terminator = lex->token_terminator;
lex->token_terminator = s + 1;
report_invalid_token(lex);
}
/*
* We've got a real alphanumeric token here. If it
* happens to be true, false, or null, all is well. If
* not, error out.
*/
lex->prev_token_terminator = lex->token_terminator;
lex->token_terminator = p;
if (p - s == 4)
{
if (memcmp(s, "true", 4) == 0)
lex->token_type = JSON_TOKEN_TRUE;
else if (memcmp(s, "null", 4) == 0)
lex->token_type = JSON_TOKEN_NULL;
else
report_invalid_token(lex);
}
else if (p - s == 5 && memcmp(s, "false", 5) == 0)
lex->token_type = JSON_TOKEN_FALSE;
else
report_invalid_token(lex);
}
} /* end of switch */
}
/*
* The next token in the input stream is known to be a string; lex it.
*/
static void
json_lex_string(JsonLexContext *lex)
{
char *s;
for (s = lex->token_start + 1; *s != '"'; s++)
{
/* Per RFC4627, these characters MUST be escaped. */
if ((unsigned char) *s < 32)
{
/* A NUL byte marks the (premature) end of the string. */
if (*s == '\0')
{
lex->token_terminator = s;
report_invalid_token(lex);
}
/* Since *s isn't printable, exclude it from the context string */
lex->token_terminator = s;
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type json"),
errdetail("Character with value 0x%02x must be escaped.",
(unsigned char) *s),
report_json_context(lex)));
}
else if (*s == '\\')
{
/* OK, we have an escape character. */
s++;
if (*s == '\0')
{
lex->token_terminator = s;
report_invalid_token(lex);
}
else if (*s == 'u')
{
int i;
int ch = 0;
for (i = 1; i <= 4; i++)
{
s++;
if (*s == '\0')
{
lex->token_terminator = s;
report_invalid_token(lex);
}
else if (*s >= '0' && *s <= '9')
ch = (ch * 16) + (*s - '0');
else if (*s >= 'a' && *s <= 'f')
ch = (ch * 16) + (*s - 'a') + 10;
else if (*s >= 'A' && *s <= 'F')
ch = (ch * 16) + (*s - 'A') + 10;
else
{
lex->token_terminator = s + pg_mblen(s);
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type json"),
errdetail("\"\\u\" must be followed by four hexadecimal digits."),
report_json_context(lex)));
}
}
}
else if (strchr("\"\\/bfnrt", *s) == NULL)
{
/* Not a valid string escape, so error out. */
lex->token_terminator = s + pg_mblen(s);
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type json"),
errdetail("Escape sequence \"\\%s\" is invalid.",
extract_mb_char(s)),
report_json_context(lex)));
}
}
}
/* Hooray, we found the end of the string! */
lex->token_terminator = s + 1;
}
/*
* Lex one token from the input stream.
*/
static void
json_lex(JsonLexContext *lex)
{
char *s;
/* Skip leading whitespace. */
s = lex->token_terminator;
while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r')
s++;
lex->token_start = s;
/* Determine token type. */
if (strchr("{}[],:", s[0]) != NULL)
{
/* strchr() is willing to match a zero byte, so test for that. */
if (s[0] == '\0')
{
/* End of string. */
lex->token_start = NULL;
lex->token_terminator = s;
}
else
{
/* Single-character token, some kind of punctuation mark. */
lex->token_terminator = s + 1;
}
lex->token_type = JSON_VALUE_INVALID;
}
else if (*s == '"')
{
/* String. */
json_lex_string(lex);
lex->token_type = JSON_VALUE_STRING;
}
else if (*s == '-')
{
/* Negative number. */
json_lex_number(lex, s + 1);
lex->token_type = JSON_VALUE_NUMBER;
}
else if (*s >= '0' && *s <= '9')
{
/* Positive number. */
json_lex_number(lex, s);
lex->token_type = JSON_VALUE_NUMBER;
}
else
{
char *p;
/*
* We're not dealing with a string, number, legal punctuation mark, or
* end of string. The only legal tokens we might find here are true,
* false, and null, but for error reporting purposes we scan until we
* see a non-alphanumeric character. That way, we can report the
* whole word as an unexpected token, rather than just some
* unintuitive prefix thereof.
*/
for (p = s; JSON_ALPHANUMERIC_CHAR(*p); p++)
/* skip */ ;
if (p == s)
{
/*
* We got some sort of unexpected punctuation or an otherwise
* unexpected character, so just complain about that one
* character. (It can't be multibyte because the above loop
* will advance over any multibyte characters.)
*/
lex->token_terminator = s + 1;
report_invalid_token(lex);
}
/*
* We've got a real alphanumeric token here. If it happens to be
* true, false, or null, all is well. If not, error out.
*/
lex->token_terminator = p;
if (p - s == 4)
{
if (memcmp(s, "true", 4) == 0)
lex->token_type = JSON_VALUE_TRUE;
else if (memcmp(s, "null", 4) == 0)
lex->token_type = JSON_VALUE_NULL;
else
report_invalid_token(lex);
}
else if (p - s == 5 && memcmp(s, "false", 5) == 0)
lex->token_type = JSON_VALUE_FALSE;
else
report_invalid_token(lex);
}
}