/* * Lex one token from the input stream. */ static inline void json_lex(JsonLexContext *lex) { char *s; int len; /* Skip leading whitespace. */ s = lex->token_terminator; len = s - lex->input; while (len < lex->input_length && (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r')) { if (*s == '\n') ++lex->line_number; ++s; ++len; } lex->token_start = s; /* Determine token type. */ if (len >= lex->input_length) { lex->token_start = NULL; lex->prev_token_terminator = lex->token_terminator; lex->token_terminator = s; lex->token_type = JSON_TOKEN_END; } else switch (*s) { /* Single-character token, some kind of punctuation mark. */ case '{': lex->prev_token_terminator = lex->token_terminator; lex->token_terminator = s + 1; lex->token_type = JSON_TOKEN_OBJECT_START; break; case '}': lex->prev_token_terminator = lex->token_terminator; lex->token_terminator = s + 1; lex->token_type = JSON_TOKEN_OBJECT_END; break; case '[': lex->prev_token_terminator = lex->token_terminator; lex->token_terminator = s + 1; lex->token_type = JSON_TOKEN_ARRAY_START; break; case ']': lex->prev_token_terminator = lex->token_terminator; lex->token_terminator = s + 1; lex->token_type = JSON_TOKEN_ARRAY_END; break; case ',': lex->prev_token_terminator = lex->token_terminator; lex->token_terminator = s + 1; lex->token_type = JSON_TOKEN_COMMA; break; case ':': lex->prev_token_terminator = lex->token_terminator; lex->token_terminator = s + 1; lex->token_type = JSON_TOKEN_COLON; break; case '"': /* string */ json_lex_string(lex); lex->token_type = JSON_TOKEN_STRING; break; case '-': /* Negative number. */ json_lex_number(lex, s + 1); lex->token_type = JSON_TOKEN_NUMBER; break; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': /* Positive number. */ json_lex_number(lex, s); lex->token_type = JSON_TOKEN_NUMBER; break; default: { char *p; /* * We're not dealing with a string, number, legal * punctuation mark, or end of string. The only legal * tokens we might find here are true, false, and null, * but for error reporting purposes we scan until we see a * non-alphanumeric character. That way, we can report * the whole word as an unexpected token, rather than just * some unintuitive prefix thereof. */ for (p = s; p - s < lex->input_length - len && JSON_ALPHANUMERIC_CHAR(*p); p++) /* skip */ ; /* * We got some sort of unexpected punctuation or an * otherwise unexpected character, so just complain about * that one character. */ if (p == s) { lex->prev_token_terminator = lex->token_terminator; lex->token_terminator = s + 1; report_invalid_token(lex); } /* * We've got a real alphanumeric token here. If it * happens to be true, false, or null, all is well. If * not, error out. */ lex->prev_token_terminator = lex->token_terminator; lex->token_terminator = p; if (p - s == 4) { if (memcmp(s, "true", 4) == 0) lex->token_type = JSON_TOKEN_TRUE; else if (memcmp(s, "null", 4) == 0) lex->token_type = JSON_TOKEN_NULL; else report_invalid_token(lex); } else if (p - s == 5 && memcmp(s, "false", 5) == 0) lex->token_type = JSON_TOKEN_FALSE; else report_invalid_token(lex); } } /* end of switch */ }
/*------------------------------------------------------------------------- * The next token in the input stream is known to be a number; lex it. * * In JSON, a number consists of four parts: * * (1) An optional minus sign ('-'). * * (2) Either a single '0', or a string of one or more digits that does not * begin with a '0'. * * (3) An optional decimal part, consisting of a period ('.') followed by * one or more digits. (Note: While this part can be omitted * completely, it's not OK to have only the decimal point without * any digits afterwards.) * * (4) An optional exponent part, consisting of 'e' or 'E', optionally * followed by '+' or '-', followed by one or more digits. (Note: * As with the decimal part, if 'e' or 'E' is present, it must be * followed by at least one digit.) * * The 's' argument to this function points to the ostensible beginning * of part 2 - i.e. the character after any optional minus sign, and the * first character of the string if there is none. * *------------------------------------------------------------------------- */ static inline void json_lex_number(JsonLexContext *lex, char *s) { bool error = false; char *p; int len; len = s - lex->input; /* Part (1): leading sign indicator. */ /* Caller already did this for us; so do nothing. */ /* Part (2): parse main digit string. */ if (*s == '0') { s++; len++; } else if (*s >= '1' && *s <= '9') { do { s++; len++; } while (len < lex->input_length && *s >= '0' && *s <= '9'); } else error = true; /* Part (3): parse optional decimal portion. */ if (len < lex->input_length && *s == '.') { s++; len++; if (len == lex->input_length || *s < '0' || *s > '9') error = true; else { do { s++; len++; } while (len < lex->input_length && *s >= '0' && *s <= '9'); } } /* Part (4): parse optional exponent. */ if (len < lex->input_length && (*s == 'e' || *s == 'E')) { s++; len++; if (len < lex->input_length && (*s == '+' || *s == '-')) { s++; len++; } if (len == lex->input_length || *s < '0' || *s > '9') error = true; else { do { s++; len++; } while (len < lex->input_length && *s >= '0' && *s <= '9'); } } /* * Check for trailing garbage. As in json_lex(), any alphanumeric stuff * here should be considered part of the token for error-reporting * purposes. */ for (p = s; len < lex->input_length && JSON_ALPHANUMERIC_CHAR(*p); p++, len++) error = true; lex->prev_token_terminator = lex->token_terminator; lex->token_terminator = p; if (error) report_invalid_token(lex); }
/* * Lex one token from the input stream. */ static void json_lex(JsonLexContext *lex) { char *s; /* Skip leading whitespace. */ s = lex->token_terminator; while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++; lex->token_start = s; /* Determine token type. */ if (strchr("{}[],:", s[0]) != NULL) { /* strchr() is willing to match a zero byte, so test for that. */ if (s[0] == '\0') { /* End of string. */ lex->token_start = NULL; lex->token_terminator = s; } else { /* Single-character token, some kind of punctuation mark. */ lex->token_terminator = s + 1; } lex->token_type = JSON_VALUE_INVALID; } else if (*s == '"') { /* String. */ json_lex_string(lex); lex->token_type = JSON_VALUE_STRING; } else if (*s == '-') { /* Negative number. */ json_lex_number(lex, s + 1); lex->token_type = JSON_VALUE_NUMBER; } else if (*s >= '0' && *s <= '9') { /* Positive number. */ json_lex_number(lex, s); lex->token_type = JSON_VALUE_NUMBER; } else { char *p; /* * We're not dealing with a string, number, legal punctuation mark, or * end of string. The only legal tokens we might find here are true, * false, and null, but for error reporting purposes we scan until we * see a non-alphanumeric character. That way, we can report the * whole word as an unexpected token, rather than just some * unintuitive prefix thereof. */ for (p = s; JSON_ALPHANUMERIC_CHAR(*p); p++) /* skip */ ; if (p == s) { /* * We got some sort of unexpected punctuation or an otherwise * unexpected character, so just complain about that one * character. (It can't be multibyte because the above loop * will advance over any multibyte characters.) */ lex->token_terminator = s + 1; report_invalid_token(lex); } /* * We've got a real alphanumeric token here. If it happens to be * true, false, or null, all is well. If not, error out. */ lex->token_terminator = p; if (p - s == 4) { if (memcmp(s, "true", 4) == 0) lex->token_type = JSON_VALUE_TRUE; else if (memcmp(s, "null", 4) == 0) lex->token_type = JSON_VALUE_NULL; else report_invalid_token(lex); } else if (p - s == 5 && memcmp(s, "false", 5) == 0) lex->token_type = JSON_VALUE_FALSE; else report_invalid_token(lex); } }