// http://dev.w3.org/csswg/css-syntax/#consume-a-string-token
MediaQueryToken MediaQueryTokenizer::consumeStringTokenUntil(UChar endingCodePoint)
{
StringBuilder output;
while (true) {
UChar cc = consume();
if (cc == endingCodePoint || cc == kEndOfFileMarker) {
// The "reconsume" here deviates from the spec, but is required to avoid consuming past the EOF
if (cc == kEndOfFileMarker)
reconsume(cc);
return MediaQueryToken(StringToken, output.toString());
}
if (isNewLine(cc)) {
reconsume(cc);
return MediaQueryToken(BadStringToken);
}
if (cc == '\\') {
if (m_input.nextInputChar() == kEndOfFileMarker)
continue;
if (isNewLine(m_input.nextInputChar()))
consume();
else
output.append(consumeEscape());
} else {
output.append(cc);
}
}
}
MediaQueryToken MediaQueryTokenizer::hyphenMinus(UChar cc)
{
if (nextCharsAreNumber(cc)) {
reconsume(cc);
return consumeNumericToken();
}
if (nextCharsAreIdentifier(cc)) {
reconsume(cc);
return consumeIdentLikeToken();
}
return MediaQueryToken(DelimiterToken, cc);
}
bool CSSTokenizer::nextCharsAreIdentifier()
{
UChar first = consume();
bool areIdentifier = nextCharsAreIdentifier(first);
reconsume(first);
return areIdentifier;
}
// http://www.w3.org/TR/css3-syntax/#consume-a-name
CSSParserString CSSTokenizer::consumeName()
{
// Names without escapes get handled without allocations
for (unsigned size = 0; ; ++size) {
UChar cc = m_input.peekWithoutReplacement(size);
if (cc == '\0' || cc == '\\')
break;
if (!isNameChar(cc)) {
unsigned startOffset = m_input.offset();
m_input.advance(size);
return m_input.rangeAsCSSParserString(startOffset, size);
}
}
StringBuilder result;
while (true) {
UChar cc = consume();
if (isNameChar(cc)) {
result.append(cc);
continue;
}
if (twoCharsAreValidEscape(cc, m_input.nextInputChar())) {
result.append(consumeEscape());
continue;
}
reconsume(cc);
return registerString(result.toString());
}
}
CSSParserToken CSSTokenizer::hyphenMinus(UChar cc)
{
if (nextCharsAreNumber(cc)) {
reconsume(cc);
return consumeNumericToken();
}
if (m_input.peek(0) == '-' && m_input.peek(1) == '>') {
consume(2);
return CSSParserToken(CDCToken);
}
if (nextCharsAreIdentifier(cc)) {
reconsume(cc);
return consumeIdentLikeToken();
}
return CSSParserToken(DelimiterToken, cc);
}
bool MediaQueryTokenizer::nextCharsAreNumber()
{
UChar first = consume();
bool areNumber = nextCharsAreNumber(first);
reconsume(first);
return areNumber;
}
CSSParserToken CSSTokenizer::reverseSolidus(UChar cc)
{
if (twoCharsAreValidEscape(cc, m_input.nextInputChar())) {
reconsume(cc);
return consumeIdentLikeToken();
}
return CSSParserToken(DelimiterToken, cc);
}
CSSParserToken CSSTokenizer::plusOrFullStop(UChar cc)
{
if (nextCharsAreNumber(cc)) {
reconsume(cc);
return consumeNumericToken();
}
return CSSParserToken(DelimiterToken, cc);
}
CSSParserToken CSSTokenizer::letterU(UChar cc)
{
if (m_input.nextInputChar() == '+'
&& (isASCIIHexDigit(m_input.peek(1)) || m_input.peek(1) == '?')) {
consume();
return consumeUnicodeRange();
}
reconsume(cc);
return consumeIdentLikeToken();
}
// http://www.w3.org/TR/css3-syntax/#consume-a-name
String MediaQueryTokenizer::consumeName()
{
// FIXME: Is this as efficient as it can be?
// The possibility of escape chars mandates a copy AFAICT.
StringBuilder result;
while (true) {
UChar cc = consume();
if (isNameChar(cc)) {
result.append(cc);
continue;
}
if (twoCharsAreValidEscape(cc, m_input.nextInputChar())) {
result.append(consumeEscape());
continue;
}
reconsume(cc);
return result.toString();
}
}
// http://dev.w3.org/csswg/css-syntax/#consume-a-string-token
CSSParserToken CSSTokenizer::consumeStringTokenUntil(UChar endingCodePoint)
{
// Strings without escapes get handled without allocations
for (unsigned size = 0; ; size++) {
UChar cc = m_input.peekWithoutReplacement(size);
if (cc == endingCodePoint) {
unsigned startOffset = m_input.offset();
m_input.advance(size + 1);
return CSSParserToken(StringToken, m_input.rangeAsCSSParserString(startOffset, size));
}
if (isNewLine(cc)) {
m_input.advance(size);
return CSSParserToken(BadStringToken);
}
if (cc == '\0' || cc == '\\')
break;
}
StringBuilder output;
while (true) {
UChar cc = consume();
if (cc == endingCodePoint || cc == kEndOfFileMarker)
return CSSParserToken(StringToken, registerString(output.toString()));
if (isNewLine(cc)) {
reconsume(cc);
return CSSParserToken(BadStringToken);
}
if (cc == '\\') {
if (m_input.nextInputChar() == kEndOfFileMarker)
continue;
if (isNewLine(m_input.nextInputChar()))
consumeSingleWhitespaceIfNext(); // This handles \r\n for us
else
output.append(consumeEscape());
} else {
output.append(cc);
}
}
}
// http://dev.w3.org/csswg/css-syntax/#consume-url-token
CSSParserToken CSSTokenizer::consumeUrlToken()
{
consumeUntilNonWhitespace();
StringBuilder result;
while (true) {
UChar cc = consume();
if (cc == ')' || cc == kEndOfFileMarker) {
// The "reconsume" here deviates from the spec, but is required to avoid consuming past the EOF
if (cc == kEndOfFileMarker)
reconsume(cc);
return CSSParserToken(UrlToken, result.toString());
}
if (isHTMLSpace(cc)) {
consumeUntilNonWhitespace();
if (consumeIfNext(')') || m_input.nextInputChar() == kEndOfFileMarker)
return CSSParserToken(UrlToken, result.toString());
break;
}
if (cc == '"' || cc == '\'' || cc == '(' || isNonPrintableCodePoint(cc))
break;
if (cc == '\\') {
if (twoCharsAreValidEscape(cc, m_input.nextInputChar())) {
result.append(consumeEscape());
continue;
}
break;
}
result.append(cc);
}
consumeBadUrlRemnants();
return CSSParserToken(BadUrlToken);
}
CSSParserToken CSSTokenizer::nameStart(UChar cc)
{
reconsume(cc);
return consumeIdentLikeToken();
}
CSSParserToken CSSTokenizer::asciiDigit(UChar cc)
{
reconsume(cc);
return consumeNumericToken();
}
MediaQueryToken MediaQueryTokenizer::nameStart(UChar cc)
{
reconsume(cc);
return consumeIdentLikeToken();
}
MediaQueryToken MediaQueryTokenizer::asciiDigit(UChar cc)
{
reconsume(cc);
return consumeNumericToken();
}
