Array f_token_get_all(CStrRef source) {
Scanner scanner(source.data(), source.size(),
Scanner::AllowShortTags | Scanner::ReturnAllTokens);
ScannerToken tok;
Location loc;
int tokid;
Array res;
while ((tokid = scanner.getNextToken(tok, loc))) {
if (tokid < 256) {
res.append(String::FromChar((char)tokid));
} else {
Array p = CREATE_VECTOR3(tokid, String(tok.text()), loc.line0);
res.append(p);
}
}
return res;
}
Array f_token_get_all(const String& source) {
Scanner scanner(source.data(), source.size(),
RuntimeOption::GetScannerType() | Scanner::ReturnAllTokens);
ScannerToken tok;
Location loc;
int tokid;
Array res;
while ((tokid = scanner.getNextToken(tok, loc))) {
if (tokid < 256) {
res.append(String::FromChar((char)tokid));
} else {
Array p = make_packed_array(tokid, String(tok.text()), loc.line0);
res.append(p);
}
}
return res;
}
Array HHVM_FUNCTION(token_get_all, const String& source) {
Scanner scanner(source.data(), source.size(),
RuntimeOption::GetScannerType() | Scanner::ReturnAllTokens);
ScannerToken tok;
Location loc;
int tokid;
Array res = Array::Create();
while ((tokid = scanner.getNextToken(tok, loc))) {
loop_start: // For after seeing a T_INLINE_HTML, see below
if (tokid < 256) {
res.append(String::FromChar((char)tokid));
} else {
String value;
const int tokVal = get_user_token_id(tokid);
if (tokVal == UserTokenId_T_XHP_LABEL) {
value = String(":" + tok.text());
} else if (tokVal == UserTokenId_T_XHP_CATEGORY_LABEL) {
value = String("%" + tok.text());
} else if (tokVal == UserTokenId_T_INLINE_HTML) {
// Consecutive T_INLINE_HTML tokens should be merged together to
// match Zend behaviour.
value = String(tok.text());
int line = loc.line0;
tokid = scanner.getNextToken(tok, loc);
while (tokid == T_INLINE_HTML) {
value += String(tok.text());
tokid = scanner.getNextToken(tok, loc);
}
Array p = make_packed_array(
tokVal,
value,
line
);
res.append(p);
if (tokid) {
// We have a new token to deal with, jump to the beginning
// of the loop, but don't fetch the next token, hence the
// goto.
goto loop_start;
} else {
// Break out otherwise we end up appending an empty token to
// the end of the array
break;
}
} else {
value = String(tok.text());
}
Array p = make_packed_array(
// Convert the internal token ID to a user token ID
tokVal,
value,
loc.line0
);
res.append(p);
}
}
return res;
}
