Array HHVM_FUNCTION(token_get_all, const String& source) { Scanner scanner(source.data(), source.size(), RuntimeOption::GetScannerType() | Scanner::ReturnAllTokens); ScannerToken tok; Location loc; int tokid; Array res = Array::Create(); while ((tokid = scanner.getNextToken(tok, loc))) { loop_start: // For after seeing a T_INLINE_HTML, see below if (tokid < 256) { res.append(String::FromChar((char)tokid)); } else { String value; const int tokVal = get_user_token_id(tokid); if (tokVal == UserTokenId_T_XHP_LABEL) { value = String(":" + tok.text()); } else if (tokVal == UserTokenId_T_XHP_CATEGORY_LABEL) { value = String("%" + tok.text()); } else if (tokVal == UserTokenId_T_INLINE_HTML) { // Consecutive T_INLINE_HTML tokens should be merged together to // match Zend behaviour. value = String(tok.text()); int line = loc.line0; tokid = scanner.getNextToken(tok, loc); while (tokid == T_INLINE_HTML) { value += String(tok.text()); tokid = scanner.getNextToken(tok, loc); } Array p = make_packed_array( tokVal, value, line ); res.append(p); if (tokid) { // We have a new token to deal with, jump to the beginning // of the loop, but don't fetch the next token, hence the // goto. goto loop_start; } else { // Break out otherwise we end up appending an empty token to // the end of the array break; } } else { value = String(tok.text()); } Array p = make_packed_array( // Convert the internal token ID to a user token ID tokVal, value, loc.line0 ); res.append(p); } } return res; }
Array f_token_get_all(CStrRef source) { Scanner scanner(source.data(), source.size(), Scanner::AllowShortTags | Scanner::ReturnAllTokens); ScannerToken tok; Location loc; int tokid; Array res; while ((tokid = scanner.getNextToken(tok, loc))) { if (tokid < 256) { res.append(String::FromChar((char)tokid)); } else { Array p = CREATE_VECTOR3(tokid, String(tok.text()), loc.line0); res.append(p); } } return res; }
Array f_token_get_all(const String& source) { Scanner scanner(source.data(), source.size(), RuntimeOption::GetScannerType() | Scanner::ReturnAllTokens); ScannerToken tok; Location loc; int tokid; Array res; while ((tokid = scanner.getNextToken(tok, loc))) { if (tokid < 256) { res.append(String::FromChar((char)tokid)); } else { Array p = make_packed_array(tokid, String(tok.text()), loc.line0); res.append(p); } } return res; }