static CharType parse_escape(const CharType *&c) {
int point = 0;
switch (c[1]) {
case 'x':
for (int i = 2; i <= 3; ++i) {
int res = RegEx_hex2int(c[i]);
if (res == -1)
return '\0';
point = (point << 4) + res;
}
c = &c[3];
return CharType(point);
case 'u':
for (int i = 2; i <= 5; ++i) {
int res = RegEx_hex2int(c[i]);
if (res == -1)
return '\0';
point = (point << 4) + res;
}
c = &c[5];
return CharType(point);
case '0': ++c; return '\0';
case 'a': ++c; return '\a';
case 'e': ++c; return '\e';
case 'f': ++c; return '\f';
case 'n': ++c; return '\n';
case 'r': ++c; return '\r';
case 't': ++c; return '\t';
case 'v': ++c; return '\v';
case 'b': ++c; return '\b';
default: break;
}
return (++c)[0];
}
inline std::basic_ostream<CharType>& operator << (std::basic_ostream<CharType>& os, const Mat3x2& mat)
{
return os << CharType('(')
<< Float2(mat._11, mat._12) << CharType(',')
<< Float2(mat._21, mat._22) << CharType(',')
<< Float2(mat._31, mat._32) << CharType(')');
}
inline std::basic_ostream<CharType>& operator <<(std::basic_ostream<CharType>& os, const HSV& hsv)
{
return os << CharType('(')
<< hsv.h << CharType(',')
<< hsv.s << CharType(',')
<< hsv.v << CharType(')')
<< hsv.a << CharType(')');
}
inline std::basic_ostream<CharType>& operator << (std::basic_ostream<CharType>& os, const Mat4x4& mat)
{
return os << CharType('(')
<< mat.r[0] << CharType(',')
<< mat.r[1] << CharType(',')
<< mat.r[2] << CharType(',')
<< mat.r[3] << CharType(')');
}
inline std::basic_ostream<CharType>& operator <<(std::basic_ostream<CharType>& os, const Line& line)
{
return os << CharType('(')
<< line.begin.x << CharType(',')
<< line.begin.y << CharType(',')
<< line.end.x << CharType(',')
<< line.end.y << CharType(')');
}
String<CharType> GenerateRandomString(ULONG seed, size_t str_length)
{
String<CharType> r_str(str_length);
FOR(i, str_length)
{
CharType c = CharType('A') + CharType(RtlRandomEx(&seed) % 26);
r_str << c;
}
const CharType *c_str() const noexcept(noexcept(CharType(0)))
{
if (m_buf == nullptr) {
return empty;
} else {
// Terminating the string with the null character.
*m_bufEnd = CharType(0);
return m_buf;
}
}
void load(std::basic_string<CharType> & s)
{
unsigned int l;
load(l);
s.resize(l);
// note breaking a rule here - could be a problem on some platform
if (l)
load_impl(const_cast<CharType *>(s.data()),
get_mpi_datatype(CharType()),l);
}
QC_NET_NAMESPACE_BEGIN
//==============================================================================
// URLEncoder::Encode
//
/**
Converts a Unicode string into the MIME @c x-www-form-urlencoded format.
To convert a String, each Unicode character is examined in turn:
- The ASCII characters 'a' through 'z', 'A' through 'Z', '0' through '9',
and ".", "-", "*", "_" remain the same.
- The space character ' '(U+20) is converted into a plus sign '+'.
- All other characters are converted into their UTF-8 equivalent and the
subsequent bytes are encoded as the 3-byte string "%xy",
where xy is the two-digit hexadecimal representation of the byte.
*/
//==============================================================================
String URLEncoder::Encode(const String& uri)
{
const char included[] = {'.', '-', '*', '_'};
const char* pInclEnd = included + sizeof(included);
String sRet;
//
// 1. Convert the URL to UTF-8. Of course, it may already be encoded
// as UTF-8 if that is our internal string encoding.
//
ByteString utf8 = StringUtils::ToUTF8(uri);
sRet.reserve(utf8.length());
const Byte* pEnd = (const Byte*)utf8.data() + utf8.length();
for(const Byte* pByte=(const Byte*)utf8.data(); pByte<pEnd; ++pByte)
{
if((isalnum(*pByte)
|| std::find(included, pInclEnd, *pByte) != pInclEnd))
{
sRet += CharType(*pByte);
}
else
{
//
// 2. Escape any disallowed characters
//
char buffer[10];
sprintf(buffer, "%%%02X", (unsigned)*pByte);
sRet += StringUtils::FromLatin1(buffer);
}
}
return sRet;
}
inline std::basic_istream<CharType>& operator >> (std::basic_istream<CharType>& is, HSV& hsv)
{
CharType unused;
is >> unused
>> hsv.h >> unused
>> hsv.s >> unused
>> hsv.v >> unused;
if (unused == CharType(','))
{
is >> hsv.a >> unused;
}
//
// URI references require encoding and escaping of certain characters.
// The disallowed characters include all non-ASCII characters, plus the
// excluded characters listed in Section 2.4 of [RFC 2396], except for
// the number sign (#) and percent sign (%) characters and the square bracket
// characters re-allowed in [RFC 2732].
//
// The excluded characters listed in Section 2.4 of IETF RFC 2396:
//
// control: 0x00-0x1F and 0x7F
// space: 0x20
// delims: "<" | ">" | "#" | "%" | <">
// unwise = "{" | "}" | "|" | "\" | "^" | "[" | "]" | "`"
//
// 1) Each disallowed character is converted to UTF-8 as 1 or more bytes
// 2) Any bytes corresponding to a disallowed character are escaped
// with the URI escaping mechanism (that is, converted to %HH, where
// HH is the hexadecimal notation of the byte value)
// 3) The original character is replaced by the resulting character sequence
//
// Note that this normalization process is idempotent: repeated normalization
// does not change a normalized URI reference.
//==============================================================================
String URLEncoder::RawEncode(const String& uri)
{
const char excluded[] = {'<', '>', '"', '{', '}', '|', '\\', '^', '\''};
const char* pExclEnd = excluded + sizeof(excluded);
String sRet;
//
// 1. Convert the URL to UTF-8. Of course, it may already be encoded
// as UTF-8 if that is our internal string encoding.
//
ByteString utf8 = StringUtils::ToUTF8(uri);
sRet.reserve(utf8.length());
const Byte* pEnd = (const Byte*)utf8.data() + utf8.length();
for(const Byte* pByte=(const Byte*)utf8.data(); pByte<pEnd; ++pByte)
{
if((*pByte > 0x20 && *pByte < 0x7F)
&& std::find(excluded, pExclEnd, *pByte) == pExclEnd)
{
sRet += CharType(*pByte);
}
else
{
//
// 2. Escape any disallowed characters
//
char buffer[10];
sprintf(buffer, "%%%02X", (unsigned)*pByte);
sRet += StringUtils::FromLatin1(buffer);
}
}
return sRet;
}
inline std::basic_ostream<CharType>& operator << (std::basic_ostream<CharType>& os, const Ray& ray)
{
return os << CharType('(') << ray.origin << CharType(',') << ray.direction << CharType(')');
}
void GDTokenizerText::_advance() {
if (error_flag) {
//parser broke
_make_error(last_error);
return;
}
if (code_pos>=len) {
_make_token(TK_EOF);
return;
}
#define GETCHAR(m_ofs) ((m_ofs+code_pos)>=len?0:_code[m_ofs+code_pos])
#define INCPOS(m_amount) { code_pos+=m_amount; column+=m_amount; }
while (true) {
bool is_node_path = false;
StringMode string_mode=STRING_DOUBLE_QUOTE;
switch(GETCHAR(0)) {
case 0:
_make_token(TK_EOF);
break;
case '\\':
INCPOS(1);
if (GETCHAR(0)=='\r') {
INCPOS(1);
}
if (GETCHAR(0)!='\n') {
_make_error("Expected newline after '\\'.");
return;
}
INCPOS(1);
while(GETCHAR(0)==' ' || GETCHAR(0)=='\t') {
INCPOS(1);
}
continue;
case '\t':
case '\r':
case ' ':
INCPOS(1);
continue;
case '\n': {
line++;
INCPOS(1);
column=0;
int i=0;
while(GETCHAR(i)==' ' || GETCHAR(i)=='\t') {
i++;
}
_make_newline(i);
return;
}
#if 1 //py style tokenizer
case '#': { // line comment skip
while(GETCHAR(0)!='\n') {
code_pos++;
if (GETCHAR(0)==0) { //end of file
//_make_error("Unterminated Comment");
_make_token(TK_EOF);
return;
}
}
INCPOS(1);
column=0;
line++;
int i=0;
while(GETCHAR(i)==' ' || GETCHAR(i)=='\t') {
i++;
}
_make_newline(i);
return;
} break;
#endif
case '/': {
switch(GETCHAR(1)) {
#if 0 // c style tokenizer
case '*': { // block comment
int pos = code_pos+2;
int new_line=line;
int new_col=column+2;
while(true) {
if (_code[pos]=='0') {
_make_error("Unterminated Comment");
code_pos=pos;
return;
}
if (_code[pos]=='*' && _code[pos+1]=='/') {
new_col+=2;
pos+=2; //compensate
break;
} else if (_code[pos]=='\n') {
new_line++;
new_col=0;
} else {
new_col++;
}
pos++;
}
column=new_col;
line=new_line;
code_pos=pos;
continue;
} break;
case '/': { // line comment skip
while(GETCHAR(0)!='\n') {
code_pos++;
if (GETCHAR(0)==0) { //end of file
_make_error("Unterminated Comment");
return;
}
}
INCPOS(1);
column=0;
line++;
continue;
} break;
#endif
case '=': { // diveq
_make_token(TK_OP_ASSIGN_DIV);
INCPOS(1);
} break;
default:
_make_token(TK_OP_DIV);
}
} break;
case '=': {
if (GETCHAR(1)=='=') {
_make_token(TK_OP_EQUAL);
INCPOS(1);
} else
_make_token(TK_OP_ASSIGN);
} break;
case '<': {
if (GETCHAR(1)=='=') {
_make_token(TK_OP_LESS_EQUAL);
INCPOS(1);
} else if (GETCHAR(1)=='<') {
if (GETCHAR(2)=='=') {
_make_token(TK_OP_ASSIGN_SHIFT_LEFT);
INCPOS(1);
} else {
_make_token(TK_OP_SHIFT_LEFT);
}
INCPOS(1);
} else
_make_token(TK_OP_LESS);
} break;
case '>': {
if (GETCHAR(1)=='=') {
_make_token(TK_OP_GREATER_EQUAL);
INCPOS(1);
} else if (GETCHAR(1)=='>') {
if (GETCHAR(2)=='=') {
_make_token(TK_OP_ASSIGN_SHIFT_RIGHT);
INCPOS(1);
} else {
_make_token(TK_OP_SHIFT_RIGHT);
}
INCPOS(1);
} else {
_make_token(TK_OP_GREATER);
}
} break;
case '!': {
if (GETCHAR(1)=='=') {
_make_token(TK_OP_NOT_EQUAL);
INCPOS(1);
} else {
_make_token(TK_OP_NOT);
}
} break;
//case '"' //string - no strings in shader
//case '\'' //string - no strings in shader
case '{':
_make_token(TK_CURLY_BRACKET_OPEN);
break;
case '}':
_make_token(TK_CURLY_BRACKET_CLOSE);
break;
case '[':
_make_token(TK_BRACKET_OPEN);
break;
case ']':
_make_token(TK_BRACKET_CLOSE);
break;
case '(':
_make_token(TK_PARENTHESIS_OPEN);
break;
case ')':
_make_token(TK_PARENTHESIS_CLOSE);
break;
case ',':
_make_token(TK_COMMA);
break;
case ';':
_make_token(TK_SEMICOLON);
break;
case '?':
_make_token(TK_QUESTION_MARK);
break;
case ':':
_make_token(TK_COLON); //for methods maybe but now useless.
break;
case '^': {
if (GETCHAR(1)=='=') {
_make_token(TK_OP_ASSIGN_BIT_XOR);
INCPOS(1);
} else {
_make_token(TK_OP_BIT_XOR);
}
} break;
case '~':
_make_token(TK_OP_BIT_INVERT);
break;
case '&': {
if (GETCHAR(1)=='&') {
_make_token(TK_OP_AND);
INCPOS(1);
} else if (GETCHAR(1)=='=') {
_make_token(TK_OP_ASSIGN_BIT_AND);
INCPOS(1);
} else {
_make_token(TK_OP_BIT_AND);
}
} break;
case '|': {
if (GETCHAR(1)=='|') {
_make_token(TK_OP_OR);
INCPOS(1);
} else if (GETCHAR(1)=='=') {
_make_token(TK_OP_ASSIGN_BIT_OR);
INCPOS(1);
} else {
_make_token(TK_OP_BIT_OR);
}
} break;
case '*': {
if (GETCHAR(1)=='=') {
_make_token(TK_OP_ASSIGN_MUL);
INCPOS(1);
} else {
_make_token(TK_OP_MUL);
}
} break;
case '+': {
if (GETCHAR(1)=='=') {
_make_token(TK_OP_ASSIGN_ADD);
INCPOS(1);
//} else if (GETCHAR(1)=='+') {
// _make_token(TK_OP_PLUS_PLUS);
// INCPOS(1);
} else {
_make_token(TK_OP_ADD);
}
} break;
case '-': {
if (GETCHAR(1)=='=') {
_make_token(TK_OP_ASSIGN_SUB);
INCPOS(1);
//} else if (GETCHAR(1)=='-') {
// _make_token(TK_OP_MINUS_MINUS);
// INCPOS(1);
} else {
_make_token(TK_OP_SUB);
}
} break;
case '%': {
if (GETCHAR(1)=='=') {
_make_token(TK_OP_ASSIGN_MOD);
INCPOS(1);
} else {
_make_token(TK_OP_MOD);
}
} break;
case '@':
if( CharType(GETCHAR(1))!='"' && CharType(GETCHAR(1))!='\'' ) {
_make_error("Unexpected '@'");
return;
}
INCPOS(1);
is_node_path=true;
case '\'':
case '"': {
if (GETCHAR(0)=='\'')
string_mode=STRING_SINGLE_QUOTE;
int i=1;
if (string_mode==STRING_DOUBLE_QUOTE && GETCHAR(i)=='"' && GETCHAR(i+1)=='"') {
i+=2;
string_mode=STRING_MULTILINE;
}
String str;
while(true) {
if (CharType(GETCHAR(i))==0) {
_make_error("Unterminated String");
return;
} else if( string_mode==STRING_DOUBLE_QUOTE && CharType(GETCHAR(i))=='"' ) {
break;
} else if( string_mode==STRING_SINGLE_QUOTE && CharType(GETCHAR(i))=='\'' ) {
break;
} else if( string_mode==STRING_MULTILINE && CharType(GETCHAR(i))=='\"' && CharType(GETCHAR(i+1))=='\"' && CharType(GETCHAR(i+2))=='\"') {
i+=2;
break;
} else if( string_mode!=STRING_MULTILINE && CharType(GETCHAR(i))=='\n') {
_make_error("Unexpected EOL at String.");
return;
} else if (CharType(GETCHAR(i))=='\\') {
//escaped characters...
i++;
CharType next = GETCHAR(i);
if (next==0) {
_make_error("Unterminated String");
return;
}
CharType res=0;
switch(next) {
case 'a': res=7; break;
case 'b': res=8; break;
case 't': res=9; break;
case 'n': res=10; break;
case 'v': res=11; break;
case 'f': res=12; break;
case 'r': res=13; break;
case '\'': res='\''; break;
case '\"': res='\"'; break;
case '\\': res='\\'; break;
case '/': res='/'; break; //wtf
case 'u': {
//hexnumbarh - oct is deprecated
i+=1;
for(int j=0;j<4;j++) {
CharType c = GETCHAR(i+j);
if (c==0) {
_make_error("Unterminated String");
return;
}
if (!((c>='0' && c<='9') || (c>='a' && c<='f') || (c>='A' && c<='F'))) {
_make_error("Malformed hex constant in string");
return;
}
CharType v;
if (c>='0' && c<='9') {
v=c-'0';
} else if (c>='a' && c<='f') {
v=c-'a';
v+=10;
} else if (c>='A' && c<='F') {
v=c-'A';
v+=10;
} else {
ERR_PRINT("BUG");
v=0;
}
res<<=4;
res|=v;
}
i+=3;
} break;
default: {
_make_error("Invalid escape sequence");
return;
} break;
}
str+=res;
} else {
str+=CharType(GETCHAR(i));
}
i++;
}
INCPOS(i);
if (is_node_path) {
_make_constant(NodePath(str));
} else {
_make_constant(str);
}
} break;
case 0xFFFF: {
_make_token(TK_CURSOR);
} break;
default: {
if (_is_number(GETCHAR(0)) || (GETCHAR(0)=='.' && _is_number(GETCHAR(1)))) {
// parse number
bool period_found=false;
bool exponent_found=false;
bool hexa_found=false;
bool sign_found=false;
String str;
int i=0;
while(true) {
if (GETCHAR(i)=='.') {
if (period_found || exponent_found) {
_make_error("Invalid numeric constant at '.'");
return;
}
period_found=true;
} else if (GETCHAR(i)=='x') {
if (hexa_found || str.length()!=1 || !( (i==1 && str[0]=='0') || (i==2 && str[1]=='0' && str[0]=='-') ) ) {
_make_error("Invalid numeric constant at 'x'");
return;
}
hexa_found=true;
} else if (!hexa_found && GETCHAR(i)=='e') {
if (hexa_found || exponent_found) {
_make_error("Invalid numeric constant at 'e'");
return;
}
exponent_found=true;
} else if (_is_number(GETCHAR(i))) {
//all ok
} else if (hexa_found && _is_hex(GETCHAR(i))) {
} else if ((GETCHAR(i)=='-' || GETCHAR(i)=='+') && exponent_found) {
if (sign_found) {
_make_error("Invalid numeric constant at '-'");
return;
}
sign_found=true;
} else
break;
str+=CharType(GETCHAR(i));
i++;
}
if (!( _is_number(str[str.length()-1]) || (hexa_found && _is_hex(str[str.length()-1])))) {
_make_error("Invalid numeric constant: "+str);
return;
}
INCPOS(str.length());
if (hexa_found) {
int val = str.hex_to_int();
_make_constant(val);
} else if (period_found) {
real_t val = str.to_double();
//print_line("*%*%*%*% to convert: "+str+" result: "+rtos(val));
_make_constant(val);
} else {
int val = str.to_int();
_make_constant(val);
}
return;
}
if (GETCHAR(0)=='.') {
//parse period
_make_token(TK_PERIOD);
break;
}
if (_is_text_char(GETCHAR(0))) {
// parse identifier
String str;
str+=CharType(GETCHAR(0));
int i=1;
while(_is_text_char(GETCHAR(i))) {
str+=CharType(GETCHAR(i));
i++;
}
bool identifier=false;
if (str=="null") {
_make_constant(Variant());
} else if (str=="true") {
_make_constant(true);
} else if (str=="false") {
_make_constant(false);
} else {
bool found=false;
struct _bit { Variant::Type type; const char *text;};
//built in types
static const _bit type_list[]={
//types
{Variant::BOOL,"bool"},
{Variant::INT,"int"},
{Variant::REAL,"float"},
{Variant::STRING,"String"},
{Variant::VECTOR2,"vec2"},
{Variant::VECTOR2,"Vector2"},
{Variant::RECT2,"Rect2"},
{Variant::MATRIX32,"Matrix32"},
{Variant::MATRIX32,"mat32"},
{Variant::VECTOR3,"vec3"},
{Variant::VECTOR3,"Vector3"},
{Variant::_AABB,"AABB"},
{Variant::_AABB,"Rect3"},
{Variant::PLANE,"Plane"},
{Variant::QUAT,"Quat"},
{Variant::MATRIX3,"mat3"},
{Variant::MATRIX3,"Matrix3"},
{Variant::TRANSFORM,"trn"},
{Variant::TRANSFORM,"Transform"},
{Variant::COLOR,"Color"},
{Variant::IMAGE,"Image"},
{Variant::_RID,"RID"},
{Variant::OBJECT,"Object"},
{Variant::INPUT_EVENT,"InputEvent"},
{Variant::NODE_PATH,"NodePath"},
{Variant::DICTIONARY,"dict"},
{Variant::DICTIONARY,"Dictionary"},
{Variant::ARRAY,"Array"},
{Variant::RAW_ARRAY,"RawArray"},
{Variant::INT_ARRAY,"IntArray"},
{Variant::REAL_ARRAY,"FloatArray"},
{Variant::STRING_ARRAY,"StringArray"},
{Variant::VECTOR2_ARRAY,"Vector2Array"},
{Variant::VECTOR3_ARRAY,"Vector3Array"},
{Variant::COLOR_ARRAY,"ColorArray"},
{Variant::VARIANT_MAX,NULL},
};
{
int idx=0;
while(type_list[idx].text) {
if (str==type_list[idx].text) {
_make_type(type_list[idx].type);
found=true;
break;
}
idx++;
}
}
if (!found) {
//built in func?
for(int i=0;i<GDFunctions::FUNC_MAX;i++) {
if (str==GDFunctions::get_func_name(GDFunctions::Function(i))) {
_make_built_in_func(GDFunctions::Function(i));
found=true;
break;
}
}
//keywor
}
if (!found) {
struct _kws { Token token; const char *text;};
static const _kws keyword_list[]={
//ops
{TK_OP_IN,"in"},
{TK_OP_NOT,"not"},
{TK_OP_OR,"or"},
{TK_OP_AND,"and"},
//func
{TK_PR_FUNCTION,"func"},
{TK_PR_FUNCTION,"function"},
{TK_PR_CLASS,"class"},
{TK_PR_EXTENDS,"extends"},
{TK_PR_TOOL,"tool"},
{TK_PR_STATIC,"static"},
{TK_PR_EXPORT,"export"},
{TK_PR_SETGET,"setget"},
{TK_PR_VAR,"var"},
{TK_PR_PRELOAD,"preload"},
{TK_PR_ASSERT,"assert"},
{TK_PR_YIELD,"yield"},
{TK_PR_CONST,"const"},
//controlflow
{TK_CF_IF,"if"},
{TK_CF_ELIF,"elif"},
{TK_CF_ELSE,"else"},
{TK_CF_FOR,"for"},
{TK_CF_WHILE,"while"},
{TK_CF_DO,"do"},
{TK_CF_SWITCH,"switch"},
{TK_CF_BREAK,"break"},
{TK_CF_CONTINUE,"continue"},
{TK_CF_RETURN,"return"},
{TK_CF_PASS,"pass"},
{TK_SELF,"self"},
{TK_ERROR,NULL}
};
int idx=0;
found=false;
while(keyword_list[idx].text) {
if (str==keyword_list[idx].text) {
_make_token(keyword_list[idx].token);
found=true;
break;
}
idx++;
}
}
if (!found)
identifier=true;
}
if (identifier) {
_make_identifier(str);
}
INCPOS(str.length());
return;
}
_make_error("Unknown character");
return;
} break;
}
INCPOS(1);
break;
}
}
inline std::basic_ostream<CharType>& operator << (std::basic_ostream<CharType>& os, const Vector2D<Type>& v)
{
return os << CharType('(') << v.x << CharType(',') << v.y << CharType(')');
}
Value CharValue(char16_t c){return Value( CharType(),2,c);}
//
// TBuf::NextChar()
//
// Get the next char from the buffer, returning FALSE
// when there is no data left.
//
Bool TBuf::NextChar(U8 *c)
{
ASSERT(c);
U8 n;
// Filter comments and carriage returns
do
{
ASSERT(bufPos <= bufSize);
// Check if at end of data
if (bufPos == bufSize)
{
// Are we reading a string constant
if (constantOn)
{
TokenError("reached end of data in string constant");
}
return (FALSE);
}
// Get the next character
n = (U8) bufData[bufPos++];
// Adjust parse position
if (n == TBUF_CH_EOL)
{
// Are we reading a string constant
if (constantOn)
{
TokenError("newline in string constant");
}
lastXPos = xPos;
xPos = 0;
yPos++;
}
else if (n != TBUF_CH_CR)
{
xPos++;
}
// if not in a constant, check commenting
if (!constantOn)
{
// turn line comment mode on
if (!commentOn && (CharType(n) == COMMENT))
{
commentOn = TRUE;
}
// turn line comment mode off
if (commentOn && (n == TBUF_CH_EOL))
{
commentOn = FALSE;
}
}
}
while(commentOn || (n == TBUF_CH_CR));
// convert tabs to spaces
if (n == TBUF_CH_TAB)
{
n = TBUF_CH_SPACE;
}
// set the char value
*c = n;
return(TRUE);
}
//
// TBuf::NextToken
//
// Reads the next token into 'dest'. Skips any preceeding spaces,
// stopping when either a space, a punctuation char, or end of data
// occurs. Generates an error if token exceeds max token length.
//
// Returns :
//
// TR_OK - if token was read with no problems
// TR_PUN - if punctuation char was found
// TR_EOF - if end of data reached
//
TBufResult TBuf::NextToken()
{
ASSERT(setup);
U8 c;
U32 tokenLen = 0;
char *sPtr;
// Have we peeked at the next token
if (peekState == PS_ON)
{
peekState = PS_OFF;
Utils::Strcpy(prevToken, lastToken);
Utils::Strcpy(lastToken, peekToken);
return (peekVal);
}
// Setup peek operation
if (peekState == PS_PRE)
{
// Store next token in peek buffer
sPtr = peekToken;
// We are now in peek mode
peekState = PS_ON;
}
else
{
// Save previous token
Utils::Strcpy(prevToken, lastToken);
// Store next token in last token buffer
sPtr = lastToken;
}
// Clear token buffer
*sPtr = '\0';
// Read next token
for (;;)
{
// Have we reached the end of the data
if (!NextChar(&c))
{
// Return token if we have one
return (tokenLen ? TR_OK : TR_EOF);
}
// Finish reading constant
if (constantOn && c == constantChar)
{
constantOn = FALSE;
}
// Ignore these checks if reading a constant
if (!constantOn)
{
// What char have we got
if ((c == TBUF_CH_SPACE) || (c == TBUF_CH_EOL))
{
// Have we read a token
if (tokenLen)
{
return (TR_OK);
}
// Ignore this character
continue;
}
// Check for punctuation
if (CharType(c) == PUNCTUATION)
{
// Haven't read anything yet
if (!tokenLen)
{
sPtr[0] = c;
sPtr[1] = '\0';
return(TR_PUN);
}
// Successfully read a token
StepBack();
return(TR_OK);
}
}
// Add this char to the current token
if (tokenLen < TBUF_MAX_TOKEN - 1)
{
sPtr[tokenLen++] = c;
sPtr[tokenLen] = '\0';
}
else
{
TokenError("maximum token length [%d bytes] exceeded", TBUF_MAX_TOKEN-1);
}
}
}
inline std::basic_ostream<CharType>& operator << (std::basic_ostream<CharType>& os, const Line& line)
{
return os << CharType('(') << line.p[0] << CharType(',') << line.p[1] << CharType(')');
}
return afc::math::min<const std::size_t>(std::numeric_limits<std::size_t>::max() / sizeof(CharType) - 1,
std::size_t(std::numeric_limits<std::ptrdiff_t>::max()));
}
static const CharType empty[1];
// If not nullptr then one character is reserved for the terminating character.
// Non-[] deleter is specified since placement allocation is used.
CharType *m_buf;
CharType *m_bufEnd;
std::size_t m_capacity;
};
}
template<typename CharType, afc::AllocMode allocMode>
const CharType afc::FastStringBuffer<CharType, allocMode>::empty[1] = {CharType(0)};
template<typename CharType, afc::AllocMode allocMode>
inline std::size_t afc::FastStringBuffer<CharType, allocMode>::nextStorageSize(const std::size_t capacity) noexcept(noexcept(badAlloc()))
{
static_assert(allocMode == afc::AllocMode::pow2 || allocMode == afc::AllocMode::accurate, "Unsupported allocMode.");
// FastStringBuffer::reserve() does not expand storage if capacity == 0.
assert(capacity > 0);
if (allocMode == afc::AllocMode::pow2) {
constexpr std::size_t maxStorageSize = maxCapacity() + 1;
const std::size_t requestedStorageSize = capacity + 1;
// FastStringBuffer::reserve(std::size_t) does not expand storage if capacity == 0.
/* Since newStorageSize is always a power of two, the first value that