Ejemplo n.º 1
0
	static CharType parse_escape(const CharType *&c) {

		int point = 0;
		switch (c[1]) {
			case 'x':
				for (int i = 2; i <= 3; ++i) {
					int res = RegEx_hex2int(c[i]);
					if (res == -1)
						return '\0';
					point = (point << 4) + res;
				}
				c = &c[3];
				return CharType(point);
			case 'u':
				for (int i = 2; i <= 5; ++i) {
					int res = RegEx_hex2int(c[i]);
					if (res == -1)
						return '\0';
					point = (point << 4) + res;
				}
				c = &c[5];
				return CharType(point);
			case '0': ++c; return '\0';
			case 'a': ++c; return '\a';
			case 'e': ++c; return '\e';
			case 'f': ++c; return '\f';
			case 'n': ++c; return '\n';
			case 'r': ++c; return '\r';
			case 't': ++c; return '\t';
			case 'v': ++c; return '\v';
			case 'b': ++c; return '\b';
			default: break;
		}
		return (++c)[0];
	}
Ejemplo n.º 2
0
	inline std::basic_ostream<CharType>& operator << (std::basic_ostream<CharType>& os, const Mat3x2& mat)
	{
		return os << CharType('(')
			<< Float2(mat._11, mat._12) << CharType(',')
			<< Float2(mat._21, mat._22) << CharType(',')
			<< Float2(mat._31, mat._32) << CharType(')');
	}
Ejemplo n.º 3
0
	inline std::basic_ostream<CharType>& operator <<(std::basic_ostream<CharType>& os, const HSV& hsv)
	{
		return os << CharType('(')
			<< hsv.h << CharType(',')
			<< hsv.s << CharType(',')
			<< hsv.v << CharType(')')
			<< hsv.a << CharType(')');
	}
Ejemplo n.º 4
0
inline std::basic_ostream<CharType>& operator << (std::basic_ostream<CharType>& os, const Mat4x4& mat)
{
    return os << CharType('(')
           << mat.r[0] << CharType(',')
           << mat.r[1] << CharType(',')
           << mat.r[2] << CharType(',')
           << mat.r[3] << CharType(')');
}
Ejemplo n.º 5
0
	inline std::basic_ostream<CharType>& operator <<(std::basic_ostream<CharType>& os, const Line& line)
	{
		return	os << CharType('(')
			<< line.begin.x << CharType(',')
			<< line.begin.y << CharType(',')
			<< line.end.x << CharType(',')
			<< line.end.y << CharType(')');
	}
Ejemplo n.º 6
0
String<CharType> GenerateRandomString(ULONG seed, size_t str_length)
{
    String<CharType> r_str(str_length);

    FOR(i, str_length)
    {
        CharType c = CharType('A') + CharType(RtlRandomEx(&seed) % 26);
        r_str << c;
    }
Ejemplo n.º 7
0
		const CharType *c_str() const noexcept(noexcept(CharType(0)))
		{
			if (m_buf == nullptr) {
				return empty;
			} else {
				// Terminating the string with the null character.
				*m_bufEnd = CharType(0);
				return m_buf;
			}
		}
Ejemplo n.º 8
0
 void load(std::basic_string<CharType> & s)
 {
     unsigned int l;
     load(l);
     s.resize(l);
     // note breaking a rule here - could be a problem on some platform
     if (l)
       load_impl(const_cast<CharType *>(s.data()),
                 get_mpi_datatype(CharType()),l);
 }
Ejemplo n.º 9
0
QC_NET_NAMESPACE_BEGIN

//==============================================================================
// URLEncoder::Encode
//
/**
   Converts a Unicode string into the MIME @c x-www-form-urlencoded format.

   To convert a String, each Unicode character is examined in turn: 

   - The ASCII characters 'a' through 'z', 'A' through 'Z', '0' through '9', 
      and ".", "-", "*", "_" remain the same. 
   - The space character ' '(U+20) is converted into a plus sign '+'. 
   - All other characters are converted into their UTF-8 equivalent and the
     subsequent bytes are encoded as the 3-byte string "%xy", 
     where xy is the two-digit hexadecimal representation of the byte. 
*/
//==============================================================================
String URLEncoder::Encode(const String& uri)
{
	const char included[] = {'.', '-', '*', '_'};
	const char* pInclEnd = included + sizeof(included);

	String sRet;

	//
	// 1. Convert the URL to UTF-8.  Of course, it may already be encoded
	//    as UTF-8 if that is our internal string encoding.
	//
	ByteString utf8 = StringUtils::ToUTF8(uri);

	sRet.reserve(utf8.length());
	const Byte* pEnd =  (const Byte*)utf8.data() + utf8.length();

	for(const Byte* pByte=(const Byte*)utf8.data(); pByte<pEnd; ++pByte)
	{
		if((isalnum(*pByte)
			|| std::find(included, pInclEnd, *pByte) != pInclEnd))
		{
			sRet += CharType(*pByte);
		}
		else
		{
			//
			// 2. Escape any disallowed characters
			//
			char buffer[10];
			sprintf(buffer, "%%%02X", (unsigned)*pByte);
			sRet += StringUtils::FromLatin1(buffer);
		}
	}
	
	return sRet;
}
Ejemplo n.º 10
0
	inline std::basic_istream<CharType>& operator >> (std::basic_istream<CharType>& is, HSV& hsv)
	{
		CharType unused;

		is >> unused
			>> hsv.h >> unused
			>> hsv.s >> unused
			>> hsv.v >> unused;

		if (unused == CharType(','))
		{
			is >> hsv.a >> unused;
		}
Ejemplo n.º 11
0
//
// URI references require encoding and escaping of certain characters.
// The disallowed characters include all non-ASCII characters, plus the
// excluded characters listed in Section 2.4 of [RFC 2396], except for
// the number sign (#) and percent sign (%) characters and the square bracket
// characters re-allowed in [RFC 2732].
//
// The excluded characters listed in Section 2.4 of IETF RFC 2396:
//
//  control: 0x00-0x1F and 0x7F
//  space: 0x20
//  delims: "<" | ">" | "#" | "%" | <">
//  unwise      = "{" | "}" | "|" | "\" | "^" | "[" | "]" | "`"
//
// 1) Each disallowed character is converted to UTF-8 as 1 or more bytes
// 2) Any bytes corresponding to a disallowed character are escaped
//    with the URI escaping mechanism (that is, converted to %HH, where
//    HH is the hexadecimal notation of the byte value)
// 3) The original character is replaced by the resulting character sequence
//
// Note that this normalization process is idempotent: repeated normalization
// does not change a normalized URI reference.
//==============================================================================
String URLEncoder::RawEncode(const String& uri)
{
	const char excluded[] = {'<', '>', '"', '{', '}', '|', '\\', '^', '\''};
	const char* pExclEnd = excluded + sizeof(excluded);

	String sRet;

	//
	// 1. Convert the URL to UTF-8.  Of course, it may already be encoded
	//    as UTF-8 if that is our internal string encoding.
	//
	ByteString utf8 = StringUtils::ToUTF8(uri);

	sRet.reserve(utf8.length());
	const Byte* pEnd =  (const Byte*)utf8.data() + utf8.length();

	for(const Byte* pByte=(const Byte*)utf8.data(); pByte<pEnd; ++pByte)
	{
		if((*pByte > 0x20 && *pByte < 0x7F)
			&& std::find(excluded, pExclEnd, *pByte) == pExclEnd)
		{
			sRet += CharType(*pByte);
		}
		else
		{
			//
			// 2. Escape any disallowed characters
			//
			char buffer[10];
			sprintf(buffer, "%%%02X", (unsigned)*pByte);
			sRet += StringUtils::FromLatin1(buffer);
		}
	}
	
	return sRet;
}
Ejemplo n.º 12
0
	inline std::basic_ostream<CharType>& operator << (std::basic_ostream<CharType>& os, const Ray& ray)
	{
		return	os << CharType('(') << ray.origin << CharType(',') << ray.direction << CharType(')');
	}
Ejemplo n.º 13
0
void GDTokenizerText::_advance() {

	if (error_flag) {
		//parser broke
		_make_error(last_error);
		return;
	}

	if (code_pos>=len) {
		_make_token(TK_EOF);
		return;
	}
#define GETCHAR(m_ofs) ((m_ofs+code_pos)>=len?0:_code[m_ofs+code_pos])
#define INCPOS(m_amount) { code_pos+=m_amount; column+=m_amount; }
	while (true) {


		bool is_node_path  = false;
		StringMode string_mode=STRING_DOUBLE_QUOTE;

		switch(GETCHAR(0)) {
			case 0:
				_make_token(TK_EOF);
				break;
			case '\\':
				INCPOS(1);
				if (GETCHAR(0)=='\r') {
					INCPOS(1);
				}

				if (GETCHAR(0)!='\n') {
					_make_error("Expected newline after '\\'.");
					return;
				}

				INCPOS(1);

				while(GETCHAR(0)==' ' || GETCHAR(0)=='\t') {
					INCPOS(1);
				}

				continue;
			case '\t':
			case '\r':
			case ' ':
				INCPOS(1);
				continue;
			case '\n': {
				line++;
				INCPOS(1);
				column=0;
				int i=0;
				while(GETCHAR(i)==' ' || GETCHAR(i)=='\t') {
					i++;
				}

				_make_newline(i);
				return;
			}
#if 1 //py style tokenizer
			case '#': { // line comment skip

				while(GETCHAR(0)!='\n') {
					code_pos++;
					if (GETCHAR(0)==0) { //end of file
						//_make_error("Unterminated Comment");
						_make_token(TK_EOF);
						return;
					}
				}
				INCPOS(1);
				column=0;
				line++;
				int i=0;
				while(GETCHAR(i)==' ' || GETCHAR(i)=='\t') {
					i++;
				}
				_make_newline(i);
				return;

			} break;
#endif
			case '/': {

				switch(GETCHAR(1)) {
#if 0 // c style tokenizer
					case '*': { // block comment
						int pos = code_pos+2;
						int new_line=line;
						int new_col=column+2;

						while(true) {
							if (_code[pos]=='0') {
								_make_error("Unterminated Comment");
								code_pos=pos;
								return;
							}
							if (_code[pos]=='*' && _code[pos+1]=='/') {
								new_col+=2;
								pos+=2; //compensate
								break;
							} else if (_code[pos]=='\n') {
								new_line++;
								new_col=0;
							} else {
								new_col++;
							}
							pos++;
						}

						column=new_col;
						line=new_line;
						code_pos=pos;
						continue;

					} break;
					case '/': { // line comment skip

						while(GETCHAR(0)!='\n') {
							code_pos++;
							if (GETCHAR(0)==0) { //end of file
								_make_error("Unterminated Comment");
								return;
							}
						}
						INCPOS(1);
						column=0;
						line++;
						continue;

					} break;
#endif
					case '=': { // diveq

						_make_token(TK_OP_ASSIGN_DIV);
						INCPOS(1);

					} break;
					default:
						_make_token(TK_OP_DIV);

				}
			} break;
			case '=': {
				if (GETCHAR(1)=='=') {
					_make_token(TK_OP_EQUAL);
					INCPOS(1);

				} else
					_make_token(TK_OP_ASSIGN);

			} break;
			case '<': {
				if (GETCHAR(1)=='=') {

					_make_token(TK_OP_LESS_EQUAL);
					INCPOS(1);
				} else if (GETCHAR(1)=='<') {
					if (GETCHAR(2)=='=') {
						_make_token(TK_OP_ASSIGN_SHIFT_LEFT);
						INCPOS(1);
					} else {
						_make_token(TK_OP_SHIFT_LEFT);
					}
					INCPOS(1);
				} else
					_make_token(TK_OP_LESS);

			} break;
			case '>': {
				if (GETCHAR(1)=='=') {
					_make_token(TK_OP_GREATER_EQUAL);
					INCPOS(1);
				} else if (GETCHAR(1)=='>') {
					if (GETCHAR(2)=='=') {
						_make_token(TK_OP_ASSIGN_SHIFT_RIGHT);
						INCPOS(1);

					} else {
						_make_token(TK_OP_SHIFT_RIGHT);
					}
					INCPOS(1);
				} else {
					_make_token(TK_OP_GREATER);
				}

			} break;
			case '!': {
				if (GETCHAR(1)=='=') {
					_make_token(TK_OP_NOT_EQUAL);
					INCPOS(1);
				} else {
					_make_token(TK_OP_NOT);
				}

			} break;
			//case '"' //string - no strings in shader
			//case '\'' //string - no strings in shader
			case '{':
				_make_token(TK_CURLY_BRACKET_OPEN);
				break;
			case '}':
				_make_token(TK_CURLY_BRACKET_CLOSE);
				break;
			case '[':
				_make_token(TK_BRACKET_OPEN);
				break;
			case ']':
				_make_token(TK_BRACKET_CLOSE);
				break;
			case '(':
				_make_token(TK_PARENTHESIS_OPEN);
				break;
			case ')':
				_make_token(TK_PARENTHESIS_CLOSE);
				break;
			case ',':
				_make_token(TK_COMMA);
				break;
			case ';':
				_make_token(TK_SEMICOLON);
				break;
			case '?':
				_make_token(TK_QUESTION_MARK);
				break;
			case ':':
				_make_token(TK_COLON); //for methods maybe but now useless.
				break;
			case '^': {
				if (GETCHAR(1)=='=') {
					_make_token(TK_OP_ASSIGN_BIT_XOR);
					INCPOS(1);
				} else {
					_make_token(TK_OP_BIT_XOR);
				}

			} break;
			case '~':
				_make_token(TK_OP_BIT_INVERT);
				break;
			case '&': {
				if (GETCHAR(1)=='&') {

					_make_token(TK_OP_AND);
					INCPOS(1);
				} else if (GETCHAR(1)=='=') {
					_make_token(TK_OP_ASSIGN_BIT_AND);
					INCPOS(1);
				} else {
					_make_token(TK_OP_BIT_AND);
				}
			} break;
			case '|': {
				if (GETCHAR(1)=='|') {

					_make_token(TK_OP_OR);
					INCPOS(1);
				} else if (GETCHAR(1)=='=') {
					_make_token(TK_OP_ASSIGN_BIT_OR);
					INCPOS(1);
				} else {
					_make_token(TK_OP_BIT_OR);
				}
			} break;
			case '*': {

				if (GETCHAR(1)=='=') {
					_make_token(TK_OP_ASSIGN_MUL);
					INCPOS(1);
				} else {
					_make_token(TK_OP_MUL);
				}
			} break;
			case '+': {

				if (GETCHAR(1)=='=') {
					_make_token(TK_OP_ASSIGN_ADD);
					INCPOS(1);
				//}  else if (GETCHAR(1)=='+') {
				//	_make_token(TK_OP_PLUS_PLUS);
				//	INCPOS(1);
				} else {
					_make_token(TK_OP_ADD);
				}

			} break;
			case '-': {

				if (GETCHAR(1)=='=') {
					_make_token(TK_OP_ASSIGN_SUB);
					INCPOS(1);
				//}  else if (GETCHAR(1)=='-') {
				//	_make_token(TK_OP_MINUS_MINUS);
				//	INCPOS(1);
				} else {
					_make_token(TK_OP_SUB);
				}
			} break;
			case '%': {

				if (GETCHAR(1)=='=') {
					_make_token(TK_OP_ASSIGN_MOD);
					INCPOS(1);
				} else {
					_make_token(TK_OP_MOD);
				}
			} break;
			case '@':
				if( CharType(GETCHAR(1))!='"' && CharType(GETCHAR(1))!='\'' ) {
					_make_error("Unexpected '@'");
					return;
				}
				INCPOS(1);
				is_node_path=true;
				
			case '\'':
			case '"': {
	
				if (GETCHAR(0)=='\'')
					string_mode=STRING_SINGLE_QUOTE;
																	
																	
				int i=1;
				if (string_mode==STRING_DOUBLE_QUOTE && GETCHAR(i)=='"' && GETCHAR(i+1)=='"') {
					i+=2;
					string_mode=STRING_MULTILINE;

				}


				String str;
				while(true) {
					if (CharType(GETCHAR(i))==0) {

						_make_error("Unterminated String");
						return;
					} else if( string_mode==STRING_DOUBLE_QUOTE && CharType(GETCHAR(i))=='"' ) {
						break;
					} else if( string_mode==STRING_SINGLE_QUOTE && CharType(GETCHAR(i))=='\'' ) {
						break;
					} else if( string_mode==STRING_MULTILINE && CharType(GETCHAR(i))=='\"' &&  CharType(GETCHAR(i+1))=='\"' && CharType(GETCHAR(i+2))=='\"') {
						i+=2;
						break;
					} else if( string_mode!=STRING_MULTILINE && CharType(GETCHAR(i))=='\n') {
						_make_error("Unexpected EOL at String.");
						return;

					} else if (CharType(GETCHAR(i))=='\\') {
						//escaped characters...
						i++;
						CharType next = GETCHAR(i);
						if (next==0) {
							_make_error("Unterminated String");
							return;
						}
						CharType res=0;

						switch(next) {

							case 'a': res=7; break;
							case 'b': res=8; break;
							case 't': res=9; break;
							case 'n': res=10; break;
							case 'v': res=11; break;
							case 'f': res=12; break;
							case 'r': res=13; break;
							case '\'': res='\''; break;
							case '\"': res='\"'; break;
							case '\\': res='\\'; break;
							case '/': res='/'; break; //wtf

							case 'u': {
								//hexnumbarh - oct is deprecated
								i+=1;
								for(int j=0;j<4;j++) {
									CharType c = GETCHAR(i+j);
									if (c==0) {
										_make_error("Unterminated String");
										return;
									}
									if (!((c>='0' && c<='9') || (c>='a' && c<='f') || (c>='A' && c<='F'))) {

										_make_error("Malformed hex constant in string");
										return;
									}
									CharType v;
									if (c>='0' && c<='9') {
										v=c-'0';
									} else if (c>='a' && c<='f') {
										v=c-'a';
										v+=10;
									} else if (c>='A' && c<='F') {
										v=c-'A';
										v+=10;
									} else {
										ERR_PRINT("BUG");
										v=0;
									}

									res<<=4;
									res|=v;


								}
								i+=3;

							} break;
							default: {

								_make_error("Invalid escape sequence");
								return;
							} break;
						}

						str+=res;

					} else {
						str+=CharType(GETCHAR(i));
					}
					i++;
				}
				INCPOS(i);

				if (is_node_path) {
					_make_constant(NodePath(str));
				} else {
					_make_constant(str);
				}

			} break;
			case 0xFFFF: {
				_make_token(TK_CURSOR);
			} break;
			default: {

				if (_is_number(GETCHAR(0)) || (GETCHAR(0)=='.' && _is_number(GETCHAR(1)))) {
					// parse number
					bool period_found=false;
					bool exponent_found=false;
					bool hexa_found=false;
					bool sign_found=false;

					String str;
					int i=0;

					while(true) {
						if (GETCHAR(i)=='.') {
							if (period_found || exponent_found) {
                                _make_error("Invalid numeric constant at '.'");
								return;
							}
							period_found=true;
						} else if (GETCHAR(i)=='x') {
                            if (hexa_found || str.length()!=1 || !( (i==1 && str[0]=='0') || (i==2 && str[1]=='0' && str[0]=='-') ) ) {
                                _make_error("Invalid numeric constant at 'x'");
								return;
							}
							hexa_found=true;
                        } else if (!hexa_found && GETCHAR(i)=='e') {
							if (hexa_found || exponent_found) {
                                _make_error("Invalid numeric constant at 'e'");
								return;
							}
							exponent_found=true;
						} else if (_is_number(GETCHAR(i))) {
							//all ok
						} else if (hexa_found && _is_hex(GETCHAR(i))) {

						} else if ((GETCHAR(i)=='-' || GETCHAR(i)=='+') && exponent_found) {
							if (sign_found) {
                                _make_error("Invalid numeric constant at '-'");
								return;
							}
							sign_found=true;
						} else
							break;

						str+=CharType(GETCHAR(i));
						i++;
					}

                    if (!( _is_number(str[str.length()-1]) || (hexa_found && _is_hex(str[str.length()-1])))) {
                        _make_error("Invalid numeric constant: "+str);
						return;
					}

					INCPOS(str.length());
                    if (hexa_found) {
                        int val = str.hex_to_int();
                        _make_constant(val);
                    } else if (period_found) {
						real_t val = str.to_double();
						//print_line("*%*%*%*% to convert: "+str+" result: "+rtos(val));
						_make_constant(val);
                    } else {
						int val = str.to_int();
						_make_constant(val);

					}

					return;
				}

				if (GETCHAR(0)=='.') {
					//parse period
					_make_token(TK_PERIOD);
					break;
				}

				if (_is_text_char(GETCHAR(0))) {
					// parse identifier
					String str;
					str+=CharType(GETCHAR(0));

					int i=1;
					while(_is_text_char(GETCHAR(i))) {
						str+=CharType(GETCHAR(i));
						i++;
					}

					bool identifier=false;

					if (str=="null") {
						_make_constant(Variant());

					} else if (str=="true") {
						_make_constant(true);

					} else if (str=="false") {
						_make_constant(false);
					} else {

						bool found=false;

						struct _bit { Variant::Type type; const char *text;};
						//built in types

						static const  _bit type_list[]={
							//types
							{Variant::BOOL,"bool"},
							{Variant::INT,"int"},
							{Variant::REAL,"float"},
							{Variant::STRING,"String"},
							{Variant::VECTOR2,"vec2"},
							{Variant::VECTOR2,"Vector2"},
							{Variant::RECT2,"Rect2"},
							{Variant::MATRIX32,"Matrix32"},
							{Variant::MATRIX32,"mat32"},
							{Variant::VECTOR3,"vec3"},
							{Variant::VECTOR3,"Vector3"},
							{Variant::_AABB,"AABB"},
							{Variant::_AABB,"Rect3"},
							{Variant::PLANE,"Plane"},
							{Variant::QUAT,"Quat"},
							{Variant::MATRIX3,"mat3"},
							{Variant::MATRIX3,"Matrix3"},
							{Variant::TRANSFORM,"trn"},
							{Variant::TRANSFORM,"Transform"},
							{Variant::COLOR,"Color"},
							{Variant::IMAGE,"Image"},
							{Variant::_RID,"RID"},
							{Variant::OBJECT,"Object"},
							{Variant::INPUT_EVENT,"InputEvent"},
							{Variant::NODE_PATH,"NodePath"},
							{Variant::DICTIONARY,"dict"},
							{Variant::DICTIONARY,"Dictionary"},
							{Variant::ARRAY,"Array"},
							{Variant::RAW_ARRAY,"RawArray"},
							{Variant::INT_ARRAY,"IntArray"},
							{Variant::REAL_ARRAY,"FloatArray"},
							{Variant::STRING_ARRAY,"StringArray"},
							{Variant::VECTOR2_ARRAY,"Vector2Array"},
							{Variant::VECTOR3_ARRAY,"Vector3Array"},
							{Variant::COLOR_ARRAY,"ColorArray"},
							{Variant::VARIANT_MAX,NULL},
						};

						{


							int idx=0;

							while(type_list[idx].text) {

								if (str==type_list[idx].text) {
									_make_type(type_list[idx].type);
									found=true;
									break;
								}
								idx++;
							}
						}

						if (!found) {

							//built in func?

							for(int i=0;i<GDFunctions::FUNC_MAX;i++) {

								if (str==GDFunctions::get_func_name(GDFunctions::Function(i))) {

									_make_built_in_func(GDFunctions::Function(i));
									found=true;
									 break;
								}
							}

							//keywor
						}

						if (!found) {


							struct _kws { Token token; const char *text;};

							static const  _kws keyword_list[]={
								//ops
								{TK_OP_IN,"in"},
								{TK_OP_NOT,"not"},
								{TK_OP_OR,"or"},
								{TK_OP_AND,"and"},
								//func
								{TK_PR_FUNCTION,"func"},
								{TK_PR_FUNCTION,"function"},
								{TK_PR_CLASS,"class"},
								{TK_PR_EXTENDS,"extends"},
								{TK_PR_TOOL,"tool"},
								{TK_PR_STATIC,"static"},
								{TK_PR_EXPORT,"export"},
								{TK_PR_SETGET,"setget"},
								{TK_PR_VAR,"var"},
								{TK_PR_PRELOAD,"preload"},
								{TK_PR_ASSERT,"assert"},
								{TK_PR_YIELD,"yield"},
								{TK_PR_CONST,"const"},
								//controlflow
								{TK_CF_IF,"if"},
								{TK_CF_ELIF,"elif"},
								{TK_CF_ELSE,"else"},
								{TK_CF_FOR,"for"},
								{TK_CF_WHILE,"while"},
								{TK_CF_DO,"do"},
								{TK_CF_SWITCH,"switch"},
								{TK_CF_BREAK,"break"},
								{TK_CF_CONTINUE,"continue"},
								{TK_CF_RETURN,"return"},
								{TK_CF_PASS,"pass"},
								{TK_SELF,"self"},
								{TK_ERROR,NULL}
							};

							int idx=0;
							found=false;

							while(keyword_list[idx].text) {

								if (str==keyword_list[idx].text) {
									_make_token(keyword_list[idx].token);
									found=true;
									break;
								}
								idx++;
							}
						}

						if (!found)
							identifier=true;
					}


					if (identifier) {
						_make_identifier(str);
					}
					INCPOS(str.length());
					return;
				}

				_make_error("Unknown character");
				return;

			} break;
		}

		INCPOS(1);
		break;
	}

}
Ejemplo n.º 14
0
	inline std::basic_ostream<CharType>& operator << (std::basic_ostream<CharType>& os, const Vector2D<Type>& v)
	{
		return os << CharType('(') << v.x << CharType(',') << v.y << CharType(')');
	}
Ejemplo n.º 15
0
Value            CharValue(char16_t  c){return Value(     CharType(),2,c);}
Ejemplo n.º 16
0
// 
// TBuf::NextChar()
//
// Get the next char from the buffer, returning FALSE
// when there is no data left.
//
Bool TBuf::NextChar(U8 *c)
{
  ASSERT(c);

  U8 n;
    
  // Filter comments and carriage returns
  do
  {
    ASSERT(bufPos <= bufSize);

    // Check if at end of data
    if (bufPos == bufSize)
    {
      // Are we reading a string constant
      if (constantOn)
      {
        TokenError("reached end of data in string constant");
      }

      return (FALSE);
    }

    // Get the next character
    n = (U8) bufData[bufPos++];

    // Adjust parse position
    if (n == TBUF_CH_EOL)
    {
      // Are we reading a string constant
      if (constantOn)
      {
        TokenError("newline in string constant");
      }
    
      lastXPos = xPos;
      xPos = 0;
      yPos++;
    }
    else if (n != TBUF_CH_CR)
    {
      xPos++;
    }

    // if not in a constant, check commenting
    if (!constantOn)
    {
      // turn line comment mode on
      if (!commentOn && (CharType(n) == COMMENT))
      { 
        commentOn = TRUE;
      }

      // turn line comment mode off
      if (commentOn && (n == TBUF_CH_EOL))
      {
        commentOn = FALSE;
      }
    }
  }
  while(commentOn || (n == TBUF_CH_CR));

  // convert tabs to spaces
  if (n == TBUF_CH_TAB)
  {
    n = TBUF_CH_SPACE;
  }

  // set the char value
  *c = n;
  return(TRUE);
}
Ejemplo n.º 17
0
//
// TBuf::NextToken
//
// Reads the next token into 'dest'. Skips any preceeding spaces, 
// stopping when either a space, a punctuation char, or end of data
// occurs.  Generates an error if token exceeds max token length.
//
// Returns :
// 
//  TR_OK   - if token was read with no problems
//  TR_PUN  - if punctuation char was found
//  TR_EOF  - if end of data reached
//
TBufResult TBuf::NextToken()
{
  ASSERT(setup);

  U8 c;
  U32 tokenLen = 0;
  char *sPtr;

  // Have we peeked at the next token
  if (peekState == PS_ON)
  {
    peekState = PS_OFF;
    Utils::Strcpy(prevToken, lastToken);
    Utils::Strcpy(lastToken, peekToken);
    return (peekVal);
  }

  // Setup peek operation
  if (peekState == PS_PRE)
  {
    // Store next token in peek buffer
    sPtr = peekToken;

    // We are now in peek mode
    peekState = PS_ON;
  }
  else
  { 
    // Save previous token
    Utils::Strcpy(prevToken, lastToken);
  
    // Store next token in last token buffer
    sPtr = lastToken;
  }

  // Clear token buffer
  *sPtr = '\0';

  // Read next token
  for (;;)
  {
    // Have we reached the end of the data
    if (!NextChar(&c))
    {
      // Return token if we have one
      return (tokenLen ? TR_OK : TR_EOF);
    }

    // Finish reading constant
    if (constantOn && c == constantChar)
    { 
      constantOn = FALSE;
    }
   
    // Ignore these checks if reading a constant
    if (!constantOn)
    {
      // What char have we got
      if ((c == TBUF_CH_SPACE) || (c == TBUF_CH_EOL))
      {
        // Have we read a token
        if (tokenLen)
        {
          return (TR_OK);
        }

        // Ignore this character
        continue;
      }

      // Check for punctuation
      if (CharType(c) == PUNCTUATION)
      {
        // Haven't read anything yet
        if (!tokenLen)
        {
          sPtr[0] = c;
          sPtr[1] = '\0';
          return(TR_PUN);
        }

        // Successfully read a token
        StepBack();
        return(TR_OK);
      }
    }

    // Add this char to the current token
    if (tokenLen < TBUF_MAX_TOKEN - 1)
    {
      sPtr[tokenLen++] = c;
      sPtr[tokenLen] = '\0';
    }
    else
    {
      TokenError("maximum token length [%d bytes] exceeded", TBUF_MAX_TOKEN-1);
    }
  }
}
Ejemplo n.º 18
0
	inline std::basic_ostream<CharType>& operator << (std::basic_ostream<CharType>& os, const Line& line)
	{
		return os << CharType('(') << line.p[0] << CharType(',') << line.p[1] << CharType(')');
	}
Ejemplo n.º 19
0
			return afc::math::min<const std::size_t>(std::numeric_limits<std::size_t>::max() / sizeof(CharType) - 1,
					std::size_t(std::numeric_limits<std::ptrdiff_t>::max()));
		}

		static const CharType empty[1];

		// If not nullptr then one character is reserved for the terminating character.
		// Non-[] deleter is specified since placement allocation is used.
		CharType *m_buf;
		CharType *m_bufEnd;
		std::size_t m_capacity;
	};
}

template<typename CharType, afc::AllocMode allocMode>
const CharType afc::FastStringBuffer<CharType, allocMode>::empty[1] = {CharType(0)};

template<typename CharType, afc::AllocMode allocMode>
inline std::size_t afc::FastStringBuffer<CharType, allocMode>::nextStorageSize(const std::size_t capacity) noexcept(noexcept(badAlloc()))
{
	static_assert(allocMode == afc::AllocMode::pow2 || allocMode == afc::AllocMode::accurate, "Unsupported allocMode.");

	// FastStringBuffer::reserve() does not expand storage if capacity == 0.
	assert(capacity > 0);

	if (allocMode == afc::AllocMode::pow2) {
		constexpr std::size_t maxStorageSize = maxCapacity() + 1;
		const std::size_t requestedStorageSize = capacity + 1;

		// FastStringBuffer::reserve(std::size_t) does not expand storage if capacity == 0.
		/* Since newStorageSize is always a power of two, the first value that