Token scanner( FILE *source )
{
char c;
Token token;
if(!tokenBufferEmpty) {
tokenBufferEmpty = 1;
return tokenBuffer;
}
while( !feof(source) ){
c = fgetc(source);
while( isspace(c) )
c = fgetc(source);
if( isdigit(c) )
return getNumericToken(source, c);
else if( isalpha(c) ){
token = getWordToken(source, c);
return checkKeyword(token);
}
else
return getOperatorToken(source, c);
}
token.tok[0] = '\0';
token.type = EOFsymbol;
return token;
}
short SlkToken::get(void)
{
if (NULL == mSource || NULL == mErrorAndStringBuffer) {
return END_OF_SLK_INPUT_;
}
for (;;) {
if (*mIterator == '\0') {
if (isCanFinish()) {
newToken();
endTokenWithEof();
return END_OF_SLK_INPUT_;
} else {
if (!mIterator.Load()) {
newToken();
endTokenWithEof();
return END_OF_SLK_INPUT_;
}
}
}
int isSkip = TRUE;
//跳过注释与白空格
for (; isSkip && *mIterator != '\0';) {
isSkip = FALSE;
for (; isWhiteSpace(*mIterator); ++mIterator) {
if (*mIterator == '\n') {
++mLineNumber;
if (mCommentNum <= 0) {
mCommentOnNewLine = TRUE;
}
}
isSkip = TRUE;
}
//#引导的单行注释
if (*mIterator == '#') {
newComment();
for (; *mIterator != '\0' && *mIterator != '\n'; ++mIterator) {
if (*mIterator != '\r')
pushCommentChar(*mIterator);
}
endComment();
isSkip = TRUE;
}
//C++风格的单行注释与多行注释
if (*mIterator == '/' && (*(mIterator + 1) == '/' || *(mIterator + 1) == '*')) {
newComment();
pushCommentChar(*mIterator);
++mIterator;
if (*mIterator == '/') {
pushCommentChar(*mIterator);
++mIterator;
for (; *mIterator != '\0' && *mIterator != '\n'; ++mIterator) {
if (*mIterator != '\r')
pushCommentChar(*mIterator);
}
isSkip = TRUE;
} else if (*mIterator == '*') {
pushCommentChar(*mIterator);
++mIterator;
for (;;) {
if (*mIterator != '\0') {
if (*mIterator == '\n') {
pushCommentChar(*mIterator);
++mLineNumber;
} else if (*mIterator == '*' && *(mIterator + 1) == '/') {
pushCommentChar(*mIterator);
++mIterator;
pushCommentChar(*mIterator);
++mIterator;
break;
} else if (*mIterator != '\r') {
pushCommentChar(*mIterator);
}
} else {
if (mIterator.Load()) {
continue;
} else {
endComment();
newToken();
endTokenWithEof();
return END_OF_SLK_INPUT_;
}
}
++mIterator;
}
isSkip = TRUE;
}
endComment();
}
}
if (*mIterator != '\0')
break;
}
newToken();
if (isCanFinish())
setCanFinish(FALSE);
if (*mIterator == '{' && *(mIterator + 1) == ':') {
++mIterator;
++mIterator;
int line = mLineNumber;
//搜索脚本结束 :}
for (; *mIterator != '\0';) {
while (*mIterator != '\0' && *mIterator != ':') {
if (*mIterator == '\n')++mLineNumber;
pushTokenChar(*mIterator);
++mIterator;
}
if (*mIterator == '\0')
break;
IScriptSource::Iterator next = mIterator + 1;
if (*next == '}') {
++mIterator;
++mIterator;
break;
} else {
pushTokenChar(*mIterator);
++mIterator;
}
}
if (*mIterator == '\0') {
char* pInfo = mErrorAndStringBuffer->NewErrorInfo();
if (pInfo)
tsnprintf(pInfo, MAX_ERROR_INFO_CAPACITY, "[line %d ]:ExternScript can't finish！", line);
}
endToken();
if (myhavelinefeed(mCurToken)){
removeFirstAndLastEmptyLine();
}
return SCRIPT_CONTENT_;
} else if (isOperator(*mIterator)) {//操作符
getOperatorToken();
return getOperatorTokenValue();
} else if (*mIterator == '.' && 0 == myisdigit(*(mIterator + 1), FALSE)) {
char c = *mIterator;
++mIterator;
pushTokenChar(c);
endToken();
return DOT_;
} else if (*mIterator == '(') {
++mIterator;
pushTokenChar('(');
endToken();
return LPAREN_;
} else if (*mIterator == ')') {
++mIterator;
pushTokenChar(')');
endToken();
return RPAREN_;
} else if (*mIterator == '[') {
++mIterator;
pushTokenChar('[');
endToken();
return LBRACK_;
} else if (*mIterator == ']') {
++mIterator;
pushTokenChar(']');
endToken();
return RBRACK_;
} else if (*mIterator == '{') {
++mIterator;
pushTokenChar('{');
endToken();
return LBRACE_;
} else if (*mIterator == '}') {
++mIterator;
pushTokenChar('}');
endToken();
return RBRACE_;
} else if (*mIterator == ',') {
++mIterator;
pushTokenChar(',');
endToken();
return COMMA_;
} else if (*mIterator == ';') {
++mIterator;
pushTokenChar(';');
endToken();
return SEMI_;
} else {//关键字、标识符或常数
if (*mIterator == '"' || *mIterator == '\'') {//引号括起来的名称或关键字
int line = mLineNumber;
char c = *mIterator;
for (++mIterator; *mIterator != '\0' && *mIterator != c;) {
if (*mIterator == '\n')++mLineNumber;
if (*mIterator == '\\') {
//pushTokenChar(*mIterator);
++mIterator;
}
pushTokenChar(*mIterator);
++mIterator;
if (*mIterator == '\0') {
if (mIterator.Load()) {
continue;
} else {
char* pInfo = mErrorAndStringBuffer->NewErrorInfo();
if (pInfo)
tsnprintf(pInfo, MAX_ERROR_INFO_CAPACITY, "[line %d ]:String can't finish！", line);
endTokenWithEof();
return END_OF_SLK_INPUT_;
}
}
}
if (*mIterator != '\0') {
++mIterator;
} else {
char* pInfo = mErrorAndStringBuffer->NewErrorInfo();
if (pInfo)
tsnprintf(pInfo, MAX_ERROR_INFO_CAPACITY, "[line %d ]:String can't finish！", line);
}
endToken();
if (myhavelinefeed(mCurToken)){
removeFirstAndLastEmptyLine();
}
return STRING_;
} else {
int isNum = TRUE;
int isHex = FALSE;
if (*mIterator == '0' && *(mIterator + 1) == 'x') {
isHex = TRUE;
pushTokenChar(*mIterator);
++mIterator;
pushTokenChar(*mIterator);
++mIterator;
}
for (; !isSpecialChar(*mIterator); ++mIterator) {
if (*mIterator == '#')
break;
else if (*mIterator == '/') {
IScriptSource::Iterator next = mIterator + 1;
if (*next != '\0' && (*next == '/' || *next == '*')) {
break;
}
} else if (*mIterator == '.') {
if (!isNum) {
break;
} else {
IScriptSource::Iterator next = mIterator + 1;
if (0 == myisdigit(*next, isHex)) {
break;
}
}
} else if (0 == myisdigit(*mIterator, isHex)) {
isNum = FALSE;
}
pushTokenChar(*mIterator);
}
endToken();
if (isNum) {
return NUMBER_;
} else {
int token = handleStringOrScriptDelimiter();
if (token)
return token;
return IDENTIFIER_;
}
}
}
}
short SlkToken::get(void)
{
if (NULL == mSource || NULL == mErrorAndStringBuffer) {
return END_OF_SLK_INPUT_;
}
newToken();
for (;;) {
if (*mIterator == '\0') {
if (isCanFinish()) {
endTokenWithEof();
return END_OF_SLK_INPUT_;
} else {
if (!mIterator.Load()) {
endTokenWithEof();
return END_OF_SLK_INPUT_;
}
}
}
int isSkip = TRUE;
//跳过注释与白空格
for (; isSkip && *mIterator != '\0';) {
isSkip = FALSE;
for (; isWhiteSpace(*mIterator); ++mIterator) {
if (*mIterator == '\n')++mLineNumber;
isSkip = TRUE;
}
//#引导的单行注释
if (*mIterator == '#') {
for (; *mIterator != '\0' && *mIterator != '\n'; ++mIterator);
isSkip = TRUE;
}
//C++风格的单行注释与多行注释
if (*mIterator == '/' && (*(mIterator + 1) == '/' || *(mIterator + 1) == '*')) {
++mIterator;
if (*mIterator == '/') {
++mIterator;
for (; *mIterator != '\0' && *mIterator != '\n'; ++mIterator);
isSkip = TRUE;
} else if (*mIterator == '*') {
++mIterator;
for (;;) {
if (*mIterator != '\0') {
if (*mIterator == '\n')++mLineNumber;
if (*mIterator == '*' && *(mIterator + 1) == '/') {
++mIterator;
++mIterator;
break;
}
} else {
if (mIterator.Load()) {
continue;
} else {
endTokenWithEof();
return END_OF_SLK_INPUT_;
}
}
++mIterator;
}
isSkip = TRUE;
}
}
}
if (*mIterator != '\0')
break;
}
if (isCanFinish())
setCanFinish(FALSE);
if (*mIterator == '{' && *(mIterator + 1) == ':') {
++mIterator;
++mIterator;
int line = mLineNumber;
//搜索脚本结束 :}
for (; *mIterator != '\0';) {
while (*mIterator != '\0' && *mIterator != ':') {
if (*mIterator == '\n')++mLineNumber;
pushTokenChar(*mIterator);
++mIterator;
}
if (*mIterator == '\0')
break;
IScriptSource::Iterator next = mIterator + 1;
if (*next == '}') {
++mIterator;
++mIterator;
break;
} else {
pushTokenChar(*mIterator);
++mIterator;
}
}
if (*mIterator == '\0') {
char* pInfo = mErrorAndStringBuffer->NewErrorInfo();
if (pInfo)
tsnprintf(pInfo, MAX_ERROR_INFO_CAPACITY, "[line %d ]:ExternScript can't finish！", line);
}
endToken();
return SCRIPT_CONTENT_;
} else if (isOperator(*mIterator))//操作符
{
getOperatorToken();
return getOperatorTokenValue();
} else if (*mIterator == '.' && 0 == myisdigit(*(mIterator + 1), FALSE)) {
char c = *mIterator;
++mIterator;
pushTokenChar(c);
endToken();
return DOT_;
} else if (isDelimiter(*mIterator))//分隔符
{
char c = *mIterator;
++mIterator;
pushTokenChar(c);
endToken();
switch (c) {
case '(':
return LPAREN_;
case ')':
return RPAREN_;
case '[':
return LBRACK_;
case ']':
return RBRACK_;
case '{':
return LBRACE_;
case '}':
return RBRACE_;
case ',':
return COMMA_;
case ';':
return SEMI_;
default:
return END_OF_SLK_INPUT_;
}
} else//关键字、标识符或常数
{
if (*mIterator == '"' || *mIterator == '\'')//引号括起来的名称或关键字
{
int line = mLineNumber;
char c = *mIterator;
for (++mIterator; *mIterator != '\0' && *mIterator != c;) {
if (*mIterator == '\n')++mLineNumber;
if (*mIterator == '\\') {
//pushTokenChar(*mIterator);
++mIterator;
}
pushTokenChar(*mIterator);
++mIterator;
if (*mIterator == '\0') {
if (mIterator.Load()) {
continue;
} else {
char* pInfo = mErrorAndStringBuffer->NewErrorInfo();
if (pInfo)
tsnprintf(pInfo, MAX_ERROR_INFO_CAPACITY, "[line %d ]:String can't finish！", line);
endTokenWithEof();
return END_OF_SLK_INPUT_;
}
}
}
if (*mIterator != '\0') {
++mIterator;
} else {
char* pInfo = mErrorAndStringBuffer->NewErrorInfo();
if (pInfo)
tsnprintf(pInfo, MAX_ERROR_INFO_CAPACITY, "[line %d ]:String can't finish！", line);
}
endToken();
return STRING_;
} else {
int isNum = TRUE;
int isHex = FALSE;
if (*mIterator == '0' && *(mIterator + 1) == 'x') {
isHex = TRUE;
pushTokenChar(*mIterator);
++mIterator;
pushTokenChar(*mIterator);
++mIterator;
}
for (; *mIterator != '\0' && !isDelimiter(*mIterator) && !isWhiteSpace(*mIterator) && !isOperator(*mIterator); ++mIterator) {
if (*mIterator == '#')
break;
else if (*mIterator == '/') {
IScriptSource::Iterator next = mIterator + 1;
if (*next != '\0' && (*next == '/' || *next == '*')) {
break;
}
} else if (*mIterator == '.') {
if (!isNum) {
break;
} else {
IScriptSource::Iterator next = mIterator + 1;
if (0 == myisdigit(*next, isHex)) {
break;
}
}
} else if (0 == myisdigit(*mIterator, isHex)) {
isNum = FALSE;
}
pushTokenChar(*mIterator);
}
endToken();
if (isNum)
return NUMBER_;
else
return IDENTIFIER_;
}
}
}
