コード例 #1
0
ファイル: AcDc.c プロジェクト: Yee-Lee/compiler15_c--
Token scanner( FILE *source )
{
    char c;
    Token token;

    if(!tokenBufferEmpty) {
        tokenBufferEmpty = 1;
        return tokenBuffer;
    }

    while( !feof(source) ){
        c = fgetc(source);
        while( isspace(c) )
            c = fgetc(source);

        if( isdigit(c) )
            return getNumericToken(source, c);
        else if( isalpha(c) ){
            token = getWordToken(source, c);
            return checkKeyword(token);
        }
        else
            return getOperatorToken(source, c);
    }
    token.tok[0] = '\0';
    token.type = EOFsymbol;
    return token;
}
コード例 #2
0
ファイル: SlkToken.cpp プロジェクト: dreamanlan/DSL
short SlkToken::get(void)
{
  if (NULL == mSource || NULL == mErrorAndStringBuffer) {
    return END_OF_SLK_INPUT_;
  }

  for (;;) {
    if (*mIterator == '\0') {
      if (isCanFinish()) {
        newToken();
        endTokenWithEof();
        return END_OF_SLK_INPUT_;
      } else {
        if (!mIterator.Load()) {
          newToken();
          endTokenWithEof();
          return END_OF_SLK_INPUT_;
        }
      }
    }

    int isSkip = TRUE;
    //跳过注释与白空格
    for (; isSkip && *mIterator != '\0';) {
      isSkip = FALSE;
      for (; isWhiteSpace(*mIterator); ++mIterator) {
        if (*mIterator == '\n') {
          ++mLineNumber;
          if (mCommentNum <= 0) {
            mCommentOnNewLine = TRUE;
          }
        }
        isSkip = TRUE;
      }
      //#引导的单行注释
      if (*mIterator == '#') {
        newComment();
        for (; *mIterator != '\0' && *mIterator != '\n'; ++mIterator) {
          if (*mIterator != '\r')
            pushCommentChar(*mIterator);
        }
        endComment();
        isSkip = TRUE;
      }
      //C++风格的单行注释与多行注释
      if (*mIterator == '/' && (*(mIterator + 1) == '/' || *(mIterator + 1) == '*')) {
        newComment();
        pushCommentChar(*mIterator);
        ++mIterator;
        if (*mIterator == '/') {
          pushCommentChar(*mIterator);
          ++mIterator;
          for (; *mIterator != '\0' && *mIterator != '\n'; ++mIterator) {
            if (*mIterator != '\r')
              pushCommentChar(*mIterator);
          }
          isSkip = TRUE;
        } else if (*mIterator == '*') {
          pushCommentChar(*mIterator);
          ++mIterator;
          for (;;) {
            if (*mIterator != '\0') {
              if (*mIterator == '\n') {
                pushCommentChar(*mIterator);
                ++mLineNumber;
              } else if (*mIterator == '*' && *(mIterator + 1) == '/') {
                pushCommentChar(*mIterator);
                ++mIterator;
                pushCommentChar(*mIterator);
                ++mIterator;
                break;
              } else if (*mIterator != '\r') {
                pushCommentChar(*mIterator);
              }
            } else {
              if (mIterator.Load()) {
                continue;
              } else {
                endComment();
                newToken();
                endTokenWithEof();
                return END_OF_SLK_INPUT_;
              }
            }
            ++mIterator;
          }
          isSkip = TRUE;
        }
        endComment();
      }
    }
    if (*mIterator != '\0')
      break;
  }

  newToken();
  if (isCanFinish())
    setCanFinish(FALSE);

  if (*mIterator == '{' && *(mIterator + 1) == ':') {
    ++mIterator;
    ++mIterator;
    int line = mLineNumber;
    //搜索脚本结束 :}
    for (; *mIterator != '\0';) {
      while (*mIterator != '\0' && *mIterator != ':') {
        if (*mIterator == '\n')++mLineNumber;

        pushTokenChar(*mIterator);

        ++mIterator;
      }
      if (*mIterator == '\0')
        break;
      IScriptSource::Iterator next = mIterator + 1;
      if (*next == '}') {
        ++mIterator;
        ++mIterator;
        break;
      } else {
        pushTokenChar(*mIterator);

        ++mIterator;
      }
    }
    if (*mIterator == '\0') {
      char* pInfo = mErrorAndStringBuffer->NewErrorInfo();
      if (pInfo)
        tsnprintf(pInfo, MAX_ERROR_INFO_CAPACITY, "[line %d ]:ExternScript can't finish!", line);
    }
    endToken();
    if (myhavelinefeed(mCurToken)){
      removeFirstAndLastEmptyLine();
    }
    return SCRIPT_CONTENT_;
  } else if (isOperator(*mIterator)) {//操作符
    getOperatorToken();
    return getOperatorTokenValue();
  } else if (*mIterator == '.' && 0 == myisdigit(*(mIterator + 1), FALSE)) {
    char c = *mIterator;
    ++mIterator;

    pushTokenChar(c);
    endToken();
    return DOT_;
  } else if (*mIterator == '(') {
    ++mIterator;
    pushTokenChar('(');
    endToken();
    return LPAREN_;
  } else if (*mIterator == ')') {
    ++mIterator;
    pushTokenChar(')');
    endToken();
    return RPAREN_;
  } else if (*mIterator == '[') {
    ++mIterator;
    pushTokenChar('[');
    endToken();
    return LBRACK_;
  } else if (*mIterator == ']') {
    ++mIterator;
    pushTokenChar(']');
    endToken();
    return RBRACK_;
  } else if (*mIterator == '{') {
    ++mIterator;
    pushTokenChar('{');
    endToken();
    return LBRACE_;
  } else if (*mIterator == '}') {
    ++mIterator;
    pushTokenChar('}');
    endToken();
    return RBRACE_;
  } else if (*mIterator == ',') {
    ++mIterator;
    pushTokenChar(',');
    endToken();
    return COMMA_;
  } else if (*mIterator == ';') {
    ++mIterator;
    pushTokenChar(';');
    endToken();
    return SEMI_;
  } else {//关键字、标识符或常数
    if (*mIterator == '"' || *mIterator == '\'') {//引号括起来的名称或关键字
      int line = mLineNumber;
      char c = *mIterator;
      for (++mIterator; *mIterator != '\0' && *mIterator != c;) {
        if (*mIterator == '\n')++mLineNumber;
        if (*mIterator == '\\') {
          //pushTokenChar(*mIterator);
          ++mIterator;
        }

        pushTokenChar(*mIterator);
        ++mIterator;

        if (*mIterator == '\0') {
          if (mIterator.Load()) {
            continue;
          } else {
            char* pInfo = mErrorAndStringBuffer->NewErrorInfo();
            if (pInfo)
              tsnprintf(pInfo, MAX_ERROR_INFO_CAPACITY, "[line %d ]:String can't finish!", line);
            endTokenWithEof();
            return END_OF_SLK_INPUT_;
          }
        }
      }
      if (*mIterator != '\0') {
        ++mIterator;
      } else {
        char* pInfo = mErrorAndStringBuffer->NewErrorInfo();
        if (pInfo)
          tsnprintf(pInfo, MAX_ERROR_INFO_CAPACITY, "[line %d ]:String can't finish!", line);
      }
      endToken();
      if (myhavelinefeed(mCurToken)){
        removeFirstAndLastEmptyLine();
      }
      return STRING_;
    } else {
      int isNum = TRUE;
      int isHex = FALSE;
      if (*mIterator == '0' && *(mIterator + 1) == 'x') {
        isHex = TRUE;
        pushTokenChar(*mIterator);
        ++mIterator;
        pushTokenChar(*mIterator);
        ++mIterator;
      }
      for (; !isSpecialChar(*mIterator); ++mIterator) {
        if (*mIterator == '#')
          break;
        else if (*mIterator == '/') {
          IScriptSource::Iterator next = mIterator + 1;
          if (*next != '\0' && (*next == '/' || *next == '*')) {
            break;
          }
        } else if (*mIterator == '.') {
          if (!isNum) {
            break;
          } else {
            IScriptSource::Iterator next = mIterator + 1;
            if (0 == myisdigit(*next, isHex)) {
              break;
            }
          }
        } else if (0 == myisdigit(*mIterator, isHex)) {
          isNum = FALSE;
        }

        pushTokenChar(*mIterator);
      }
      endToken();
      if (isNum) {
        return NUMBER_;
      } else {
        int token = handleStringOrScriptDelimiter();
        if (token)
          return token;
        return IDENTIFIER_;
      }
    }
  }
}
コード例 #3
0
ファイル: SlkToken.cpp プロジェクト: CQiao/DSL
short SlkToken::get(void)
{
  if (NULL == mSource || NULL == mErrorAndStringBuffer) {
    return END_OF_SLK_INPUT_;
  }
  newToken();

  for (;;) {
    if (*mIterator == '\0') {
      if (isCanFinish()) {
        endTokenWithEof();
        return END_OF_SLK_INPUT_;
      } else {
        if (!mIterator.Load()) {
          endTokenWithEof();
          return END_OF_SLK_INPUT_;
        }
      }
    }

    int isSkip = TRUE;
    //跳过注释与白空格
    for (; isSkip && *mIterator != '\0';) {
      isSkip = FALSE;
      for (; isWhiteSpace(*mIterator); ++mIterator) {
        if (*mIterator == '\n')++mLineNumber;
        isSkip = TRUE;
      }
      //#引导的单行注释
      if (*mIterator == '#') {
        for (; *mIterator != '\0' && *mIterator != '\n'; ++mIterator);
        isSkip = TRUE;
      }
      //C++风格的单行注释与多行注释
      if (*mIterator == '/' && (*(mIterator + 1) == '/' || *(mIterator + 1) == '*')) {
        ++mIterator;
        if (*mIterator == '/') {
          ++mIterator;
          for (; *mIterator != '\0' && *mIterator != '\n'; ++mIterator);
          isSkip = TRUE;
        } else if (*mIterator == '*') {
          ++mIterator;
          for (;;) {
            if (*mIterator != '\0') {
              if (*mIterator == '\n')++mLineNumber;
              if (*mIterator == '*' && *(mIterator + 1) == '/') {
                ++mIterator;
                ++mIterator;
                break;
              }
            } else {
              if (mIterator.Load()) {
                continue;
              } else {
                endTokenWithEof();
                return END_OF_SLK_INPUT_;
              }
            }
            ++mIterator;
          }
          isSkip = TRUE;
        }
      }
    }
    if (*mIterator != '\0')
      break;
  }

  if (isCanFinish())
    setCanFinish(FALSE);

  if (*mIterator == '{' && *(mIterator + 1) == ':') {
    ++mIterator;
    ++mIterator;
    int line = mLineNumber;
    //搜索脚本结束 :}
    for (; *mIterator != '\0';) {
      while (*mIterator != '\0' && *mIterator != ':') {
        if (*mIterator == '\n')++mLineNumber;

        pushTokenChar(*mIterator);

        ++mIterator;
      }
      if (*mIterator == '\0')
        break;
      IScriptSource::Iterator next = mIterator + 1;
      if (*next == '}') {
        ++mIterator;
        ++mIterator;
        break;
      } else {
        pushTokenChar(*mIterator);

        ++mIterator;
      }
    }
    if (*mIterator == '\0') {
      char* pInfo = mErrorAndStringBuffer->NewErrorInfo();
      if (pInfo)
        tsnprintf(pInfo, MAX_ERROR_INFO_CAPACITY, "[line %d ]:ExternScript can't finish!", line);
    }
    endToken();
    return SCRIPT_CONTENT_;
  } else if (isOperator(*mIterator))//操作符
  {
    getOperatorToken();
    return getOperatorTokenValue();
  } else if (*mIterator == '.' && 0 == myisdigit(*(mIterator + 1), FALSE)) {
    char c = *mIterator;
    ++mIterator;

    pushTokenChar(c);
    endToken();
    return DOT_;
  } else if (isDelimiter(*mIterator))//分隔符
  {
    char c = *mIterator;
    ++mIterator;

    pushTokenChar(c);
    endToken();

    switch (c) {
    case '(':
      return LPAREN_;
    case ')':
      return RPAREN_;
    case '[':
      return LBRACK_;
    case ']':
      return RBRACK_;
    case '{':
      return LBRACE_;
    case '}':
      return RBRACE_;
    case ',':
      return COMMA_;
    case ';':
      return SEMI_;
    default:
      return END_OF_SLK_INPUT_;
    }
  } else//关键字、标识符或常数
  {
    if (*mIterator == '"' || *mIterator == '\'')//引号括起来的名称或关键字
    {
      int line = mLineNumber;
      char c = *mIterator;
      for (++mIterator; *mIterator != '\0' && *mIterator != c;) {
        if (*mIterator == '\n')++mLineNumber;
        if (*mIterator == '\\') {
          //pushTokenChar(*mIterator);
          ++mIterator;
        }

        pushTokenChar(*mIterator);
        ++mIterator;

        if (*mIterator == '\0') {
          if (mIterator.Load()) {
            continue;
          } else {
            char* pInfo = mErrorAndStringBuffer->NewErrorInfo();
            if (pInfo)
              tsnprintf(pInfo, MAX_ERROR_INFO_CAPACITY, "[line %d ]:String can't finish!", line);
            endTokenWithEof();
            return END_OF_SLK_INPUT_;
          }
        }
      }
      if (*mIterator != '\0') {
        ++mIterator;
      } else {
        char* pInfo = mErrorAndStringBuffer->NewErrorInfo();
        if (pInfo)
          tsnprintf(pInfo, MAX_ERROR_INFO_CAPACITY, "[line %d ]:String can't finish!", line);
      }
      endToken();
      return STRING_;
    } else {
      int isNum = TRUE;
      int isHex = FALSE;
      if (*mIterator == '0' && *(mIterator + 1) == 'x') {
        isHex = TRUE;
        pushTokenChar(*mIterator);
        ++mIterator;
        pushTokenChar(*mIterator);
        ++mIterator;
      }
      for (; *mIterator != '\0' && !isDelimiter(*mIterator) && !isWhiteSpace(*mIterator) && !isOperator(*mIterator); ++mIterator) {
        if (*mIterator == '#')
          break;
        else if (*mIterator == '/') {
          IScriptSource::Iterator next = mIterator + 1;
          if (*next != '\0' && (*next == '/' || *next == '*')) {
            break;
          }
        } else if (*mIterator == '.') {
          if (!isNum) {
            break;
          } else {
            IScriptSource::Iterator next = mIterator + 1;
            if (0 == myisdigit(*next, isHex)) {
              break;
            }
          }
        } else if (0 == myisdigit(*mIterator, isHex)) {
          isNum = FALSE;
        }

        pushTokenChar(*mIterator);
      }
      endToken();
      if (isNum)
        return NUMBER_;
      else
        return IDENTIFIER_;
    }
  }
}