예제 #1
0
bool MenuConf::readToken(FILE* f, char* token, int TokenSize)
{
    if(! isTokenChar(curChar)) {
        printf("%d, %d: Expected token character\n", lineIndex, caret);
        return false;
    }
    for(int i=0; i<TokenSize; i++) {
        if(! isTokenChar(curChar)) {
            token[i] = 0;
            return true;
        }
        token[i] = curChar;
        nextChar(f);
    }
    token[TokenSize-1] = 0;
    printf("%d, %d: Token too long\n", lineIndex, caret);
    return false;
}
bool isValidToken(const char* start, const char* end)
{
    if (end <= start)
        return false;
    for (; start < end; ++start) {
        if (!isTokenChar(*start))
            return false;
    }
    return true;
}
예제 #3
0
bool isValidToken(char first, char second) {
    
    
    switch (first) {
        case ';':
            if (!isTokenChar(second))
                return true;
            break;
        case '|':
            if (isTokenChar(second) && first == second)
                return true;
            break;
        case  '&':
            if (isTokenChar(second) && first == second)
                return true;
            break;
        case '(':
            if (!isTokenChar(second))
                return true;
            break;
    }
    return false;
}
예제 #4
0
//there can be no consecutive token characters
//the exceptions: a '&' character must be followed immediately by another '&'
//                a '|' character may be followed by a '|'
//                tokens may be followed by an open paren '('
void checkForConsecutiveTokens(char* complete_command) {
    
    int pos=0;
    while(complete_command[pos] != '\0') {
        
        //there can be no consecutive token characters
        if ( isTokenChar(complete_command[pos]) ) {
            switch (complete_command[pos]) {
                    
                case '<': case '>': case ';':
                    if ( complete_command[pos+1] == '<' || complete_command[pos+1] == '>' || complete_command[pos+1] == '|' || complete_command[pos+1] == '&' || complete_command[pos+1] == ';' || complete_command[pos+1] == ')' ) {
                        fprintf(stderr, "Invalid syntax : checkForConsecutiveTokens!");
                        exit(1);
                    }
                    break;
                case '&':
                    pos++;
                    if ( complete_command[pos+1] == '<' || complete_command[pos+1] == '>' || complete_command[pos+1] == '|' || complete_command[pos+1] == '&' || complete_command[pos+1] == ';' || complete_command[pos+1] == ')' ) {
                        fprintf(stderr, "Invalid syntax : checkForConsecutiveTokens!");
                        exit(1);
                    }
                    break;
                case '|':
                    if ( complete_command[pos+1] == '<' || complete_command[pos+1] == '>' || complete_command[pos+1] == '&' || complete_command[pos+1] == ';' || complete_command[pos+1] == ')' ) {
                        fprintf(stderr, "Invalid syntax : checkForConsecutiveTokens!");
                        exit(1);
                    }
                    
                    if ( complete_command[pos+1] == '|' ) {
                        pos++;
                        if ( complete_command[pos+1] == '<' || complete_command[pos+1] == '>' || complete_command[pos+1] == '|' || complete_command[pos+1] == '&' || complete_command[pos+1] == ';' || complete_command[pos+1] == ')' ) {
                            fprintf(stderr, "Invalid syntax : checkForConsecutiveTokens!");
                            exit(1);
                        }
                    }
                    break;
                case '(':
                    if ( complete_command[pos+1] == '<' || complete_command[pos+1] == '>' || complete_command[pos+1] == '|' || complete_command[pos+1] == '&' || complete_command[pos+1] == ';' ) {
                        fprintf(stderr, "Invalid syntax : checkForConsecutiveTokens!");
                        exit(1);
                    }
                    break;
                default: // ')'
                    break;
            }
        }
        pos++;
    }
}
예제 #5
0
 bool CharTokenizer::incrementToken()
 {
     clearAttributes();
     int32_t length = 0;
     int32_t start = bufferIndex;
     CharArray buffer(termAtt->termBuffer());
     while (true)
     {
         if (bufferIndex >= dataLen)
         {
             offset += dataLen;
             dataLen = input->read(ioBuffer.get(), 0, ioBuffer.size());
             if (dataLen == -1)
             {
                 dataLen = 0; // so next offset += dataLen won't decrement offset
                 if (length > 0)
                     break;
                 else
                     return false;
             }
             bufferIndex = 0;
         }
         
         wchar_t c = ioBuffer[bufferIndex++];
         
         if (isTokenChar(c)) // if it's a token char
         {
             if (length == 0)
                 start = offset + bufferIndex - 1;
             else if (length == buffer.size())
                 buffer = termAtt->resizeTermBuffer(1 + length);
             
             buffer[length++] = normalize(c); // buffer it, normalized
             
             if (length == MAX_WORD_LEN) // buffer overflow!
                 break;
         }
         else if (length > 0) // at non-Letter with chars
             break; // return them
     }
     
     termAtt->setTermLength(length);
     offsetAtt->setOffset(correctOffset(start), correctOffset(start + length));
 
     return true;
 }
예제 #6
0
파일: Analyzers.cpp 프로젝트: RobinWuDev/Qt
bool CharTokenizer::next(Token* token){
	int32_t length = 0;
	int32_t start = offset;
	while (true) {
		TCHAR c;
		offset++;
		if (bufferIndex >= dataLen) {
			dataLen = input->read(ioBuffer, LUCENE_IO_BUFFER_SIZE);
			if (dataLen == -1)
				dataLen = 0;
			bufferIndex = 0;
		}
		if (dataLen <= 0 ) {
			if (length > 0)
				break;
			else
				return false;
		}else
			c = ioBuffer[bufferIndex++];
		if (isTokenChar(c)) {                       // if it's a token TCHAR

			if (length == 0)			  // start of token
				start = offset-1;

			buffer[length++] = normalize(c);          // buffer it, normalized

			if (length == LUCENE_MAX_WORD_LEN)		  // buffer overflow!
				break;

		} else if (length > 0)			  // at non-Letter w/ chars
			break;					  // return 'em

	}
	buffer[length]=0;
	token->set( buffer, start, start+length);
	return true;
}
예제 #7
0
/** reads the next token from the input file into the token buffer; returns whether a token was read */
static
SCIP_Bool getNextToken(
    BLKINPUT*             blkinput            /**< BLK reading data */
)
{
    SCIP_Bool hasdot;
    BLKEXPTYPE exptype;
    char* buf;
    int tokenlen;

    assert(blkinput != NULL);
    assert(blkinput->linepos < BLK_MAX_LINELEN);

    /* check the token stack */
    if( blkinput->npushedtokens > 0 )
    {
        swapPointers(&blkinput->token, &blkinput->pushedtokens[blkinput->npushedtokens-1]);
        blkinput->npushedtokens--;
        SCIPdebugMessage("(line %d) read token again: '%s'\n", blkinput->linenumber, blkinput->token);
        return TRUE;
    }

    /* skip delimiters */
    buf = blkinput->linebuf;
    while( isDelimChar(buf[blkinput->linepos]) )
    {
        if( buf[blkinput->linepos] == '\0' )
        {
            if( !getNextLine(blkinput) )
            {
                blkinput->section = BLK_END;
                SCIPdebugMessage("(line %d) end of file\n", blkinput->linenumber);
                return FALSE;
            }
            assert(blkinput->linepos == 0);
        }
        else
            blkinput->linepos++;
    }
    assert(blkinput->linepos < BLK_MAX_LINELEN);
    assert(!isDelimChar(buf[blkinput->linepos]));

    /* check if the token is a value */
    hasdot = FALSE;
    exptype = BLK_EXP_NONE;
    if( isValueChar(buf[blkinput->linepos], buf[blkinput->linepos+1], TRUE, &hasdot, &exptype) ) /*lint !e679*/
    {
        /* read value token */
        tokenlen = 0;
        do
        {
            assert(tokenlen < BLK_MAX_LINELEN);
            assert(!isDelimChar(buf[blkinput->linepos]));
            blkinput->token[tokenlen] = buf[blkinput->linepos];
            ++tokenlen;
            ++(blkinput->linepos);
            assert(blkinput->linepos < BLK_MAX_LINELEN);
        }
        while( isValueChar(buf[blkinput->linepos], buf[blkinput->linepos+1], FALSE, &hasdot, &exptype) ); /*lint !e679*/
    }
    else
    {
        /* read non-value token */
        tokenlen = 0;
        do
        {
            assert(tokenlen < BLK_MAX_LINELEN);
            blkinput->token[tokenlen] = buf[blkinput->linepos];
            tokenlen++;
            blkinput->linepos++;
            if( tokenlen == 1 && isTokenChar(blkinput->token[0]) )
                break;
        }
        while( !isDelimChar(buf[blkinput->linepos]) && !isTokenChar(buf[blkinput->linepos]) );

        /* if the token is an equation sense '<', '>', or '=', skip a following '='
         * if the token is an equality token '=' and the next character is a '<' or '>', replace the token by the inequality sense
         */
        if( tokenlen >= 1
                && (blkinput->token[tokenlen-1] == '<' || blkinput->token[tokenlen-1] == '>' || blkinput->token[tokenlen-1] == '=')
                && buf[blkinput->linepos] == '=' )
        {
            blkinput->linepos++;
        }
        else if( blkinput->token[tokenlen-1] == '=' && (buf[blkinput->linepos] == '<' || buf[blkinput->linepos] == '>') )
        {
            blkinput->token[tokenlen-1] = buf[blkinput->linepos];
            blkinput->linepos++;
        }
    }
    assert(tokenlen < BLK_MAX_LINELEN);
    blkinput->token[tokenlen] = '\0';

    SCIPdebugMessage("(line %d) read token: '%s'\n", blkinput->linenumber, blkinput->token);

    return TRUE;
}
예제 #8
0
/** reads the next token from the input file into the token buffer; returns whether a token was read */
static
SCIP_Bool getNextToken(
   SCIP*                 scip,               /**< SCIP data structure */
   LPINPUT*              lpinput             /**< LP reading data */
   )
{
   SCIP_Bool hasdot;
   LPEXPTYPE exptype;
   char* buf;
   int tokenlen;

   assert(lpinput != NULL);
   assert(lpinput->linepos < LP_MAX_LINELEN);

   /* check the token stack */
   if( lpinput->npushedtokens > 0 )
   {
      swapPointers(&lpinput->token, &lpinput->pushedtokens[lpinput->npushedtokens-1]);
      lpinput->npushedtokens--;

      SCIPdebugMessage("(line %d) read token again: '%s'\n", lpinput->linenumber, lpinput->token);
      return TRUE;
   }

   /* skip delimiters */
   buf = lpinput->linebuf;
   while( isDelimChar(buf[lpinput->linepos]) )
   {
      if( buf[lpinput->linepos] == '\0' )
      {
         if( !getNextLine(scip, lpinput) )
         {
            lpinput->section = LP_END;
            SCIPdebugMessage("(line %d) end of file\n", lpinput->linenumber);
            return FALSE;
         }
         assert(lpinput->linepos == 0);
      }
      else
         lpinput->linepos++;
   }
   assert(lpinput->linepos < LP_MAX_LINELEN);
   assert(!isDelimChar(buf[lpinput->linepos]));

   /* check if the token is a value */
   hasdot = FALSE;
   exptype = LP_EXP_NONE;
   if( isValueChar(buf[lpinput->linepos], buf[lpinput->linepos+1], TRUE, &hasdot, &exptype) )
   {
      /* read value token */
      tokenlen = 0;
      do
      {
         assert(tokenlen < LP_MAX_LINELEN);
         assert(!isDelimChar(buf[lpinput->linepos]));
         lpinput->token[tokenlen] = buf[lpinput->linepos];
         tokenlen++;
         lpinput->linepos++;
      }
      while( isValueChar(buf[lpinput->linepos], buf[lpinput->linepos+1], FALSE, &hasdot, &exptype) );
   }
   else
   {
      /* read non-value token */
      tokenlen = 0;
      do
      {
         assert(tokenlen < LP_MAX_LINELEN);
         lpinput->token[tokenlen] = buf[lpinput->linepos];
         tokenlen++;
         lpinput->linepos++;
         if( tokenlen == 1 && isTokenChar(lpinput->token[0]) )
            break;
      }
      while( !isDelimChar(buf[lpinput->linepos]) && !isTokenChar(buf[lpinput->linepos]) );

      /* if the token is a power sign '^', skip a following '2'
       * if the token is an equation sense '<', '>', or '=', skip a following '='
       * if the token is an equality token '=' and the next character is a '<' or '>', replace the token by the inequality sense
       */
      if( tokenlen >= 1 && lpinput->token[tokenlen-1] == '^' && buf[lpinput->linepos] == '2' )
      {
         lpinput->linepos++;
      }
      if( tokenlen >= 1
         && (lpinput->token[tokenlen-1] == '<' || lpinput->token[tokenlen-1] == '>' || lpinput->token[tokenlen-1] == '=')
         && buf[lpinput->linepos] == '=' )
      {
         lpinput->linepos++;
      }
      else if( lpinput->token[tokenlen-1] == '=' && (buf[lpinput->linepos] == '<' || buf[lpinput->linepos] == '>') )
      {
         lpinput->token[tokenlen-1] = buf[lpinput->linepos];
         lpinput->linepos++;
      }
   }
   assert(tokenlen < LP_MAX_LINELEN);
   lpinput->token[tokenlen] = '\0';

   SCIPdebugMessage("(line %d) read token: '%s'\n", lpinput->linenumber, lpinput->token);

   return TRUE;
}