/** Implementation of matchc for the lexer, overrides any * base implementation in the base recognizer. * * \remark * Note that the generated code lays down arrays of ints for constant * strings so that they are int UTF32 form! */ static ANTLR3_BOOLEAN matchc(pANTLR3_LEXER lexer, ANTLR3_UCHAR c) { if (lexer->input->istream->_LA(lexer->input->istream, 1) == c) { /* Matched correctly, do consume it */ lexer->input->istream->consume(lexer->input->istream); /* Reset any failed indicator */ lexer->rec->state->failed = ANTLR3_FALSE; return ANTLR3_TRUE; } /* Failed to match, exception and recovery time. */ if (lexer->rec->state->backtracking > 0) { lexer->rec->state->failed = ANTLR3_TRUE; return ANTLR3_FALSE; } lexer->rec->exConstruct(lexer->rec); /* TODO: Implement exception creation more fully perhaps */ lexer->recover(lexer); return ANTLR3_FALSE; }
/** Implementation of matchs for the lexer, overrides any * base implementation in the base recognizer. * * \remark * Note that the generated code lays down arrays of ints for constant * strings so that they are int UTF32 form! */ static ANTLR3_BOOLEAN matchs(pANTLR3_LEXER lexer, ANTLR3_UCHAR * string) { while (*string != ANTLR3_STRING_TERMINATOR) { if (lexer->input->istream->_LA(lexer->input->istream, 1) != (*string)) { if (lexer->rec->state->backtracking > 0) { lexer->rec->state->failed = ANTLR3_TRUE; return ANTLR3_FALSE; } lexer->rec->exConstruct(lexer->rec); lexer->rec->state->failed = ANTLR3_TRUE; /* TODO: Implement exception creation more fully perhaps */ lexer->recover(lexer); return ANTLR3_FALSE; } /* Matched correctly, do consume it */ lexer->input->istream->consume(lexer->input->istream); string++; /* Reset any failed indicator */ lexer->rec->state->failed = ANTLR3_FALSE; } return ANTLR3_TRUE; }
/*! * \brief * Stops using the current input stream and reverts to any prior * input stream on the stack. * * \param lexer * Description of parameter lexer. * * Pointer to a function that abandons the current input stream, whether it * is empty or not and reverts to the previous stacked input stream. * * \remark * The function fails silently if there are no prior input streams. */ static void popCharStream (pANTLR3_LEXER lexer) { pANTLR3_INPUT_STREAM input; // If we do not have a stream stack or we are already at the // stack bottom, then do nothing. // if (lexer->rec->state->streams != NULL && lexer->rec->state->streams->size(lexer->rec->state->streams) > 0) { // We just leave the current stream to its fate, we do not close // it or anything as we do not know what the programmer intended // for it. This method can always be overridden of course. // So just find out what was currently saved on the stack and use // that now, then pop it from the stack. // input = (pANTLR3_INPUT_STREAM)(lexer->rec->state->streams->top); lexer->rec->state->streams->pop(lexer->rec->state->streams); // Now install the stream as the current one. // lexer->setCharStream(lexer, input); lexer->input->istream->rewindLast(lexer->input->istream); } return; }
/*! * \brief * Change to a new input stream, remembering the old one. * * \param lexer * Pointer to the lexer instance to switch input streams for. * * \param input * New input stream to install as the current one. * * Switches the current character input stream to * a new one, saving the old one, which we will revert to at the end of this * new one. */ static void pushCharStream (pANTLR3_LEXER lexer, pANTLR3_INPUT_STREAM input) { // Do we need a new input stream stack? // if (lexer->rec->state->streams == NULL) { // This is the first call to stack a new // stream and so we must create the stack first. // lexer->rec->state->streams = antlr3StackNew(0); if (lexer->rec->state->streams == NULL) { // Could not do this, we just fail to push it. // TODO: Consider if this is what we want to do, but then // any programmer can override this method to do something else. return; } } // We have a stack, so we can save the current input stream // into it. // lexer->input->istream->mark(lexer->input->istream); lexer->rec->state->streams->push(lexer->rec->state->streams, lexer->input, NULL); // And now we can install this new one // lexer->setCharStream(lexer, input); }
static bool newInputStream(const std::string& filename, pANTLR3_LEXER lexer) { Debug("parser") << "Including " << filename << std::endl; // Create a new input stream and take advantage of built in stream stacking // in C target runtime. // pANTLR3_INPUT_STREAM in; #ifdef CVC4_ANTLR3_OLD_INPUT_STREAM in = antlr3AsciiFileStreamNew((pANTLR3_UINT8) filename.c_str()); #else /* CVC4_ANTLR3_OLD_INPUT_STREAM */ in = antlr3FileStreamNew((pANTLR3_UINT8) filename.c_str(), ANTLR3_ENC_8BIT); #endif /* CVC4_ANTLR3_OLD_INPUT_STREAM */ if( in == NULL ) { Debug("parser") << "Can't open " << filename << std::endl; return false; } // Same thing as the predefined PUSHSTREAM(in); lexer->pushCharStream(lexer, in); // restart it //lexer->rec->state->tokenStartCharIndex = -10; //lexer->emit(lexer); // Note that the input stream is not closed when it EOFs, I don't bother // to do it here, but it is up to you to track streams created like this // and destroy them when the whole parse session is complete. Remember that you // don't want to do this until all tokens have been manipulated all the way through // your tree parsers etc as the token does not store the text it just refers // back to the input stream and trying to get the text for it will abort if you // close the input stream too early. //TODO what said before return true; }
static pANTLR3_STRING getText (pANTLR3_LEXER lexer) { if (lexer->rec->state->text) { return lexer->rec->state->text; } return lexer->input->substr( lexer->input, lexer->rec->state->tokenStartCharIndex, lexer->getCharIndex(lexer) - lexer->input->charByteSize ); }
static pANTLR3_COMMON_TOKEN emit (pANTLR3_LEXER lexer) { pANTLR3_COMMON_TOKEN token; /* We could check pointers to token factories and so on, but * we are in code that we want to run as fast as possible * so we are not checking any errors. So make sure you have installed an input stream before * trying to emit a new token. */ token = lexer->rec->state->tokFactory->newToken(lexer->rec->state->tokFactory); if (token == NULL) { return NULL; } /* Install the supplied information, and some other bits we already know * get added automatically, such as the input stream it is associated with * (though it can all be overridden of course) */ token->type = lexer->rec->state->type; token->channel = lexer->rec->state->channel; token->start = lexer->rec->state->tokenStartCharIndex; token->stop = lexer->getCharIndex(lexer) - 1; token->line = lexer->rec->state->tokenStartLine; token->charPosition = lexer->rec->state->tokenStartCharPositionInLine; if (lexer->rec->state->text != NULL) { token->textState = ANTLR3_TEXT_STRING; token->tokText.text = lexer->rec->state->text; } else { token->textState = ANTLR3_TEXT_NONE; } token->lineStart = lexer->input->currentLine; token->user1 = lexer->rec->state->user1; token->user2 = lexer->rec->state->user2; token->user3 = lexer->rec->state->user3; token->custom = lexer->rec->state->custom; lexer->rec->state->token = token; return token; }
/** Implementation of match range for the lexer, overrides any * base implementation in the base recognizer. * * \remark * Note that the generated code lays down arrays of ints for constant * strings so that they are int UTF32 form! */ static ANTLR3_BOOLEAN matchRange(pANTLR3_LEXER lexer, ANTLR3_UCHAR low, ANTLR3_UCHAR high) { ANTLR3_UCHAR c; /* What is in the stream at the moment? */ c = lexer->input->istream->_LA(lexer->input->istream, 1); if ( c >= low && c <= high) { /* Matched correctly, consume it */ lexer->input->istream->consume(lexer->input->istream); /* Reset any failed indicator */ lexer->rec->state->failed = ANTLR3_FALSE; return ANTLR3_TRUE; } /* Failed to match, execption and recovery time. */ if (lexer->rec->state->backtracking > 0) { lexer->rec->state->failed = ANTLR3_TRUE; return ANTLR3_FALSE; } lexer->rec->exConstruct(lexer->rec); /* TODO: Implement exception creation more fully */ lexer->recover(lexer); return ANTLR3_FALSE; }