// Return installed API files.
QStringList QsciAPIs::installedAPIFiles() const
{
QString qtdir = QLibraryInfo::location(QLibraryInfo::DataPath);
QDir apidir = QDir(QString("%1/qsci/api/%2").arg(qtdir).arg(lexer()->lexer()));
QStringList filenames;
QStringList filters;
filters << "*.api";
QFileInfoList flist = apidir.entryInfoList(filters, QDir::Files, QDir::IgnoreCase);
foreach (QFileInfo fi, flist)
filenames << fi.absoluteFilePath();
return filenames;
}
void DocumentEditor::setLanguage(const QString &language_) {
QsciLexer* l = lexer();
//detach lexer from document before delete it
setLexer(0);
if(l != 0) {
delete l;
l = 0;
}
//set the new lexer
l = LexerManager::getInstance().lexerFactory(language_, this);
setLexer(l);
//reload settings for lexer
Settings settings;
settings.applyToDocument(this);
}
int main(int argc, char *argv[])
{
if((in = fopen(argv[1], "r")) == NULL)
{
printf("lexer crashed: bad read file \n");
return -1;
}
if((out = fopen(argv[2], "w")) == NULL)
{
printf("lexer crashed: bad write file \n");
return -1;
}
lexer();
fclose(in);
fclose(out);
return 0;
}
const Result *parse(Source source, Result *(Parser::RequirementParser::*rule)(), std::string &errors)
{
Input input(source);
Parser::RequirementLexer lexer(input);
Parser::RequirementParser parser(lexer);
try {
const Result *result = (parser.*rule)();
errors = parser.errors;
if (errors.empty())
return result;
else
::free((void *)result);
} catch (const antlr::TokenStreamException &ex) {
errors = ex.toString() + "\n";
}
return NULL; // signal failure
}
void repl(std::ostream& out)
{
Lexer lexer(0, &out);
ResultType result;
Parser parser(lexer, result);
while (true)
{
auto request = input();
if (request == "exit") break;
std::stringstream stream_in(request);
lexer.restart(stream_in);
if (0 == parser.parse())
{
std::cout << result << std::endl;
}
}
}
/// handle specialized clipboard text, with leading "(fp_lib_table", OR
/// spreadsheet formatted text.
void paste_text( const wxString& cb_text ) override
{
FP_LIB_TABLE_GRID* tbl = (FP_LIB_TABLE_GRID*) m_grid->GetTable();
size_t ndx = cb_text.find( "(fp_lib_table" );
if( ndx != std::string::npos )
{
// paste the FP_LIB_TABLE_ROWs of s-expression (fp_lib_table), starting
// at column 0 regardless of current cursor column.
STRING_LINE_READER slr( TO_UTF8( cb_text ), "Clipboard" );
LIB_TABLE_LEXER lexer( &slr );
FP_LIB_TABLE tmp_tbl;
bool parsed = true;
try
{
tmp_tbl.Parse( &lexer );
}
catch( PARSE_ERROR& pe )
{
DisplayError( m_dialog, pe.What() );
parsed = false;
}
if( parsed )
{
// make sure the table is big enough...
if( tmp_tbl.GetCount() > (unsigned) tbl->GetNumberRows() )
tbl->AppendRows( tmp_tbl.GetCount() - tbl->GetNumberRows() );
for( unsigned i = 0; i < tmp_tbl.GetCount(); ++i )
tbl->rows.replace( i, tmp_tbl.At( i ).clone() );
}
m_grid->AutoSizeColumns( false );
}
else
{
// paste spreadsheet formatted text.
GRID_TRICKS::paste_text( cb_text );
m_grid->AutoSizeColumns( false );
}
}
MaterialPtr Compiler::Compile(Sys_GraphicsPtr pGraphics, const std::string& src, const boost::filesystem::path& filename)
{
m_file = filename;
logger() << "-- compiling material(" << m_file << "): \n";
Lexer lexer(src);
MaterialParser parser(nullptr, logger());
if(false == parser.Parse(&lexer, m_file == "" ? "./" : m_file.parent_path()))
{
logger() << "== failed.\n";
return MaterialPtr();
}
logger() << "== ok.\n";
return parser.CreateObject(pGraphics);
}
ASTNode *
do_native_parse(unsigned char *buffer, size_t length)
{
ASTNode *result;
antlr::CharInputBuffer input(buffer, length, false);
CPPLexer lexer(input);
lexer.setFilename("unknown");
CPPParser parser(lexer);
parser.setFilename("unknown");
parser.init();
result = parser.translation_unit();
parser.uninit();
return result;
}
parser::parser(std::istream &file) :
m_queue()
{
lexer lexer(file);
struct lexer::token *token;
for (token = lexer.next_token(); token != NULL; token = lexer.next_token())
{
if (token->type != lexer::Separator)
{
if (token->type != lexer::Identifier)
throw std::exception();
else
m_queue.push(automate(lexer, token));
}
delete token;
}
}
/**
* @brief compile handler which starts lexing and parsing
*
* @param raw_code pl0 source code
* @retval int TRUE or FALSE
*/
int compile(FILE *raw_code) {
SOURCECODE pl0_code = sc_init();
int status;
puts("Start lexical scanning...");
lexer(pl0_code, raw_code);
puts("Finished lexical scanning!\n");
puts("Start parsing...\n");
status = init_parsing(pl0_code);
sc_destroy(pl0_code);
return status;
}
TEST(sexpr, lexer)
{
io::LineSpan span;
sexpr::Lexer lexer(io::from_string, "<lexer-test1>"_s, " foo( ) 123\"\" \n"_s);
EXPECT_EQ(lexer.peek(), sexpr::TOK_TOKEN);
EXPECT_EQ(lexer.val_string(), "foo"_s);
EXPECT_EQ(lexer.span().error_str("test"_s),
"<lexer-test1>:1:2: error: test\n"
" foo( ) 123\"\" \n"
" ^~~\n"_s
);
lexer.adv();
EXPECT_EQ(lexer.peek(), sexpr::TOK_OPEN);
EXPECT_EQ(lexer.span().error_str("test"_s),
"<lexer-test1>:1:5: error: test\n"
" foo( ) 123\"\" \n"
" ^\n"_s
);
lexer.adv();
EXPECT_EQ(lexer.peek(), sexpr::TOK_CLOSE);
EXPECT_EQ(lexer.span().error_str("test"_s),
"<lexer-test1>:1:7: error: test\n"
" foo( ) 123\"\" \n"
" ^\n"_s
);
lexer.adv();
EXPECT_EQ(lexer.peek(), sexpr::TOK_TOKEN);
EXPECT_EQ(lexer.val_string(), "123"_s);
EXPECT_EQ(lexer.span().error_str("test"_s),
"<lexer-test1>:1:9: error: test\n"
" foo( ) 123\"\" \n"
" ^~~\n"_s
);
lexer.adv();
EXPECT_EQ(lexer.peek(), sexpr::TOK_STRING);
EXPECT_EQ(lexer.val_string(), ""_s);
EXPECT_EQ(lexer.span().error_str("test"_s),
"<lexer-test1>:1:12: error: test\n"
" foo( ) 123\"\" \n"
" ^~\n"_s
);
lexer.adv();
EXPECT_EQ(lexer.peek(), sexpr::TOK_EOF);
}
int main( int argc, char *argv[] )
{
ANTLR_USING_NAMESPACE(std);
ANTLR_USING_NAMESPACE(antlr);
ANTLR_USING_NAMESPACE(VmiCalls);
if( argc < 2 )
exit( 0 );
try {
ifstream input( argv[1] );
VmiCallsLexer lexer(input);
TokenBuffer buffer(lexer);
VmiCallsParser parser(buffer);
ASTFactory ast_factory;
parser.initializeASTFactory( ast_factory );
parser.setASTFactory( &ast_factory );
parser.vmiCallsFile();
RefAST a = parser.getAST();
VmiCallsHtmlEmitter tree_walker;
tree_walker.initializeASTFactory( ast_factory );
tree_walker.setASTFactory( &ast_factory );
tree_walker.vmiCalls( a );
/*
cout << "Tree:" << endl;
cout << a->toStringTree() << endl;
*/
}
catch( ANTLRException& e )
{
cerr << "exception: " << e.getMessage() << endl;
return -1;
}
catch( exception& e )
{
cerr << "exception: " << e.what() << endl;
return -1;
}
return 0;
}
void BaseCommandsQsciApi::updateAutoCompletionList(const QStringList& context, QStringList& list) {
BaseCommandsQsciLexer* lex = static_cast<BaseCommandsQsciLexer*>(lexer());
auto commands = lex->commands();
for (auto it = context.begin(); it != context.end(); ++it) {
QString val = *it;
for (size_t i = 0; i < commands.size(); ++i) {
core::CommandInfo cmd = commands[i];
if (canSkipCommand(cmd)) {
continue;
}
QString jval;
common::ConvertFromString(cmd.name, &jval);
if (jval.startsWith(val, Qt::CaseInsensitive)) {
list.append(jval + "?1");
}
}
}
}
//#################### LOADING METHODS ####################
InputBinding_CPtr BindingFile::load(const std::string& filename)
{
std::map<InputAction,Inputter_CPtr> inputters;
// Construct the string -> key map.
std::map<std::string,SDLKey> stringToKey;
for(int i=0; i<SDLK_LAST; ++i)
{
SDLKey key = SDLKey(i);
std::string name = SDL_GetKeyName(key);
stringToKey.insert(std::make_pair(name,key));
}
XMLLexer_Ptr lexer(new XMLLexer(filename));
XMLParser parser(lexer);
XMLElement_CPtr root = parser.parse();
XMLElement_CPtr bindingElt = root->find_unique_child("binding");
std::vector<XMLElement_CPtr> actionElts = bindingElt->find_children("action");
for(size_t i=0, size=actionElts.size(); i<size; ++i)
{
const XMLElement_CPtr& actionElt = actionElts[i];
const std::string& actionName = actionElt->attribute("name");
const std::string& actionInput = actionElt->attribute("input");
InputAction action = lexical_cast<InputAction>(actionName);
Inputter_CPtr inputter;
std::map<std::string,SDLKey>::const_iterator jt = stringToKey.find(actionInput);
if(jt != stringToKey.end())
{
inputter.reset(new KeyInputter(jt->second));
}
else if(actionInput == "left mouse") inputter.reset(new MouseButtonInputter(MOUSE_BUTTON_LEFT));
else if(actionInput == "middle mouse") inputter.reset(new MouseButtonInputter(MOUSE_BUTTON_MIDDLE));
else if(actionInput == "right mouse") inputter.reset(new MouseButtonInputter(MOUSE_BUTTON_RIGHT));
else throw Exception("Unknown input action: " + actionInput);
inputters.insert(std::make_pair(action, inputter));
}
return InputBinding_CPtr(new InputBinding(inputters));
}
// Add auto-completion words based on the last partial word entered.
void QsciAPIs::lastPartialWord(const QString &word, QStringList &with_context, bool &unambig)
{
if (lexer()->caseSensitive())
{
QMap<QString, WordIndexList>::const_iterator it;
QStringList::const_iterator wit = QsciAPIsPrepared::lowerBound(prep->words, word);
if (wit == prep->words.end())
it = prep->wdict.end();
else
it = prep->wdict.find(*wit);
while (it != prep->wdict.end())
{
if (!it.key().startsWith(word))
break;
addAPIEntries(it.data(), false, with_context, unambig);
++it;
}
}
else
{
QMap<QString, QString>::const_iterator it;
QStringList::const_iterator wit = QsciAPIsPrepared::lowerBound(prep->words, word);
if (wit == prep->words.end())
it = prep->cdict.end();
else
it = prep->cdict.find(*wit);
while (it != prep->cdict.end())
{
if (!it.key().startsWith(word))
break;
addAPIEntries(prep->wdict[it.data()], false, with_context, unambig);
++it;
}
}
}
// Here's where we do the real work...
static void parseFile(const string& f)
{
try
{
ifstream s(f.c_str());
// Create a scanner that reads from the input stream
JavaLexer lexer(s);
lexer.setFilename(f);
/*
while (true) {
RefToken t = lexer.nextToken();
if (t->getType() == Token::EOF_TYPE)
break;
cout << t->getText() << ":" << t->getType() << endl;
}
*/
// Create a parser that reads from the scanner
JavaRecognizer parser(lexer);
parser.setFilename(f);
// make an ast factory
ASTFactory ast_factory;
// initialize and put it in the parser...
parser.initializeASTFactory(ast_factory);
parser.setASTFactory(&ast_factory);
// start parsing at the compilationUnit rule
parser.compilationUnit();
// do something with the tree
doTreeAction( ast_factory, parser.getAST() );
}
catch (ANTLRException& e) {
cerr << "parser exception: " << e.toString() << endl;
}
catch (exception& e) {
cerr << "exception: " << e.what() << endl;
}
}
bool SharedMemory::createHandle(Handle& handle, Protection protection)
{
ASSERT_ARG(handle, handle.isNull());
RChunk chunk;
if (chunk.SetReturnedHandle(m_handle))
return false;
// Convert the name (string form) to a uint32_t.
TName globalChunkName = chunk.Name();
TLex lexer(globalChunkName);
TUint32 nameAsInt = 0;
if (lexer.Val(nameAsInt, EDecimal))
return false;
handle.m_chunkID = nameAsInt;
handle.m_size = m_size;
return true;
}
void sv_receive_command(t_sv_prop *sv, int cl)
{
int rd;
int ret_cmd;
if ((rd = E(-1, recv(CL_SOCK(cl), sv->fds[cl].rd, BUF_SIZE, 0), ERR_RECV, NO_EXIT)) > 0)
{
printf("[sv_receive_command] [client : %d] [commande : %s] [lu : %d]\n", cl, sv->fds[cl].rd, rd);
sv->cmd->cmda = lexer(sv->fds[cl].rd);
ret_cmd = execute(sv, cl);
if (!ret_cmd)
pterr(ERR_CMD_NOT_FOUND);
ft_strarray_del(&(sv->cmd->cmda));
//ft_bzero(sv->fds[cl].rd, rd);
ft_bzero(sv->fds[cl].rd, BUF_SIZE + 1);
}
else
clean_fd(&(sv->fds[cl]));
}
void lexer(t_vars *v)
{
static int loop;
if (v->alias && which_it_is(v) != BUILT_ALI)
{
if (loop < 10 && is_there_an_alias(v))
{
loop += 1;
build_argv(v);
lexer(v);
}
else if (loop == 10)
loop = 0;
}
else if (there_is_home(v))
modify_prompt_for_home(v);
loop = 0;
}
/**
* Parses a comma separated string and constructs a list out of the values
*/
void CTestCalInterimApiSuiteStepBase::TokenizeStringL(const TDesC& aString, RArray<TPtrC>& aList, TChar aSeparator)
{
TLex lexer(aString);
while(!lexer.Eos())
{
lexer.SkipSpaceAndMark();
while(!lexer.Eos() && lexer.Get() != aSeparator)
{
}
if(!lexer.Eos())
{
lexer.UnGet(); // Do not include trailing ','
}
aList.AppendL(lexer.MarkedToken());
lexer.Inc();
}
}
/*
Parses a comma separated string and constructs a list out of the values
@param aString a reference to a string to be tokenized
@param aList is an out parameter to store a list of tokens in an arry
@param aSeparator is a character used to delimit the tokens
*/
void CTestRControlChannel::TokenizeStringL(const TDesC& aString, RArray<TPtrC>& aList, TChar aSeparator)
{
TLex lexer(aString);
while(!lexer.Eos())
{
lexer.SkipSpaceAndMark();
while(!lexer.Eos() && lexer.Get() != aSeparator)
{
}
if(!lexer.Eos())
{
lexer.UnGet();
}
aList.AppendL(lexer.MarkedToken());
lexer.Inc();
}
}
TEST(sexpr, parselist)
{
sexpr::SExpr s;
sexpr::Lexer lexer(io::from_string, "<parser-test1>"_s, "(foo)(bar)\n"_s);
EXPECT_TRUE(sexpr::parse(lexer, s));
EXPECT_EQ(s._type, sexpr::LIST);
EXPECT_EQ(s._list.size(), 1);
EXPECT_EQ(s._list[0]._type, sexpr::TOKEN);
EXPECT_EQ(s._list[0]._str, "foo"_s);
EXPECT_TRUE(sexpr::parse(lexer, s));
EXPECT_EQ(s._type, sexpr::LIST);
EXPECT_EQ(s._list.size(), 1);
EXPECT_EQ(s._list[0]._type, sexpr::TOKEN);
EXPECT_EQ(s._list[0]._str, "bar"_s);
EXPECT_FALSE(sexpr::parse(lexer, s));
EXPECT_EQ(lexer.peek(), sexpr::TOK_EOF);
}
int start_shell(t_42sh *shell)
{
t_token *token;
int fd;
fd = creat(".hist42sh", 0644);
if ((my_clear() == -1))
return (-42);
while (1)
{
token = NULL;
signal(SIGINT, get_sigint);
prompt(shell);
shell->cmd = read_line(fd);
if (lexer(shell->cmd, &token, shell) == -42)
return (-42);
free_my_tok(token);
}
return (0);
}
/** Parses a comma separated string and constructs a list out of the values
@param aString The string to be tokenized
@param aList Output list containing the tokens
@param aSeparator Separator to recognize the tokens
*/
void CContactsPBAPExportUtilityClass::TokenizeStringL(const TDesC& aString, RArray<TPtrC>& aList, TChar aSeparator)
{
TLex lexer(aString);
while(!lexer.Eos())
{
lexer.SkipSpaceAndMark();
while(!lexer.Eos() && lexer.Get() != aSeparator)
{
}
if(!lexer.Eos())
{
lexer.UnGet(); // Do not include trailing ','
}
aList.AppendL(lexer.MarkedToken());
lexer.Inc();
}
}
/*
================
CmdSystemEx::ExecuteConfig
================
*/
void CmdSystemEx::ExecuteConfig( const char *filename ) {
// was just full lines dunno why BOM error should be printed for plain text configs
Lexer lexer(LEXER_FULL_LINES|LEXER_NO_BOM_WARNING);
if ( !lexer.LoadFile(filename) )
return;
try {
const Token *token;
const char *p;
while ( (token = lexer.ReadToken()) != OG_NULL ) {
p = token->GetString();
if ( p && *p )
ExecuteCmd( p, inEngineStartup );
}
}
catch( LexerError &err ) {
String errStr;
err.ToString( errStr );
User::Error( ERR_LEXER_FAILURE, errStr.c_str(), filename );
}
}
void SyntaxHighlighter::highlightBlock(const QString& text)
{
TextBlockData *blockData = static_cast<TextBlockData*>(currentBlockUserData());
if(!blockData) {
blockData = new TextBlockData;
blockData->tokens.reserve(8);
setCurrentBlockUserData(blockData);
}
else {
blockData->tokens.clear();
}
int previousState = previousBlockState();
if (previousState == -1)
previousState = ScLexer::InCode;
ScLexer lexer( text, 0, previousState );
while (lexer.offset() < text.size()) {
switch (lexer.state()) {
case ScLexer::InCode:
highlightBlockInCode(lexer);
break;
case ScLexer::InString:
highlightBlockInString(lexer);
break;
case ScLexer::InSymbol:
highlightBlockInSymbol(lexer);
break;
default:
if(lexer.state() >= ScLexer::InComment)
highlightBlockInComment(lexer);
}
}
setCurrentBlockState( lexer.state() );
}
bool CGptBind::run( )
{
ifstream in((_options->sourcefile+".sld").c_str());
GptBindLexer lexer(in);
GptBindParser parser(lexer);
// string asmProgram = parser.program(_options->sourcefile);
parser.sld_grammar(_options->sourcefile);
string cppOutput = parser.getCpp();
string hppOutput = parser.getHpp();
string makefileOutput = parser.getMakefile();
ofstream cppFile((_options->destfile + ".cpp").c_str(), ios_base::out);
if (!cppFile) {
cout << "ERRO: não foi possível abrir o arquivo: \"" << _options->destfile << "\"" << endl;
return false;
}
cppFile << cppOutput;
ofstream hppFile((_options->destfile + ".hpp").c_str(), ios_base::out);
if (!hppFile) {
cout << "ERRO: não foi possível abrir o arquivo: \"" << _options->destfile << "\"" << endl;
return false;
}
hppFile << hppOutput;
ofstream makefileFile(("Makefile." + _options->destfile).c_str(), ios_base::out);
if (!makefileFile) {
cout << "ERRO: não foi possível abrir o arquivo: \"" << _options->destfile << "\"" << endl;
return false;
}
makefileFile << makefileOutput;
return true;
}
void WriteTokens(std::istream& is, std::ostream& os)
{
Lexer lexer(is);
int currentLine = 1;
do
{
lexer.GetNext();
for (int i = currentLine; i < lexer.CurrentLine; i++)
{
os << '\n';
}
currentLine = lexer.CurrentLine;
os << lexer.CurrentToken << ' ';
} while (lexer.CurrentToken.Type != TokenType::EndOfFile);
os << '\n';
}
std::unique_ptr<ParsedFile> parseFileSyntax(const FileForParsing& ffp) {
auto file = std::unique_ptr<ParsedFile>(new ParsedFile());
Lexer lexer(ffp);
while(lexer.hasCurrentToken()) {
if(lexer.currType()==STATEMENT_END) {
lexer.advance(); //Eat ';'
continue; //Extra semicolons are ok
}
bool pub = lexer.currType()==PUB;
if(pub)
lexer.advance();
unique_ptr<Definition> definition = parseDefinition(lexer, pub);
if(definition) { //A nice definition was returned, and has eaten it's own semicolon
definition->printSignature();
file->m_definitions.push_back(std::move(definition));
}
else if(lexer.hasCurrentToken()) //Error occurred, but already printed
skipUntilNewDefinition(lexer);
}
terminateIfErrors();
return file;
}
NodeType * deserialize_lex_forwarder( const std::string & lexerClassName,
std::istream & src
)
{
// CERR << "deserialize_lex_forwarder("<<lexerClassName<<")\n";
std::auto_ptr<FlexLexer> lexer( ::s11n::cl::classload<FlexLexer>( lexerClassName ) );
if( ! lexer.get() )
{
throw ::s11n::s11n_exception(
"%s:%d: s11n::io::deserialize_lex_forwarder(): Lexer '%s' was not found by classload<FlexLexer>(). It is probably not registered with the classloader.",
__FILE__,
__LINE__,
lexerClassName.c_str() );
}
typedef s11n::io::data_node_tree_builder<NodeType> BuilderType;
typedef tree_builder_context<BuilderContext> BC;
std::auto_ptr<BuilderType> treebuilder( new BuilderType );
treebuilder->auto_delete( true ); // if we throw, let it clean up
try
{
BC::bind( lexer.get(), treebuilder.get() );
// ^^^ sets up the comm channel between the builder and lexer
Private::lex_api_hider_yylex(lexer.get(),src); // executes the lexer
}
catch ( const std::exception & ex )
{
BC::unbind( lexer.get() ); // free up lexer-to-builder binding
throw ex;
}
catch (...)
{
BC::unbind( lexer.get() ); // free up lexer-to-builder binding
throw;
}
BC::unbind( lexer.get() ); // free up lexer-to-builder binding
treebuilder->auto_delete( false ); // we're taking ownership of the children
return treebuilder->root_node();
}
