static int read_pnm_header(deark *c, lctx *d, struct page_ctx *pg, de_int64 pos1) { char tokenbuf[100]; int retval = 0; de_dbg(c, "header at %d\n", (int)pos1); de_dbg_indent(c, 1); de_dbg(c, "format: %s\n", pg->fmt_name); pg->hdr_parse_pos = pos1+2; // Skip "P?" if(!read_next_token(c, d, pg, tokenbuf, sizeof(tokenbuf))) goto done; pg->width = de_atoi64(tokenbuf); if(!read_next_token(c, d, pg, tokenbuf, sizeof(tokenbuf))) goto done; pg->height = de_atoi64(tokenbuf); de_dbg(c, "dimensions: %dx%d\n", (int)pg->width, (int)pg->height); if(fmt_is_pbm(pg->fmt)) { pg->maxval = 1; } else { if(!read_next_token(c, d, pg, tokenbuf, sizeof(tokenbuf))) goto done; pg->maxval = de_atoi64(tokenbuf); de_dbg(c, "maxval: %d\n", (int)pg->maxval); if(pg->maxval<1 || pg->maxval>65535) { de_err(c, "Invalid maxval: %d\n", (int)pg->maxval); goto done; } } retval = 1; done: de_dbg_indent(c, -1); return retval; }
/* Manage the data of an XML tag (string or sub tags) */ void Parser_XML::data(std::string& chaine) { const char current_token = read_current_token(chaine); const char next_token = read_next_token(chaine); if(ASCII_WORD(current_token)) { std::string data = string(chaine); this->setData(data); } else if(ASCII_NUM(current_token)) { std::string data = number_grammar(chaine); this->setData(data); } else if(current_token == '<' && ASCII_WORD(next_token)) { this->setData(""); Parser_XML* current_child = new Parser_XML; if(current_child != NULL) { current_child->tag(chaine); this->addChild(current_child); current_child->_parent = this; } else { std::cerr << "Error when reading this xml file." << std::endl; } data(chaine); } else if(current_token == '<' && next_token == '/') { } else { std::cerr << "Error when reading a data." << std::endl; } }
/* function to allow parsing */ static int _parse_bitstream_data(bitstream_parsed_t *dest, bitstream_parser_t *parser) { gint advance; int err; /* First parse the header of the bitstream */ /* Then synchronize the bitstream data */ parser->state = STATE_UNSYNCHED; err = synchronize_bitstream(parser); if (err) { debit_log(L_BITSTREAM,"Could not synchronize bitstream"); return err; } /* Then launch the computation */ do { advance = read_next_token(dest, parser); } while(advance > 0); if (advance < 0) { debit_log(L_BITSTREAM,"Error parsing bitstream: %i", advance); } return advance; }
lexer::lexer(text_range const& range, error_tag_sink& error_sink) : start(range.get_start()) , end(range.get_end()) , pos(range.get_start()) , error_sink(&error_sink) { current_token = read_next_token(); }
Call::Call() : Token(true) { read_next_token(); function_name = next_token(); read_next_token(); // "(" token String args_terminator = ")"; while (args_terminator != peek_next_token()) { ParseTree* tree = new ParseTree(); tree->buildTree(); args_list.push_back(tree); } read_next_token(); // args terminator }
void functionHandler(tree& symbolTable, funcS& funcTable){ tree localVar; param* parameter = new param(20); param::statment in; read_next_token(); String name(next_token()); read_next_token(); if(String(next_token()) == "params"){ read_next_token(); int i = 1; while(String (next_token()) != "smarap"){// puts all the local variables in a local symbol table if(next_token_type == NAME){ in.para = String (next_token()); in.value = 0; parameter->push_back(in); i += 1; } read_next_token(); } } // after we got all the local variables set the function body funcTable.insert(name, 0 , statmentOp(symbolTable, localVar, funcTable), 0 , parameter); }
bool mmoTokenizer::LoadString(const char* buffer) { if(buffer != m_arrCodeBuff) { strncpy(m_arrCodeBuff, buffer, MAX_CODE_LEN); m_nCodeLen = strlen(buffer); } while(m_nCurPos < m_nCodeLen) { bool ret = read_next_token(); if(!ret) return false; } return true; }
/* * Read in a rc file from PATH and process it looking for the * specified DEBUG_VALUE or TAG_FIND token. It passes back the * returned debug value in DEBUG_P. Passes back the matching TOKEN of * TOKEN_SIZE. * * Returns FILE_NOT_FOUND, FILE_FOUND, or TOKEN_FOUND. */ static int read_rc_file(const char *path, const long debug_value, const char *tag_find, long *debug_p, char *token, const int token_size) { FILE *infile; int found_b = 0; char next_token[64]; long new_debug; /* open the path */ infile = fopen(path, "r"); if (infile == NULL) { return FILE_NOT_FOUND; } /* run through the tokens, looking for a match */ while (read_next_token(infile, &new_debug, next_token, sizeof(next_token)) == 1) { /* are we looking for a tag? */ if (tag_find != NULL && strcmp(tag_find, next_token) == 0) { found_b = 1; break; } /* are we looking for a debug-value? */ if (debug_value > 0 && debug_value == new_debug) { found_b = 1; break; } } (void)fclose(infile); SET_POINTER(debug_p, new_debug); if (token != NULL) { (void)loc_snprintf(token, token_size, "config file token: %s", next_token); } if (found_b) { return TOKEN_FOUND; } else { return FILE_FOUND; } }
/* Read ptrdicturation from a stream. */ void ptrdict_from_stream(section_t *root, FILE *f) { /* Note: We might want to switch to XML, i.e., using libXML2 eventually. This will make this stuff a lot easier, too. */ parser_t p; section_t *s; char keyword[MAX_KEYWORD+1]; char token[MAX_TOKEN+1]; char value[MAX_VALUE+1]; BOOL is_token; p.column = 1; p.row = 1; p.f = f; /* First line needs to be section with the name of the root section. */ read_next_keyword(&p, keyword, MAX_KEYWORD, &is_token); if (!strcmp(keyword, "section")) { /* Okay, it's the long format. */ read_next_value(&p, value, MAX_VALUE); if (strcmp(value, root->name)) { printf("[ptrdict_read] Error: Expected '%s' as the name of the first section in line %i.\n", root->name, p.row); exit(1); } ptrdict_lf_read_section(root, &p); } else if (!strcmp(keyword, root->name)) { /* Okay, it's the short format. */ read_next_token(&p, token, MAX_TOKEN); if (strcmp(token, "{")) { printf("[ptrdict_read] '{' expected in line %i.\n", p.row); exit(1); } ptrdict_sf_read_section(root, &p); } else { printf("[ptrdict_read] Error: Keyword 'section' or '%s' expected in line %i.\n", root->name, p.row); exit(1); } }
int LDAP_CALL ldap_create_sort_keylist ( LDAPsortkey ***sortKeyList, const char *string_rep ) { int count = 0; LDAPsortkey **pointer_array = NULL; const char *current_position = NULL; int retval = 0; int i = 0; /* Figure out how many there are */ if (NULL == string_rep) { return LDAP_PARAM_ERROR; } if (NULL == sortKeyList) { return LDAP_PARAM_ERROR; } count = count_tokens(string_rep); if (0 == count) { *sortKeyList = NULL; return LDAP_PARAM_ERROR; } /* Allocate enough memory for the pointers */ pointer_array = (LDAPsortkey**)NSLDAPI_MALLOC(sizeof(LDAPsortkey*) * (count + 1) ); if (NULL == pointer_array) { return LDAP_NO_MEMORY; } /* Now walk along the string, allocating and filling in the LDAPsearchkey structure */ current_position = string_rep; for (i = 0; i < count; i++) { if (0 != (retval = read_next_token(¤t_position,&(pointer_array[i])))) { pointer_array[count] = NULL; ldap_free_sort_keylist(pointer_array); *sortKeyList = NULL; return retval; } } pointer_array[count] = NULL; *sortKeyList = pointer_array; return LDAP_SUCCESS; }
void read_next_keyword(parser_t *p, char *s, size_t n, BOOL *is_token) { int i = 0; char c; *is_token = FALSE; while (isblank(c = read_char(p))) p->column++; if (iskeywordchar(c)) { /* This is really a keyword */ while (iskeywordchar(c)) { s[i] = c; i++; if (i >= n) { printf("[read_next_keyword] Error: Keyword too long in line %i.\n", p->row); exit(1); } c = read_char(p); p->column++; } s[i] = 0; ungetc(c, p->f); } else { /* This might be a token */ *is_token = TRUE; ungetc(c, p->f); p->column--; read_next_token(p, s, n); } }
int jdip::context_parser::handle_scope(definition_scope *scope, token_t& token, unsigned inherited_flags) { definition* decl; token = read_next_token(scope); for (;;) { switch (token.type) { case TT_TYPENAME: case TT_DECFLAG: case TT_DECLTYPE: case TT_DECLARATOR: case_TT_DECLARATOR: case TT_CLASS: case TT_STRUCT: case TT_ENUM: case TT_UNION: case TT_TILDE: decl = NULL; handle_declarator_block: if (handle_declarators(scope, token, inherited_flags, decl)) { FATAL_RETURN(1); while (token.type != TT_ENDOFCODE and token.type != TT_SEMICOLON and token.type != TT_LEFTBRACE and token.type != TT_RIGHTBRACE) token = read_next_token(scope); } handled_declarator_block: if (token.type != TT_SEMICOLON) { if (token.type == TT_LEFTBRACE || token.type == TT_ASM) { if (!(decl and decl->flags & DEF_FUNCTION)) { token.report_error(herr, "Unexpected opening brace here; declaration is not a function"); FATAL_RETURN(1); handle_function_implementation(lex,token,scope,herr); } else ((definition_function*)decl)->implementation = handle_function_implementation(lex,token,scope,herr); if (token.type != TT_RIGHTBRACE && token.type != TT_SEMICOLON) { token.report_error(herr, "Expected closing symbol to function"); continue; } } else { token.report_errorf(herr, "Expected semicolon before %s following declaration"); #if FATAL_ERRORS return 1; #else semicolon_bail: while (token.type != TT_SEMICOLON && token.type != TT_LEFTBRACE && token.type != TT_RIGHTBRACE && token.type != TT_ENDOFCODE) token = read_next_token(scope); if (token.type == TT_LEFTBRACE) { size_t depth = 1; while (token.type != TT_ENDOFCODE) { token = read_next_token(scope); if (token.type == TT_LEFTBRACE) ++depth; else if (token.type == TT_RIGHTBRACE) if (!--depth) break; } } #endif } } break; case TT_EXTERN: token = read_next_token(scope); if (token.type == TT_STRINGLITERAL) { token = read_next_token(scope); if (token.type == TT_LEFTBRACE) { FATAL_RETURN_IF(handle_scope(scope, token, inherited_flags), 1); if (token.type != TT_RIGHTBRACE) { token.report_error(herr, "Expected closing brace to extern block"); FATAL_RETURN(1); } break; } } else if (token.type == TT_TEMPLATE) { token = read_next_token(scope); if (token.type != TT_CLASS and token.type != TT_STRUCT and token.type != TT_DECLARATOR and token.type != TT_DEFINITION) { token.report_errorf(herr, "Expected template specialization following `extern template' directive; %s unhandled"); FATAL_RETURN(1); } } goto handle_declarator_block; case TT_COMMA: token.report_error(herr, "Unexpected comma at this point."); return 1; case TT_SEMICOLON: /* Printing a warning here is advisable but unnecessary. */ break; case TT_NAMESPACE: if (handle_namespace(scope,token)) return 1; break; case TT_LEFTPARENTH: { token.report_error(herr, "Stray opening parenthesis."); #if FATAL_ERRORS return 1; #else int bc = 1; while (bc) { token = read_next_token(scope); bc += token.type == TT_LEFTPARENTH; bc -= token.type == TT_RIGHTPARENTH; } #endif } break; case TT_RIGHTPARENTH: token.report_error(herr, "Stray closing parenthesis."); return 1; case TT_LEFTBRACKET: token.report_error(herr, "Stray opening bracket."); return 1; case TT_RIGHTBRACKET: token.report_error(herr, "Stray closing bracket."); return 1; case TT_RIGHTBRACE: return 0; case TT_LEFTBRACE: { token.report_error(herr, "Expected scope declaration before opening brace."); #if FATAL_ERRORS return 1; #else int bc = 1; while (bc) { token = read_next_token(scope); if (token.type == TT_ENDOFCODE) { token.report_error(herr, "Expected closing brace before end of code."); return 1; } bc += token.type == TT_LEFTBRACE; bc -= token.type == TT_RIGHTBRACE; } #endif } break; case TT_TYPEDEF: token = read_next_token(scope); if (handle_declarators(scope,token,inherited_flags | DEF_TYPENAME)) FATAL_RETURN(1); break; case TT_PUBLIC: if (scope->flags & DEF_CLASS) { inherited_flags &= ~(DEF_PRIVATE | DEF_PROTECTED); } else token.report_error(herr, "Unexpected `public' token outside class scope."); if ((token = read_next_token(scope)).type != TT_COLON) token.report_error(herr, "Colon expected following `public' token"); break; case TT_PRIVATE: if (scope->flags & DEF_CLASS) { inherited_flags &= ~(DEF_PRIVATE | DEF_PROTECTED); inherited_flags |= DEF_PRIVATE; } else token.report_error(herr, "Unexpected `private' token outside class scope."); if ((token = read_next_token(scope)).type != TT_COLON) token.report_error(herr, "Colon expected following `private' token"); break; case TT_PROTECTED: if (scope->flags & DEF_CLASS) { inherited_flags &= ~(DEF_PRIVATE | DEF_PROTECTED); inherited_flags |= DEF_PROTECTED; } else token.report_error(herr, "Unexpected `protected' token outside class scope."); if ((token = read_next_token(scope)).type != TT_COLON) token.report_error(herr, "Colon expected following `protected' token"); break; case TT_USING: token = read_next_token(scope); if (token.type == TT_NAMESPACE) { token = lex->get_token(herr); if (token.type == TT_IDENTIFIER) { definition* d = scope->look_up(token.content.toString()); if (!d) { token.report_errorf(herr, "Expected id to use before %s"); FATAL_RETURN(1); } else { if (d->flags & DEF_NAMESPACE) scope->use_namespace((definition_scope*)d); else token.report_error(herr, "Expected namespace name following `namespace' token"); } token = read_next_token(scope); } else token.report_error(herr, "Expected namespace name following `namespace' token"); } else { definition *usedef = read_qualified_definition(lex, scope, token, this, herr); if (usedef) scope->use_general(usedef->name, usedef); else { token.report_errorf(herr, "Using directive does not specify an object"); FATAL_RETURN(1); } if (token.type != TT_SEMICOLON) { token.report_errorf(herr, "Expected semicolon before %s to terminate using directive"); FATAL_RETURN(1); } } break; case TT_SCOPE: token = read_next_token(global); continue; case TT_DEFINITION: { if (token.def->flags & DEF_NAMESPACE) { definition_scope* dscope = (definition_scope*)token.def; token = read_next_token(scope); if (token.type == TT_SCOPE) { token = read_next_token(dscope); continue; } token.report_errorf(herr, "Expected `::' here to access namespace members"); FATAL_RETURN(1); break; } if (token.def->flags & DEF_TEMPLATE) goto case_TT_DECLARATOR; } case TT_IDENTIFIER: { string tname(token.content.toString()); if (tname == scope->name and (scope->flags & DEF_CLASS)) { token = read_next_token(scope); if (token.type != TT_LEFTPARENTH) { token.report_errorf(herr, "Expected constructor parmeters before %s"); break; } full_type ft; ft.def = scope; token = read_next_token(scope); read_function_params(ft.refs, lex, token, scope, this, herr); if (handle_declarators(scope,token,ft,inherited_flags | DEF_TYPENAME,decl)) FATAL_RETURN(1); goto handled_declarator_block; } token.report_error(herr, "Unexpected identifier in this scope; `" + tname + "' does not name a type"); } break; case TT_TEMPLATE: if (handle_template(scope, token, inherited_flags)) { FATAL_RETURN(1); goto semicolon_bail; } break; case TT_OPERATORKW: token = read_next_token(scope); if (token.type != TT_DECLARATOR and token.type != TT_DECFLAG and token.type != TT_DECLTYPE) { token.report_errorf(herr, "Expected cast type to overload before %s"); FATAL_RETURN(1); } else { lex_buffer lb(lex); while (token.type != TT_LEFTPARENTH and token.type != TT_LEFTBRACE and token.type != TT_SEMICOLON and token.type != TT_ENDOFCODE) lb.push(token), token = read_next_token(scope); if (token.type != TT_LEFTPARENTH) { token.report_error(herr, "Expected function parmeters before %s"); FATAL_RETURN(1); break; } token.type = TT_ENDOFCODE; lb.push(token); token.type = TT_LEFTPARENTH; lb.reset(); token_t kick = lb.get_token(herr); full_type ft = read_fulltype(&lb, kick, scope, this, herr); string opname; { ref_stack my_func_refs; read_referencers_post(my_func_refs, lex, token, scope, this, herr); if (my_func_refs.empty() or my_func_refs.top().type != ref_stack::RT_FUNCTION) { token.report_error(herr, "Expected function parameters for operator overload"); return 1; } opname = "operator " + ft.toString(); ft.refs.append_c(my_func_refs); } definition_function *const df = new definition_function(opname, scope, ft.def, ft.refs, ft.flags, inherited_flags); decl = df; decpair ins = scope->declare(decl->name, decl); if (!ins.inserted) { arg_key k(df->referencers); decl = ((definition_function*)ins.def)->overload(k, df, herr); } goto handled_declarator_block; } break; case TT_ASM: case TT_SIZEOF: case TT_ISEMPTY: case TT_OPERATOR: case TT_ELLIPSIS: case TT_LESSTHAN: case TT_GREATERTHAN: case TT_COLON: case TT_DECLITERAL: case TT_HEXLITERAL: case TT_OCTLITERAL: case TT_STRINGLITERAL: case TT_CHARLITERAL: case TT_NEW: case TT_DELETE: case TTM_CONCAT: case TTM_TOSTRING: case TT_INVALID: #include <User/token_cases.h> default: token.report_errorf(herr, "Unexpected %s in this scope"); break; case TT_ENDOFCODE: return 0; } token = read_next_token(scope); } }
int jdip::context_parser::handle_scope(definition_scope *scope, token_t& token, unsigned inherited_flags) { definition* decl; token = read_next_token(scope); for (;;) { switch (token.type) { case TT_TYPENAME: case TT_DECFLAG: case TT_DECLTYPE: case TT_DECLARATOR: case_TT_DECLARATOR: case TT_CLASS: case TT_STRUCT: case TT_ENUM: case TT_UNION: case TT_TILDE: decl = NULL; handle_declarator_block: if (handle_declarators(scope, token, inherited_flags, decl)) { FATAL_RETURN(1); while (token.type != TT_ENDOFCODE and token.type != TT_SEMICOLON and token.type != TT_LEFTBRACE and token.type != TT_RIGHTBRACE) token = read_next_token(scope); } handled_declarator_block: if (token.type != TT_SEMICOLON) { if (token.type == TT_LEFTBRACE || token.type == TT_ASM) { if (!(decl and decl->flags & DEF_OVERLOAD)) { token.report_error(herr, "Unexpected opening brace here; declaration is not a function"); FATAL_RETURN(1); handle_function_implementation(lex,token,scope,herr); } else { definition_overload *ovr = ((definition_overload*)decl); if (ovr->implementation != NULL) { token.report_error(herr, "Multiple implementations of function" FATAL_TERNARY("", "; old implementation discarded")); delete_function_implementation(ovr->implementation); } ovr->implementation = handle_function_implementation(lex,token,scope,herr); } if (token.type != TT_RIGHTBRACE && token.type != TT_SEMICOLON) { token.report_error(herr, "Expected closing symbol to function"); continue; } } else { token.report_errorf(herr, "Expected semicolon before %s following declaration"); #if FATAL_ERRORS return 1; #else semicolon_bail: while (token.type != TT_SEMICOLON && token.type != TT_LEFTBRACE && token.type != TT_RIGHTBRACE && token.type != TT_ENDOFCODE) token = read_next_token(scope); if (token.type == TT_LEFTBRACE) { size_t depth = 1; while (token.type != TT_ENDOFCODE) { token = read_next_token(scope); if (token.type == TT_LEFTBRACE) ++depth; else if (token.type == TT_RIGHTBRACE) if (!--depth) break; } } #endif } } break; case TT_EXTERN: token = read_next_token(scope); if (token.type == TT_STRINGLITERAL) { token = read_next_token(scope); if (token.type == TT_LEFTBRACE) { FATAL_RETURN_IF(handle_scope(scope, token, inherited_flags), 1); if (token.type != TT_RIGHTBRACE) { token.report_error(herr, "Expected closing brace to extern block"); FATAL_RETURN(1); } break; } } else if (token.type == TT_TEMPLATE) { token = read_next_token(scope); if (token.type != TT_CLASS and token.type != TT_STRUCT and token.type != TT_DECLARATOR and token.type != TT_DECFLAG and token.type != TT_DEFINITION) { token.report_errorf(herr, "Expected template specialization following `extern template' directive; %s unhandled"); FATAL_RETURN(1); } if (handle_template_extern(scope, token, inherited_flags)) FATAL_RETURN(1); break; } goto handle_declarator_block; case TT_COMMA: token.report_error(herr, "Unexpected comma at this point."); return 1; case TT_SEMICOLON: /* Printing a warning here is advisable but unnecessary. */ break; case TT_NAMESPACE: if (handle_namespace(scope,token)) return 1; break; case TT_LEFTPARENTH: { token.report_error(herr, "Stray opening parenthesis."); #if FATAL_ERRORS return 1; #else int bc = 1; while (bc) { token = read_next_token(scope); bc += token.type == TT_LEFTPARENTH; bc -= token.type == TT_RIGHTPARENTH; } #endif } break; case TT_RIGHTPARENTH: token.report_error(herr, "Stray closing parenthesis."); return 1; case TT_LEFTBRACKET: token.report_error(herr, "Stray opening bracket."); return 1; case TT_RIGHTBRACKET: token.report_error(herr, "Stray closing bracket."); return 1; case TT_RIGHTBRACE: return 0; case TT_LEFTBRACE: { token.report_error(herr, "Expected scope declaration before opening brace."); #if FATAL_ERRORS return 1; #else int bc = 1; while (bc) { token = read_next_token(scope); if (token.type == TT_ENDOFCODE) { token.report_error(herr, "Expected closing brace before end of code."); return 1; } bc += token.type == TT_LEFTBRACE; bc -= token.type == TT_RIGHTBRACE; } #endif } break; case TT_TYPEDEF: token = read_next_token(scope); if (handle_declarators(scope,token,inherited_flags | DEF_TYPENAME)) FATAL_RETURN(1); break; case TT_PUBLIC: if (scope->flags & DEF_CLASS) { inherited_flags &= ~(DEF_PRIVATE | DEF_PROTECTED); } else token.report_error(herr, "Unexpected `public' token outside class scope."); if ((token = read_next_token(scope)).type != TT_COLON) token.report_error(herr, "Colon expected following `public' token"); break; case TT_PRIVATE: if (scope->flags & DEF_CLASS) { inherited_flags &= ~(DEF_PRIVATE | DEF_PROTECTED); inherited_flags |= DEF_PRIVATE; } else token.report_error(herr, "Unexpected `private' token outside class scope."); if ((token = read_next_token(scope)).type != TT_COLON) token.report_error(herr, "Colon expected following `private' token"); break; case TT_PROTECTED: if (scope->flags & DEF_CLASS) { inherited_flags &= ~(DEF_PRIVATE | DEF_PROTECTED); inherited_flags |= DEF_PROTECTED; } else token.report_error(herr, "Unexpected `protected' token outside class scope."); if ((token = read_next_token(scope)).type != TT_COLON) token.report_error(herr, "Colon expected following `protected' token"); break; case TT_FRIEND: if (!(scope->flags & DEF_CLASS)) { token.report_error(herr, "`friend' statement may only appear in a class or structure"); FATAL_RETURN(1); while ((token = read_next_token(scope)).type != TT_SEMICOLON && token.type != TT_RIGHTBRACE && token.type != TT_ENDOFCODE); } else { if (handle_friend(scope, token, (definition_class*)scope)) FATAL_RETURN(1); if (token.type == TT_SEMICOLON) token = read_next_token(scope); else { token.report_errorf(herr, "Expected semicolon before %s"); FATAL_RETURN(1); } } continue; case TT_USING: token = read_next_token(scope); if (token.type == TT_NAMESPACE) { token = lex->get_token_in_scope(scope, herr); if (token.type == TT_DEFINITION) { definition *d = read_qualified_definition(token, scope); if (!d) { token.report_errorf(herr, "Expected namespace-name following `namespace' token"); FATAL_RETURN(1); } else { if (d->flags & DEF_NAMESPACE) scope->use_namespace((definition_scope*)d); else token.report_error(herr, "Expected namespace-name following `namespace' token"); } if (token.type == TT_SEMICOLON) token = read_next_token(scope); else { token.report_errorf(herr, "Expected semicolon before %s"); FATAL_RETURN(1); } } else { token.report_errorf(herr, "Expected namespace to use before %s"); FATAL_RETURN(1); } } else { definition *usedef = read_qualified_definition(token, scope); if (usedef) scope->use_general(usedef->name, usedef); else { token.report_errorf(herr, "Using directive does not specify an object"); FATAL_RETURN(1); } if (token.type != TT_SEMICOLON) { token.report_errorf(herr, "Expected semicolon before %s to terminate using directive"); FATAL_RETURN(1); } } continue; case TT_SCOPE: token = read_next_token(ctex->get_global()); continue; case TT_MEMBEROF: token.report_error(herr, "Unexpected (scope::*) reference"); return 1; case TT_STATIC_ASSERT: token.report_error(herr, "Unimplemented: static assert"); break; case TT_AUTO: token.report_error(herr, "Unimplemented: `auto' type inference"); break; case TT_CONSTEXPR: token.report_error(herr, "Unimplemented: const expressions outside enum"); break; case TT_DEFINITION: { if (token.def->flags & DEF_NAMESPACE) { definition_scope* dscope = (definition_scope*)token.def; token = read_next_token(scope); if (token.type == TT_SCOPE) { token = read_next_token(dscope); continue; } token.report_errorf(herr, "Expected `::' here to access namespace members"); FATAL_RETURN(1); break; } if (token.def->flags & DEF_TEMPLATE) goto case_TT_DECLARATOR; } case TT_IDENTIFIER: { string tname(token.content.toString()); if (tname == scope->name and (scope->flags & DEF_CLASS)) { token = read_next_token(scope); if (token.type != TT_LEFTPARENTH) { token.report_errorf(herr, "Expected constructor parmeters before %s"); break; } full_type ft; ft.def = scope; token = read_next_token(scope); read_function_params(ft.refs, token, scope); if (handle_declarators(scope,token,ft,inherited_flags | DEF_TYPENAME,decl)) FATAL_RETURN(1); goto handled_declarator_block; } token.report_error(herr, "Unexpected identifier in this scope (" + scope->name + "); `" + tname + "' does not name a type"); } break; case TT_TEMPLATE: if (handle_template(scope, token, inherited_flags)) { FATAL_RETURN(1); goto semicolon_bail; } break; case TT_OPERATORKW: { full_type ft = read_operatorkw_cast_type(token, scope); if (!ft.def) return 1; if (!(decl = scope->overload_function("(cast)", ft, inherited_flags, token, herr))) return 1; goto handled_declarator_block; } break; case TT_ASM: case TT_SIZEOF: case TT_ISEMPTY: case TT_ALIGNOF: case TT_ALIGNAS: case TT_OPERATOR: case TT_ELLIPSIS: case TT_LESSTHAN: case TT_GREATERTHAN: case TT_COLON: case TT_DECLITERAL: case TT_HEXLITERAL: case TT_OCTLITERAL: case TT_STRINGLITERAL: case TT_CHARLITERAL: case TT_NEW: case TT_DELETE: case TTM_CONCAT: case TTM_TOSTRING: case TT_INVALID: case TT_CONST_CAST: case TT_STATIC_CAST: case TT_DYNAMIC_CAST: case TT_REINTERPRET_CAST: case TT_NOEXCEPT: case TT_TYPEID: #include <User/token_cases.h> default: token.report_errorf(herr, "Unexpected %s in this scope"); break; case TT_ENDOFCODE: return 0; } token = read_next_token(scope); } }
const char* peek_next_token(void) { read_next_token(); token_has_been_peeked = true; return next_token(); }
/* Read the current section until 'endsection' is reached. */ void ptrdict_lf_read_section(section_t *self, parser_t *parser) { BOOL section_done = FALSE; section_t *s; property_t *p; char keyword[MAX_KEYWORD+1]; char value[MAX_VALUE+1]; char token[MAX_TOKEN+1]; BOOL is_token; self->provided = TRUE; if (self->provided_notification) *self->provided_notification = TRUE; while (!section_done && !at_end_of_file(parser)) { read_next_keyword(parser, keyword, MAX_KEYWORD, &is_token); if (is_token) { printf("[ptrdict_lf_read_section] Keyword expected in line %i.\n", parser->row); exit(1); } if (!strcmp(keyword, "endsection")) { read_next_value(parser, value, MAX_VALUE); if (strcmp(value, self->name)) { printf("[ptrdict_lf_read_section] Current open section is '%s', cannot close section '%s' in line %i.\n", self->name, value, parser->row); exit(1); } if (self->kind != SK_SECTION) { printf("[ptrdict_lf_read_section] Current open object '%s' is not a section (line %i).\n", self->name, parser->row); exit(1); } finish_line(parser); section_done = TRUE; } else if (!strcmp(keyword, "endmodule")) { read_next_value(parser, value, MAX_VALUE); if (strcmp(value, self->name)) { printf("[ptrdict_lf_read_section] Current open module is '%s', cannot close module '%s' in line %i.\n", self->name, value, parser->row); exit(1); } if (self->kind != SK_MODULE) { printf("[ptrdict_lf_read_section] Current open object '%s' is not a section (line %i).\n", self->name, parser->row); exit(1); } finish_line(parser); section_done = TRUE; } else if (!strcmp(keyword, "section")) { read_next_value(parser, value, MAX_VALUE); finish_line(parser); s = ptrdict_find_section(self, value); if (!s) { printf("[ptrdict_lf_read_section] Unknown section '%s' in line %i.\n", value, parser->row); ptrdict_enum_subsections(self, stdout); exit(1); } else if (s->kind == SK_MODULE) { printf("[ptrdict_lf_read_section] '%s' is a module identifier (line %i).\n", value, parser->row); exit(1); } ptrdict_lf_read_section(s, parser); } else if (!strcmp(keyword, "module")) { read_next_value(parser, value, MAX_VALUE); finish_line(parser); s = ptrdict_find_section(self, value); if (!s) { printf("[ptrdict_lf_read_section] Unknown section '%s' in line %i.\n", value, parser->row); ptrdict_enum_subsections(self, stdout); exit(1); } else if (s->kind == SK_SECTION) { printf("[ptrdict_lf_read_section] '%s' is a section identifier (line %i).\n", value, parser->row); exit(1); } ptrdict_lf_read_section(s, parser); } else { p = ptrdict_find_property(self, keyword); if (!p) { printf("[ptrdict_lf_read_section] Unknown keyword '%s' in line %i.\n" "Possibilities are 'section', 'module' or one of the properties of section '%s', which are:\n", keyword, parser->row, self->name); ptrdict_enum_properties(self, stdout); exit(1); } read_next_token(parser, token, MAX_TOKEN); if (strcmp(token, "=")) { printf("[ptrdict_lf_read_section] '=' expected for assignment of property '%s' in line %i.\n", keyword, parser->row); exit(1); } read_next_value(parser, value, MAX_VALUE); finish_line(parser); ptrdict_set_property(p, value); } } if (!section_done) { printf("[ptrdict_lf_read_section] Error: End-of-file reached, but keyword 'endsection' is missing (line %i).\n", parser->row); exit(1); } }
doVector* statmentOp(tree& symbolTable, tree& localVar, funcS& funcTable){ doVector* ifStatments; ifStatments = new doVector(20);// 20 default starting capacity opNode exp; read_next_token(); if(String (next_token()) == "nufed"){return 0;} doVector::statment loop; while(String (next_token()) != "fi" || String(next_token()) != "od" || String (next_token()) != "nufed"){ String next(next_token()); if(next == "output"){ loop.c = output; exp.parseOperateTemp(String (next_token()), 0, symbolTable, loop.operate); ifStatments->push_back(loop); }else if(next == "set"){ loop.c = set; read_next_token(); loop.text = String (next_token()); exp.parseOperateTemp(String (next_token()), 0, symbolTable, loop.operate); ifStatments->push_back(loop); }else if(next == "var"){ loop.c = var; read_next_token(); loop.text = String (next_token()); loop.text = String (next_token()); exp.parseOperateTemp(String (next_token()), 0, symbolTable, loop.operate); ifStatments->push_back(loop); }else if(next == "text"){ loop.c = text; read_next_token(); loop.text = String (next_token()); ifStatments->push_back(loop); }else if(next == "do"){ loop.c = dos; exp.parseOperateTemp(String (next_token()), 0, symbolTable, loop.operate); loop.block = statmentOp(symbolTable, localVar, funcTable); ifStatments->push_back(loop); }else if(next == "else"){ loop.c = elses; loop.block = elseStatmentOp(symbolTable, localVar, funcTable); ifStatments->push_back(loop); }else if(next == "if"){ loop.c = ifs; exp.parseOperateTemp(String (next_token()), 0, symbolTable, loop.operate); loop.block = statmentOp(symbolTable, localVar, funcTable); ifStatments->push_back(loop); }else if(next == "call"){ loop.c = call; exp.parseOperateTemp(String (next_token()), 0, symbolTable, loop.operate); ifStatments->push_back(loop); }else if(next == "return"){ loop.c = returns; exp.parseOperateTemp(String (next_token()), 0, symbolTable, loop.operate); ifStatments->push_back(loop); }else if(next == "defun"){ loop.c = func; peek_next_token(); loop.text = String (next_token()); ifStatments->push_back(loop); functionHandler(symbolTable, tempFunc); } peek_next_token(); if(String (next_token()) == "fi" || String (next_token()) == "od" || String (next_token()) == "nufed" || next_token_type == END){ read_next_token(); break;}else{ read_next_token(); } } return ifStatments; }
doVector* elseStatmentOp(tree& symbolTable, tree& localVar, funcS& funcTable){ doVector* ifStatments; ifStatments = new doVector(20);// 20 default starting capacity opNode exp; read_next_token(); doVector::statment loop;// temporary place holder of the elements to be pushed to the vector while(String (next_token()) != "fi"){ String next(next_token()); if(next == "output"){ loop.c = output; exp.parseOperateTemp(String (next_token()), 0, symbolTable, loop.operate); ifStatments->push_back(loop); }else if(next == "set"){ loop.c = set; read_next_token(); loop.text = String (next_token()); exp.parseOperateTemp(String (next_token()), 0, symbolTable, loop.operate); ifStatments->push_back(loop); }else if(next == "var"){ loop.c = var; read_next_token(); loop.text = String (next_token()); exp.parseOperateTemp(String (next_token()), 0, symbolTable, loop.operate); ifStatments->push_back(loop); }else if(next == "text"){ loop.c = text; read_next_token(); loop.text = String (next_token()); ifStatments->push_back(loop); }else if(next == "do"){ loop.c = dos; exp.parseOperateTemp(String (next_token()), 0, symbolTable, loop.operate); loop.block = statmentOp(symbolTable, localVar, funcTable); ifStatments->push_back(loop); }else if(next == "else"){ loop.c = elses; loop.block = statmentOp(symbolTable, localVar, funcTable); ifStatments->push_back(loop); }else if(next == "if"){ loop.c = ifs; exp.parseOperateTemp(String (next_token()), 0, symbolTable, loop.operate); loop.block = statmentOp(symbolTable, localVar, funcTable); ifStatments->push_back(loop); loop.c = call; exp.parseOperateTemp(String (next_token()), 0, symbolTable, loop.operate); ifStatments->push_back(loop); }else if(next == "return"){ loop.c = returns; exp.parseOperateTemp(String (next_token()), 0, symbolTable, loop.operate); ifStatments->push_back(loop); }else if(next == "defun"){ loop.c = func; peek_next_token(); loop.text = String (next_token()); ifStatments->push_back(loop); functionHandler(symbolTable, tempFunc); } peek_next_token(); if(String (next_token()) == "fi"){ break;}else{ read_next_token(); } } return ifStatments; }
/** * Given a program struct, will process the (already opened) input file and * begin compilation line by line. */ void process_input_program(struct program *program){ print_asterisk(GRN_C, stdout); printf("Processing File...\n"); char *tok; char buf[MAX_LINE_LEN+1]; buf[MAX_LINE_LEN] = 0; // parse input file do{ program->line_count++; #ifdef DEBUG fprintf(stderr, "*** Reading Line %d...\n", program->line_count); #endif fgetpos(program->in, &program->str_line); tok = read_next_token(buf, program->in, MAX_LINE_LEN); #ifdef DEBUG fprintf(stderr, "Read Token '%s'\n", tok); #endif if(!tok){ continue; // just a whitespace line, move on } else{ process_token(tok, program); } // the instruction processor should have consumed all relevant tokens, // check if there is garbage at the end of the line if(!program->error_code) check_garbage(program); }while(!check_EOF(program->in) && !program->error_code); if(program->error_code) return; // resolve constants/labels #ifdef DEBUG fprintf(stderr, "LAST TERM: '%s' TRANS: %d\n", program->end_term->term, program->end_term->trans); #endif translate_terms(program->terms, program); // write terms to out file struct Term *t = program->terms; write_terms(t, program); // process warnings if(warnings){ check_warnings(program); } #ifdef DEBUG // why did we quit? if(program->error_code){ fprintf(stderr, "Stopped processing because of an error.\n"); } else if(check_EOF(program->in)){ fprintf(stderr, "Stopped processing because EOF reached.\n"); } #endif }
/* * List the tags that in the files. */ static void list_tags(void) { char path[1024], *path_p, token[80]; default_t *def_p; const char *home_p; long new_debug = 0; FILE *rc_file; /* do we need to have a home variable? */ if (inpath == NULL) { /* first we try to read the RC file from the current directory */ rc_file = fopen(DEFAULT_CONFIG, "r"); if (rc_file == NULL) { /* if no file in current directory, try home directory */ home_p = getenv(HOME_ENVIRON); if (home_p == NULL) { (void)fprintf(stderr, "%s: could not find variable '%s'\n", argv_program, HOME_ENVIRON); exit(1); } (void)loc_snprintf(path, sizeof(path), "%s/%s", home_p, DEFAULT_CONFIG); path_p = path; rc_file = fopen(path, "r"); /* we don't check for error right here */ } else { path_p = DEFAULT_CONFIG; } } else { /* open the specified file */ rc_file = fopen(inpath, "r"); /* we assume that if the file was specified, it must be there */ if (rc_file == NULL) { (void)fprintf(stderr, "%s: could not read '%s': ", argv_program, inpath); perror(""); exit(1); } path_p = inpath; } if (rc_file != NULL) { (void)fprintf(stderr, "Tags available from '%s':\n", path_p); while (read_next_token(rc_file, &new_debug, token, sizeof(token)) == 1) { if (verbose_b) { (void)fprintf(stderr, "%s (%#lx):\n", token, new_debug); dump_debug(new_debug); } else { (void)fprintf(stderr, "%s\n", token); } } (void)fclose(rc_file); } (void)fprintf(stderr, "\n"); (void)fprintf(stderr, "Tags available by default:\n"); for (def_p = defaults; def_p->de_string != NULL; def_p++) { if (verbose_b) { (void)fprintf(stderr, "%s (%#lx):\n", def_p->de_string, def_p->de_flags); dump_debug(def_p->de_flags); } else { (void)fprintf(stderr, "%s\n", def_p->de_string); } } }
/* Read the current section until 'endsection' is reached. */ void ptrdict_sf_read_section(section_t *self, parser_t *parser) { BOOL section_done = FALSE; section_t *s; property_t *p; char keyword[MAX_KEYWORD+1]; char value[MAX_VALUE+1]; char token[MAX_TOKEN+1]; BOOL is_token; self->provided = TRUE; if (self->provided_notification) *self->provided_notification = TRUE; while (!section_done && !at_end_of_file(parser)) { read_next_keyword(parser, keyword, MAX_KEYWORD, &is_token); if (is_token) strcpy(token, keyword); else read_next_token(parser, token, MAX_TOKEN); if (!strcmp(token, "};")) { section_done = TRUE; } else if (!strcmp(token, "}")) { read_next_token(parser, token, MAX_TOKEN); if (strcmp(token, ";")) { printf("[ptrdict_sf_read_section] ';' expected in line %i.\n", parser->row); exit(1); } section_done = TRUE; } else if (!strcmp(token, "{};")) { /* This is an empty module. */ s = ptrdict_find_section(self, keyword); if (!s) { printf("[ptrdict_sf_read_section] Unknown section '%s' in line %i.\n", keyword, parser->row); ptrdict_enum_subsections(self, stdout); exit(1); } if (s->kind != SK_MODULE) { printf("[ptrdict_sf_read_section] Module expected, but section encountered in line %i.", parser->row); exit(1); } s->provided = TRUE; if (s->provided_notification) *s->provided_notification = TRUE; } else if (!strcmp(token, "{")) { /* We have a section or module */ s = ptrdict_find_section(self, keyword); if (!s) { printf("[ptrdict_sf_read_section] Unknown section '%s' in line %i.\n", keyword, parser->row); ptrdict_enum_subsections(self, stdout); exit(1); } ptrdict_sf_read_section(s, parser); } else if (!strcmp(token, "=")) { p = ptrdict_find_property(self, keyword); if (!p) { printf("[ptrdict_sf_read_section] Unknown property '%s' of section '%s' in line %i.\n" "Possibilities are:\n", keyword, self->name, parser->row); ptrdict_enum_properties(self, stdout); exit(1); } read_next_value(parser, value, MAX_VALUE); ptrdict_set_property(p, value); read_next_token(parser, token, MAX_TOKEN); if (strcmp(token, ";")) { printf("[ptrdict_sf_read_section] ';' expected in line %i.\n", parser->row); exit(1); } } else { printf("[ptrdict_sf_read_section] Syntax error in line %i. Token = '%s'\n", parser->row, token); exit(1); } } if (!section_done) { printf("[ptrdict_sf_read_section] Error: End-of-file reached, but file is incomplete."); exit(1); } }
token_sp lexer::read_token() { token_sp t = read_next_token(); std::swap(t, current_token); return t; }
void lexer::advance_token() { current_token = read_next_token(); }