static void register_no_return_funcs(void)
{
struct token *token;
const char *func;
char name[256];
if (option_project == PROJ_NONE)
strcpy(name, "no_return_funcs");
else
snprintf(name, 256, "%s.no_return_funcs", option_project_str);
token = get_tokens_file(name);
if (!token)
return;
if (token_type(token) != TOKEN_STREAMBEGIN)
return;
token = token->next;
while (token_type(token) != TOKEN_STREAMEND) {
if (token_type(token) != TOKEN_IDENT)
return;
func = show_ident(token->ident);
add_function_hook(func, &__match_nullify_path_hook, NULL);
token = token->next;
}
clear_token_alloc();
}
static void register_shifters(void)
{
char filename[256];
struct token *token;
char *name;
int *val;
snprintf(filename, sizeof(filename), "%s.bit_shifters", option_project_str);
token = get_tokens_file(filename);
if (!token)
return;
if (token_type(token) != TOKEN_STREAMBEGIN)
return;
token = token->next;
while (token_type(token) != TOKEN_STREAMEND) {
if (token_type(token) != TOKEN_IDENT)
return;
name = alloc_string(show_ident(token->ident));
token = token->next;
if (token_type(token) != TOKEN_NUMBER)
return;
val = malloc(sizeof(int));
*val = atoi(token->number);
insert_struct(shifters, name, val);
token = token->next;
}
clear_token_alloc();
}
static void register_ignored_macros(void)
{
struct token *token;
char *macro;
char name[256];
if (option_project == PROJ_NONE)
strcpy(name, "ignored_macros");
else
snprintf(name, 256, "%s.ignored_macros", option_project_str);
token = get_tokens_file(name);
if (!token)
return;
if (token_type(token) != TOKEN_STREAMBEGIN)
return;
token = token->next;
while (token_type(token) != TOKEN_STREAMEND) {
if (token_type(token) != TOKEN_IDENT)
return;
macro = alloc_string(show_ident(token->ident));
add_ptr_list(&__ignored_macros, macro);
token = token->next;
}
clear_token_alloc();
}
Object * s_exp(void) {
Object *cell = _nil;
switch(token_type()) {
case T_NUMBER:
cell = number(atoi(token.token));
match(T_NUMBER);
break;
case T_SYMBOL:
cell = symbol(token.token);
match(T_SYMBOL);
break;
case T_LEFTPAREN:
match(T_LEFTPAREN);
cell = s_exp_list();
match(T_RIGHTPAREN);
break;
case T_END:
break;
default:
printf("%s:%d error, did not expect type %d %s\n",
__FILE__, __LINE__, token_type(), type_str(token_type()));
exit(-1);
}
return cell;
}
static void register_funcs_from_file(void)
{
struct token *token;
const char *func;
int arg;
token = get_tokens_file("kernel.dma_funcs");
if (!token)
return;
if (token_type(token) != TOKEN_STREAMBEGIN)
return;
token = token->next;
while (token_type(token) != TOKEN_STREAMEND) {
if (token_type(token) != TOKEN_IDENT)
return;
func = show_ident(token->ident);
token = token->next;
if (token_type(token) != TOKEN_NUMBER)
return;
arg = atoi(token->number);
add_function_hook(func, &match_dma_func, INT_PTR(arg));
token = token->next;
}
clear_token_alloc();
}
static void register_silenced_functions(void)
{
struct token *token;
char *func;
char name[256];
silenced_funcs = create_function_hashtable(500);
if (option_project == PROJ_NONE)
return;
snprintf(name, 256, "%s.silenced_functions", option_project_str);
token = get_tokens_file(name);
if (!token)
return;
if (token_type(token) != TOKEN_STREAMBEGIN)
return;
token = token->next;
while (token_type(token) != TOKEN_STREAMEND) {
if (token_type(token) != TOKEN_IDENT)
return;
func = alloc_string(show_ident(token->ident));
insert_func(silenced_funcs, func, INT_PTR(1));
token = token->next;
}
clear_token_alloc();
}
static
parse_err_t parse_list(parse_state_t *state, node_t **result)
{
parse_err_t status = PARSE_OK;
node_t *child = NULL, *next = NULL;
switch(token_type(state->ts)) {
case TOK_END:
status = PARSE_TOKEN_UNDERFLOW;
*result = NULL;
break;
case TOK_RPAREN:
/* ... ) */
/* check but don't consume this RPAREN because the parse_sexpr
that calls this function will check for it when this function
returns */
*result = NULL;
break;
case TOK_DOT:
token_chomp(state->ts);
switch(token_type(state->ts)) {
case TOK_END:
status = PARSE_TOKEN_UNDERFLOW;
break;
case TOK_RPAREN:
status = PARSE_UNEXPECTED_RPAREN;
break;
case TOK_DOT:
status = PARSE_UNEXPECTED_DOT;
break;
default:
status = parse_sexpr(state, result);
break;
}
break;
default:
status = parse_sexpr(state, &child);
if(status == PARSE_OK) {
status = parse_list(state, &next);
if(status == PARSE_OK) {
*result = node_cons_new(state->ms, child, next);
} else {
node_droproot(next);
}
}
}
return status;
}
struct token *get_string_constant(SCTX_ struct token *token, struct expression *expr)
{
struct string *string = token->string;
struct token *next = token->next, *done = NULL;
int stringtype = token_type(token);
int is_wide = stringtype == TOKEN_WIDE_STRING;
static char buffer[MAX_STRING];
int len = 0;
int bits;
while (!done) {
switch (token_type(next)) {
case TOKEN_WIDE_STRING:
is_wide = 1;
case TOKEN_STRING:
next = next->next;
break;
default:
done = next;
}
}
bits = is_wide ? 32 : sctxp bits_in_char;
while (token != done) {
unsigned v;
const char *p = token->string->data;
const char *end = p + token->string->length - 1;
while (p < end) {
p = parse_escape(sctx_ p, &v, end, bits, token->pos);
if (len < MAX_STRING)
buffer[len] = v;
len++;
}
token = token->next;
}
if (len > MAX_STRING) {
warning(sctx_ token->pos, "trying to concatenate %d-character string (%d bytes max)", len, MAX_STRING);
len = MAX_STRING;
}
if (len >= string->length) /* can't cannibalize */
string = __alloc_string(sctx_ len+1);
string->length = len+1;
string->used = string->length;
memcpy(string->data, buffer, len);
string->data[len] = '\0';
expr->string = string;
expr->wide = is_wide;
return token;
}
int test_parse_expression_list()
{
int result = 0;
char source[] = "class_name.sub_name(87, var_name);\n";
test_msg_start("Testing Expression Parsing - Expression List");
pC = source;
pC = advance(pC, pT);
tk = token_type(pT);
parse_expression(0);
/* pointer to code should be in correct position */
if(pC - source != 34) { result++; }
/* token should be correct */
if(strcmp(pT, ";") != 0) { result++; }
if(result == PASSED)
{
test_msg_end(PASSED);
} else {
test_msg_end(FAILED);
}
return result;
}
static CNode * parse_string(CNode * n, char **saveptr)
{
char *q;
CNode *m, *last_child=NULL;
while ((q = strtok_r(NULL, " ", saveptr))) {
switch (token_type(q)) {
case CLOSE_TOK :
q[strlen(q)-1]='\0';
assert(strcmp(q, n->label)==0,
"Constituent tree: Labels do not match.");
return n;
break;
case OPEN_TOK:
m = make_CNode(q+1);
m = parse_string(m, saveptr);
break;
case WORD_TOK:
m = make_CNode(q);
break;
default:
assert(0, "Constituent tree: Illegal token type");
}
if (n->child == NULL) {
last_child = n->child = m;
}
else {
last_child->next = m;
last_child = m;
}
}
assert(0, "Constituent tree: Constituent did not close");
return NULL;
}
int test_parse_expression_nested_sub()
{
int result = 0;
char source[] = "data1(data2(12));\n";
test_msg_start("Testing Expression Parsing - Nested Subroutine");
pC = source;
pC = advance(pC, pT);
tk = token_type(pT);
parse_expression(0);
/* pointer to code should be in correct position */
if(pC - source != 17) { result++; }
/* token should be correct */
if(strcmp(pT, ";") != 0) { result++; }
if(result == PASSED)
{
test_msg_end(PASSED);
} else {
test_msg_end(FAILED);
}
return result;
}
int test_token_type()
{
int result = 0;
int i = 0;
test_msg_start("Testing Token Type Recognition");
while(keywords[i] != NULL)
{
strcpy(pT, keywords[i]);
tk = -1; /* be certain this value is changed */
tk = token_type(pT);
if(tk != KEYWORD) { result++; }
i++;
}
if(result == PASSED)
{
test_msg_end(PASSED);
} else {
test_msg_end(FAILED);
}
return result;
}
bool try_to_recognize_tokens(const string &str, string_v &tokens)
{
int len = str.length();
if (len == 0)
return true;
for (int l = len ; l > 0 ; l--)
{
string s = str.substr(0, l);
if (token_type(s) != invalid)
{
tokens.push_back(s);
if (try_to_recognize_tokens(str.substr(l, len-l), tokens))
return true;
tokens.pop_back();
return false;
}
}
return false;
}
int test_parse_expression()
{
int result = 0;
char source[] = "-(-(2 + (-12 / 12)*83));\n";
test_msg_start("Testing Expression Parsing - Numerical");
pC = source;
pC = advance(pC, pT);
tk = token_type(pT);
parse_expression(0);
/* pointer to code should be in correct position */
if(pC - source != 24) { result++; }
/* token should be correct */
if(strcmp(pT, ";") != 0) { result++; }
if(result == PASSED)
{
test_msg_end(PASSED);
} else {
test_msg_end(FAILED);
}
return result;
}
/*---------------------------------------------------------------------*/
obj_t *
readobj( FILE *file ) {
token_t *tok = parse_token( file );
if( !tok ) {
return 0L;
} else {
switch( tok->tok ) {
case TOKEN_OPENPAR:
return (obj_t *)readlist( file );
case TOKEN_SYMBOL:
return (obj_t *)make_symbol( tok->val );
case TOKEN_STRING:
return (obj_t *)make_string( tok->val );
case TOKEN_INT:
return (obj_t *)make_integer( atol( tok->val ) );
default:
fprintf( stderr, "Illegal %s: %s\n",
token_type( tok ),
tok->val );
return (obj_t *)NIL;
}
}
}
/** \brief Reset the token reader to the start of the formula.
The syntax flags will be reset to a value appropriate for the
start of a formula.
\post #m_iPos==0, #m_iSynFlags = noOPT | noBC | noPOSTOP | noSTR
\throw nothrow
\sa ESynCodes
*/
void ParserTokenReader::ReInit()
{
m_iPos = 0;
m_iSynFlags = sfSTART_OF_LINE;
m_iBrackets = 0;
m_UsedVar.clear();
m_lastTok = token_type();
}
void match(int type) {
if (type != token_type()) {
printf("Error - expected %d %s , got '%s'\n",
type, type_str(type), token.token);
printf("Line %d, word %d\n", token.line, token.word);
exit(-1);
}
scanner();
}
parse_err_t parse(memory_state_t *ms, tok_state_t *ts, node_t *out_hdl)
{
parse_err_t status = PARSE_OK;
node_t *result = NULL;
parse_state_t state = { .ms = ms, .ts = ts };
if(token_type(state.ts) == TOK_INIT) {
token_chomp(state.ts);
}
status = parse_sexpr(&state, &result);
node_handle_update(out_hdl, result);
return status;
}
void Scanner::skipWS(char c){
do{
if(token_type(c)==WHITESPACE){
c = getChar();
}else{
break;
}
}while(true);
if(c!=END_OF_BUFFER){
putback();
}
}
static void register_returns_held_funcs(void)
{
struct token *token;
const char *func;
token = get_tokens_file("kernel.returns_held_funcs");
if (!token)
return;
if (token_type(token) != TOKEN_STREAMBEGIN)
return;
token = token->next;
while (token_type(token) != TOKEN_STREAMEND) {
if (token_type(token) != TOKEN_IDENT)
return;
func = show_ident(token->ident);
return_implies_state(func, valid_ptr_min, valid_ptr_max,
&match_returns_held, NULL);
return_implies_state(func, 0, 0, &match_returns_null,
NULL);
token = token->next;
}
clear_token_alloc();
}
string get_stripped_symbol(string &str)
{
TokenType tt = token_type(str);
if (tt == symbol)
return str.substr(1, str.length()-1);
if (tt == label)
return str.substr(0, str.length()-1);
if (tt == plain_identifier || tt == op_function)
return str;
halt;
}
static CNode * linkage_constituent_tree(Linkage linkage)
{
char *p, *q, *saveptr;
int len;
CNode * root;
p = print_flat_constituents(linkage);
len = strlen(p);
q = strtok_r(p, " ", &saveptr);
assert(token_type(q) == OPEN_TOK, "Illegal beginning of string");
root = make_CNode(q+1);
root = parse_string(root, &saveptr);
assign_spans(root, 0);
exfree(p, sizeof(char)*(len+1));
return root;
}
/** \brief Read the next token from the string. */
ParserTokenReader::token_type ParserTokenReader::ReadNextToken()
{
assert(m_pParser);
std::stack<int> FunArgs;
const char_type *szFormula = m_strFormula.c_str();
token_type tok;
// Ignore all non printable characters when reading the expression
while (szFormula[m_iPos]>0 && szFormula[m_iPos]<=0x20)
++m_iPos;
if ( IsEOF(tok) ) return SaveBeforeReturn(tok); // Check for end of formula
if ( IsOprt(tok) ) return SaveBeforeReturn(tok); // Check for user defined binary operator
if ( IsFunTok(tok) ) return SaveBeforeReturn(tok); // Check for function token
if ( IsBuiltIn(tok) ) return SaveBeforeReturn(tok); // Check built in operators / tokens
if ( IsArgSep(tok) ) return SaveBeforeReturn(tok); // Check for function argument separators
if ( IsValTok(tok) ) return SaveBeforeReturn(tok); // Check for values / constant tokens
if ( IsVarTok(tok) ) return SaveBeforeReturn(tok); // Check for variable tokens
if ( IsStrVarTok(tok) ) return SaveBeforeReturn(tok); // Check for string variables
if ( IsString(tok) ) return SaveBeforeReturn(tok); // Check for String tokens
if ( IsInfixOpTok(tok) ) return SaveBeforeReturn(tok); // Check for unary operators
if ( IsPostOpTok(tok) ) return SaveBeforeReturn(tok); // Check for unary operators
// Check String for undefined variable token. Done only if a
// flag is set indicating to ignore undefined variables.
// This is a way to conditionally avoid an error if
// undefined variables occur.
// (The GetUsedVar function must suppress the error for
// undefined variables in order to collect all variable
// names including the undefined ones.)
if ( (m_bIgnoreUndefVar || m_pFactory) && IsUndefVarTok(tok) )
return SaveBeforeReturn(tok);
// Check for unknown token
//
// !!! From this point on there is no exit without an exception possible...
//
string_type strTok;
int iEnd = ExtractToken(m_pParser->ValidNameChars(), strTok, m_iPos);
if (iEnd!=m_iPos)
Error(ecUNASSIGNABLE_TOKEN, m_iPos, strTok);
Error(ecUNASSIGNABLE_TOKEN, m_iPos, m_strFormula.substr(m_iPos));
return token_type(); // never reached
}
int test_parse_expression_keyword()
{
int result = 0, i = 0;
char *source[] =
{
"true;\n",
"null;\n",
"this;\n",
"false;\n"
};
test_msg_start("Testing Expression Parsing - Keywords");
while(i <= 3)
{
pC = source[i];
pC = advance(pC, pT);
tk = token_type(pT);
parse_expression(0);
/* pointer to code should be in correct position */
if(i <= 2)
{
if(pC - source[i] != 5) { result++; }
} else {
if(pC - source[i] != 6) { result++; }
}
/* token should be correct */
if(strcmp(pT, ";") != 0) { result++; }
i++;
}
if(result == PASSED)
{
test_msg_end(PASSED);
} else {
test_msg_end(FAILED);
}
return result;
}
/*---------------------------------------------------------------------*/
pair_t *
readlist( FILE *file ) {
token_t *tok = parse_token( file );
pair_t *res = NIL;
if( !tok ) {
fprintf( stderr, "Premature end of file\n" );
return 0L;
} else {
if( tok->tok == TOKEN_CLOPAR ) {
return res;
} else {
obj_t *car;
switch( tok->tok ) {
case TOKEN_OPENPAR:
car = (obj_t *)readlist( file );
break;
case TOKEN_SYMBOL:
car = (obj_t *)make_symbol( tok->val );
break;
case TOKEN_GUIL:
car = (obj_t *)readguil( file );
if( !car ) car = (obj_t *)NIL;
break;
case TOKEN_INT:
car = (obj_t *)make_integer( atol( tok->val ) );
break;
default:
fprintf( stderr, "Illegal %s: %s\n",
token_type( tok ),
tok->val );
car = (obj_t *)NIL;
}
return cons( car, readlist( file ) );
}
}
}
Object * s_exp_list(void) {
Object *cell = _nil;
cell = cons(s_exp(), _nil);
switch(token_type()) {
case T_RIGHTPAREN:
break;
case T_DOT:
match(T_DOT);
cell->Cons.cdr = s_exp();
break;
case T_END:
break;
default:
cell->Cons.cdr = s_exp_list();
break;
}
return cell;
}
static
parse_err_t parse_atom(parse_state_t *state, node_t **result)
{
switch(token_type(state->ts)) {
case TOK_SYM:
*result = node_symbol_new(state->ms, token_sym(state->ts));
break;
case TOK_LIT:
*result = node_value_new(state->ms, token_lit(state->ts));
break;
default:
return PARSE_EXPECTED_ATOM;
}
assert(*result);
token_chomp(state->ts);
return PARSE_OK;
}
bool LayoutParser::validateOldStyleImport()
{
QXmlStreamReader::TokenType token_type(QXmlStreamReader::Invalid);
do {
token_type = m_xml.readNext();
switch (token_type) {
case QXmlStreamReader::EndElement:
// that is what we expect:
// <import file="..."/>
// or:
// <import file="..."></import>
return true;
case QXmlStreamReader::Characters:
// just in case when we have a newline between
// opening and closing tag:
// <import file="...">
// </import>
if (not m_xml.isWhitespace()) {
error(QString::fromLatin1("Stray text in import tag: '%1'.").arg(m_xml.text().toString()));
return false;
}
break;
case QXmlStreamReader::Invalid:
case QXmlStreamReader::Comment:
break;
case QXmlStreamReader::StartElement:
error(QString::fromLatin1("Expected no child tags, because 'file' attribute exists, but got '<%1>'.").arg(m_xml.name().toString()));
return false;
default:
error("Wrong use of import tag.");
return false;
}
} while (token_type != QXmlStreamReader::Invalid);
return false;
}
void get_char_constant(SCTX_ struct token *token, unsigned long long *val)
{
const char *p = token->embedded, *end;
unsigned v;
int type = token_type(token);
switch (type) {
case TOKEN_CHAR:
case TOKEN_WIDE_CHAR:
p = token->string->data;
end = p + token->string->length;
break;
case TOKEN_CHAR_EMBEDDED_0 ... TOKEN_CHAR_EMBEDDED_3:
end = p + type - TOKEN_CHAR;
break;
default:
end = p + type - TOKEN_WIDE_CHAR;
}
p = parse_escape(sctx_ p, &v, end,
type < TOKEN_WIDE_CHAR ? sctxp bits_in_char : 32, token->pos);
if (p != end)
warning(sctx_ token->pos,
"multi-character character constant");
*val = v;
}
/* Parse the file. return 0 on success */
static int
parse(char *buffer, struct symbol *list)
{
struct symbol *curr, *last;
char *token, *pt;
char sym[1024];
last = list;
token = next_token(buffer);
pt = list->symbol;
curr = NULL;
if (token == NULL) {
printf("No profile found\n");
return (1);
}
do {
int merge = 0;
if ((curr) && (curr->symbol)) {
pt = curr->symbol;
}
if ((token[0] == '>') && (token[1] == '\0')) {
free(token);
continue;
}
if (token[0] == '=')
merge = 1;
else if (pt[strlen(pt) - 1] == '=')
merge = 1;
else if ((strlen(pt) == 2) && (strncmp(pt, "</", 2) == 0))
merge = 1;
else if ((strlen(pt) == 1) && (pt[0] == '<'))
merge = 1;
if (merge == 1) {
/* merge this with prev symbol */
strlcpy(sym, curr->symbol, 1024);
strlcat(sym, token, 1024);
if (curr->symbol)
free(curr->symbol);
curr->type = token_type(sym);
curr->symbol = get_symbol(strdup(sym));
} else {
curr = calloc(1, sizeof (struct symbol));
curr->type = token_type(token);
if (curr->type != TOKEN_ERROR)
curr->symbol = get_symbol(token);
else
curr->symbol = token;
curr->prev = last;
curr->next = NULL;
last->next = curr;
pt = curr->symbol;
last = curr;
}
} while ((token = next_token(NULL)) != NULL);
return (0);
}
