void DaoLexer_AppendToken( DaoLexer *self, DaoToken *token ) { DaoToken *tok; if( self->tokbuf->size == 0 ){ DArray_Append( self->tokens, token ); return; } tok = (DaoToken*) DArray_Back( self->tokbuf ); self->tokbuf->size -= 1; DaoToken_Assign( tok, token ); DArray_Append( self->tokens, NULL ); /* avoid copying; */ self->tokens->items.pToken[self->tokens->size-1] = tok; }
DaoToken* DaoToken_Copy( DaoToken *self ) { DaoToken* copy = DaoToken_New(); DaoToken_Assign( copy, self ); return copy; }
static int DaoParser_MakeMacroGroup( DaoParser *self, DMacroGroup *group, DMacroGroup *parent, int from, int to ) { DaoToken **toks = self->tokens->items.pToken; unsigned char tk; int i, sep, rb, prev; DMacroUnit *unit; DMacroGroup *grp, *group2; /* mingw don't like grp2 ?! */ /* for( i=from; i<to; i++ ) printf( "%s ", toks[i]->mbs ); printf("\n"); */ i = from; while( i < to ){ char *chs = toks[i]->string.mbs; self->curLine = toks[i]->line; tk = toks[i]->name; #if 0 //printf( "%i %s\n", i, chs ); #endif if( tk == DTOK_ESC_LB || tk == DTOK_ESC_LSB || tk == DTOK_ESC_LCB ){ grp = DMacroGroup_New(); grp->cpos = toks[i]->cpos; grp->parent = parent; DArray_Append( group->units, (void*)grp ); switch( tk ){ case DTOK_ESC_LB : rb = DaoParser_FindPairToken( self, DTOK_ESC_LB, DTOK_ESC_RB, i, to ); break; case DTOK_ESC_LSB : rb = DaoParser_FindPairToken( self, DTOK_ESC_LSB, DTOK_ESC_RSB, i, to ); grp->repeat = DMACRO_ZERO_OR_ONE; break; case DTOK_ESC_LCB : rb = DaoParser_FindPairToken( self, DTOK_ESC_LCB, DTOK_ESC_RCB, i, to ); grp->repeat = DMACRO_ZERO_OR_MORE; break; default : { rb = -1; DaoParser_Error( self, DAO_CTW_INV_MAC_OPEN, & toks[i]->string ); break; } } if( rb <0 ) return 0; prev = i+1; sep = DaoParser_FindOpenToken( self, DTOK_ESC_PIPE, i+1, rb, 0 ); if( sep >=0 ){ while( sep >=0 ){ group2 = DMacroGroup_New(); group2->parent = grp; if( DaoParser_MakeMacroGroup( self, group2, group2, prev, sep ) == 0 ) return 0; DArray_Append( grp->units, (void*)group2 ); prev = sep +1; sep = DaoParser_FindOpenToken( self, DTOK_ESC_PIPE, prev, rb, 0 ); if( prev < rb && sep <0 ) sep = rb; } grp->type = DMACRO_ALT; }else if( DaoParser_MakeMacroGroup( self, grp, grp, i+1, rb ) == 0 ){ return 0; } i = rb +1; self->curLine = toks[i]->line; if( toks[i]->string.mbs[0] == '\\' ){ switch( toks[i]->name ){ case DTOK_ESC_EXCLA : grp->repeat = DMACRO_ZERO; i++; break; case DTOK_ESC_QUES : grp->repeat = DMACRO_ZERO_OR_ONE; i++; break; case DTOK_ESC_STAR : grp->repeat = DMACRO_ZERO_OR_MORE; i++; break; case DTOK_ESC_PLUS : grp->repeat = DMACRO_ONE_OR_MORE; i++; break; case DTOK_ESC_SQUO : case DTOK_ESC_DQUO : case DTOK_ESC_LB : case DTOK_ESC_RB : case DTOK_ESC_LCB : case DTOK_ESC_RCB : case DTOK_ESC_LSB : case DTOK_ESC_RSB : break; default : DaoParser_Error( self, DAO_CTW_INV_MAC_REPEAT, & toks[i]->string ); return 0; } } continue; } self->curLine = toks[i]->line; unit = DMacroUnit_New(); DaoToken_Assign( unit->marker, toks[i] ); DArray_Append( group->units, (void*)unit ); switch( chs[0] ){ case '$' : if( DString_FindMBS( & toks[i]->string, "EXP", 0 ) == 1 ){ unit->type = DMACRO_EXP; }else if( DString_FindMBS( & toks[i]->string, "VAR", 0 ) == 1 ){ unit->type = DMACRO_VAR; }else if( DString_FindMBS( & toks[i]->string, "ID", 0 ) == 1 ){ unit->type = DMACRO_ID; }else if( DString_FindMBS( & toks[i]->string, "OP", 0 ) == 1 ){ unit->type = DMACRO_OP; }else if( DString_FindMBS( & toks[i]->string, "BL", 0 ) == 1 ){ unit->type = DMACRO_BL; }else if( DString_FindMBS( & toks[i]->string, "IBL", 0 ) == 1 ){ unit->type = DMACRO_IBL; } break; case '(' : case '[' : case '{' : switch( chs[0] ){ case '(' : rb = DaoParser_FindPairToken( self, DTOK_LB, DTOK_RB, i, to ); break; case '[' : rb = DaoParser_FindPairToken( self, DTOK_LSB, DTOK_RSB, i, to ); break; case '{' : rb = DaoParser_FindPairToken( self, DTOK_LCB, DTOK_RCB, i, to ); break; default : rb = -1; } if( rb <0 ) return 0; grp = DMacroGroup_New(); grp->parent = group; grp->repeat = DMACRO_AUTO; DArray_Append( group->units, (void*)grp ); if( DaoParser_MakeMacroGroup( self, grp, parent, i+1, rb ) == 0 ) return 0; i = rb; continue; case '\'' : if( chs[2] == '\'' ){ unit->marker->type = 0; switch( chs[1] ){ case '(' : unit->marker->type = unit->marker->name = DTOK_LB; break; case ')' : unit->marker->type = unit->marker->name = DTOK_RB; break; case '[' : unit->marker->type = unit->marker->name = DTOK_LSB; break; case ']' : unit->marker->type = unit->marker->name = DTOK_RSB; break; case '{' : unit->marker->type = unit->marker->name = DTOK_LCB; break; case '}' : unit->marker->type = unit->marker->name = DTOK_RCB; break; case '\'' : unit->marker->type= unit->marker->name = DTOK_ESC_SQUO;break; case '\"' : unit->marker->type= unit->marker->name = DTOK_ESC_DQUO;break; default : break; } if( unit->marker->type == 0 ){ DaoParser_Error( self, DAO_CTW_INV_MAC_SPECTOK, & toks[i]->string ); return 0; } unit->type = DMACRO_BR; DString_SetMBS( & unit->marker->string, chs+1 ); DString_Erase( & unit->marker->string, unit->marker->string.size-1, 1 ); } default : break; } if( i+1 < to && toks[i+1]->string.mbs[0] == '@' ){ char ch = toks[i+1]->string.mbs[1]; if( ch != '@' ){ if( toks[i+1]->string.size != 2 || ch < '1' || ch >'9' ){ DaoParser_Error( self, DAO_CTW_INV_MAC_INDENT, & toks[i+1]->string ); return 0; } unit->indent = ch - '0'; i ++; } } i ++; } return 1; }
static int DaoParser_MacroApply( DaoParser *self, DArray *tokens, DMacroGroup *group, DMap *tokMap, DMap *used, int level, DString *tag, int pos0, int adjust ) { DMacroUnit **units = (DMacroUnit**) group->units->items.pVoid; DMacroUnit *unit; DMacroGroup *grp; DMacroNode *node, *node2; DArray *toks = DArray_New(D_TOKEN); DaoToken *tk = DaoToken_New(); DaoToken *tt = NULL; DNode *kwnode = NULL; DMap *check = NULL; DMap one = { NULL, 0, 0, 0 }; int M, N = group->units->size; int i, j, gid = -1; int repeated = 0; int start_mbs = -1; int start_wcs = -1; int squote, dquote; if( group->repeat != DMACRO_AUTO ) level ++; for( i=0; i<N; i++ ){ unit = units[i]; if( tokens->size >0 ) pos0 = tokens->items.pToken[ tokens->size -1 ]->line; self->curLine = pos0; /* printf( "apply unit %i: %i\n", i, unit->type ); */ switch( unit->type ){ case DMACRO_TOK : squote = unit->marker->type == DTOK_ESC_SQUO; dquote = unit->marker->type == DTOK_ESC_DQUO; if( (squote && start_mbs >=0) || (dquote && start_wcs >=0) ){ int qstart = squote ? start_mbs : start_wcs; tt = tokens->items.pToken[ qstart ]; for(j=qstart+1,M=tokens->size; j<M; j++){ DaoToken *jtok = tokens->items.pToken[j]; int t = j ? tokens->items.pToken[j-1]->type : 0; if( t == DTOK_IDENTIFIER && jtok->type == t ) DString_AppendChar( & tt->string, ' ' ); DString_Append( & tt->string, & jtok->string ); } if( squote ){ DString_AppendChar( & tt->string, '\'' ); DArray_Erase( tokens, start_mbs+1, tokens->size ); }else{ DString_AppendChar( & tt->string, '\"' ); DArray_Erase( tokens, start_wcs+1, tokens->size ); } start_mbs = -1; break; }else if( squote ){ start_mbs = tokens->size; DArray_Append( tokens, unit->marker ); tt = tokens->items.pToken[ start_mbs ]; tt->type = tt->name = DTOK_MBS; DString_SetMBS( & tt->string, "\'" ); break; }else if( dquote ){ start_wcs = tokens->size; DArray_Append( tokens, unit->marker ); tt = tokens->items.pToken[ start_wcs ]; tt->type = tt->name = DTOK_WCS; DString_SetMBS( & tt->string, "\"" ); break; } DArray_Append( tokens, unit->marker ); tokens->items.pToken[ tokens->size-1 ]->cpos += adjust; break; case DMACRO_VAR : DaoToken_Assign( tk, unit->marker ); DString_Append( & tk->string, tag ); DArray_Append( tokens, tk ); break; case DMACRO_EXP : case DMACRO_ID : case DMACRO_OP : case DMACRO_BL : case DMACRO_IBL : kwnode = MAP_Find( tokMap, & unit->marker->string ); if( kwnode ==NULL ){ DaoParser_Error( self, DAO_CTW_UNDEF_MAC_MARKER, & unit->marker->string ); goto Failed; } node = (DMacroNode*) kwnode->value.pVoid; kwnode = MAP_Find( used, unit ); if( kwnode == NULL ){ DMap_Insert( used, unit, & one ); kwnode = MAP_Find( used, unit ); } check = (DMap*) kwnode->value.pVoid; repeated = 1; /* printf( ">>>\n%s level %i: \n", unit->marker->string.mbs, level ); DMacroNode_Print( node ); printf( "\n" ); */ /* search a leaf */ node2 = DMacroNode_FindLeaf( node, check, level ); if( node2 ){ /* printf( "appending tokens\n" ); DMacroNode_Print( node2 ); printf( "\n" ); */ DArray_InsertArray( tokens, tokens->size, node2->leaves, 0, -1 ); DMap_Insert( check, node2, NULL ); /* DArray_Clear( node2->leaves ); */ }else{ DMacroNode_RemoveEmptyLeftBranch( node, level ); goto Failed; } break; case DMACRO_GRP : case DMACRO_ALT : grp = (DMacroGroup*) unit; DArray_Clear( toks ); j = DaoParser_MacroApply( self, toks, grp, tokMap, used, level, tag, pos0, adjust ); switch( grp->repeat ){ case DMACRO_AUTO : case DMACRO_ONE : if( j <0 && group->type != DMACRO_ALT ) goto Failed; repeated = (j>0); if( j >=0 ){ gid = i; DArray_InsertArray( tokens, tokens->size, toks, 0, -1 ); } break; case DMACRO_ZERO_OR_ONE : gid = i; repeated = (j>0); if( j >=0 ){ DArray_InsertArray( tokens, tokens->size, toks, 0, -1 ); } break; case DMACRO_ZERO_OR_MORE : gid = i; repeated = (j>0); if( j >=0 ){ DArray_InsertArray( tokens, tokens->size, toks, 0, -1 ); } while( j >0 ){ DArray_Clear( toks ); j = DaoParser_MacroApply( self, toks, grp, tokMap, used, level, tag, pos0, adjust ); if( j >0 ){ DArray_InsertArray( tokens, tokens->size, toks, 0, -1 ); } } break; case DMACRO_ONE_OR_MORE : if( j <0 && group->type != DMACRO_ALT ) goto Failed; repeated = (j>0); if( j >=0 ){ DArray_InsertArray( tokens, tokens->size, toks, 0, -1 ); } while( j >0 ){ gid = i; DArray_Clear( toks ); j = DaoParser_MacroApply( self, toks, grp, tokMap, used, level, tag, pos0, adjust ); if( j >0 ){ DArray_InsertArray( tokens, tokens->size, toks, 0, -1 ); } } break; } break; default : goto Failed; } if( group->type == DMACRO_ALT && gid >=0 ) break; } if( group->repeat != DMACRO_AUTO ) level --; if( group->type == DMACRO_ALT && gid <0 ) goto Failed; DaoToken_Delete( tk ); DArray_Delete( toks ); return repeated; Failed : DaoToken_Delete( tk ); DArray_Delete( toks ); return -1; }
static int DaoParser_MakeMacroGroup( DaoParser *self, DMacroGroup *group, DMacroGroup *parent, int from, int to, DMap *vars, DMap *markers ) { unsigned char tk; int i, sep, rb, prev; DaoToken **toks = self->tokens->items.pToken; DMacroGroup *grp, *group2; /* mingw don't like grp2 ?! */ DMacroUnit *unit; DNode *it; /* for( i=from; i<to; i++ ) printf( "%s ", toks[i]->mbs ); printf("\n"); */ i = from; while( i < to ){ DaoToken *tok = toks[i]; char *chs = tok->string.mbs; int tk = tok->name; #if 0 printf( "%i %s\n", i, chs ); #endif self->curLine = tok->line; if( tk == DTOK_LB || tk == DTOK_LSB || tk == DTOK_LCB ){ grp = DMacroGroup_New(); grp->cpos = tok->cpos; grp->parent = parent; DArray_Append( group->units, (void*)grp ); switch( tk ){ case DTOK_LB : rb = DaoParser_FindPairToken( self, DTOK_LB, DTOK_RB, i, to ); break; case DTOK_LSB : rb = DaoParser_FindPairToken( self, DTOK_LSB, DTOK_RSB, i, to ); grp->repeat = DMACRO_ZERO_OR_ONE; break; case DTOK_LCB : rb = DaoParser_FindPairToken( self, DTOK_LCB, DTOK_RCB, i, to ); grp->repeat = DMACRO_ZERO_OR_MORE; break; default : rb = -1; DaoParser_Error( self, DAO_CTW_INV_MAC_OPEN, & tok->string ); break; } if( rb <0 ) return 0; prev = i+1; sep = DaoParser_FindOpenToken( self, DTOK_PIPE, i+1, rb, 0 ); if( sep >=0 ){ while( sep >=0 ){ group2 = DMacroGroup_New(); group2->parent = grp; if( DaoParser_MakeMacroGroup( self, group2, group2, prev, sep, vars, markers ) == 0 ) return 0; DArray_Append( grp->units, (void*)group2 ); prev = sep +1; sep = DaoParser_FindOpenToken( self, DTOK_PIPE, prev, rb, 0 ); if( prev < rb && sep <0 ) sep = rb; } grp->type = DMACRO_ALT; }else if( DaoParser_MakeMacroGroup( self, grp, grp, i+1, rb, vars, markers ) == 0 ){ return 0; } i = rb +1; tok = toks[i]; self->curLine = tok->line; switch( tok->name ){ case DTOK_NOT : grp->repeat = DMACRO_ZERO; i++; break; case DTOK_QUERY : grp->repeat = DMACRO_ZERO_OR_ONE; i++; break; case DTOK_MUL : grp->repeat = DMACRO_ZERO_OR_MORE; i++; break; case DTOK_ADD : grp->repeat = DMACRO_ONE_OR_MORE; i++; break; default : break; } continue; } self->curLine = tok->line; unit = DMacroUnit_New(); DaoToken_Assign( unit->marker, tok ); DArray_Append( group->units, (void*)unit ); if( chs[0] == '$' ){ if( DString_FindMBS( & tok->string, "EXP", 0 ) == 1 ){ unit->type = DMACRO_EXP; }else if( DString_FindMBS( & tok->string, "VAR", 0 ) == 1 ){ unit->type = DMACRO_VAR; }else if( DString_FindMBS( & tok->string, "ID", 0 ) == 1 ){ unit->type = DMACRO_ID; }else if( DString_FindMBS( & tok->string, "OP", 0 ) == 1 ){ unit->type = DMACRO_OP; }else if( DString_FindMBS( & tok->string, "BL", 0 ) == 1 ){ unit->type = DMACRO_BL; }else{ DaoParser_Error( self, DAO_CTW_INV_MAC_SPECTOK, & tok->string ); return 0; } if( vars != NULL ){ it = DMap_Find( vars, & unit->marker->string ); if( it == NULL ) it = DMap_Insert( vars, & unit->marker->string, 0 ); it->value.pInt += 1; } }else if( tk == DTOK_MBS ){ DaoLexer_Reset( self->wlexer ); DaoLexer_Tokenize( self->wlexer, chs + 1, 0 ); if( self->wlexer->tokens->size == 2 ){ DaoToken_Assign( unit->marker, self->wlexer->tokens->items.pToken[0] ); if( markers != NULL ){ it = DMap_Find( markers, & unit->marker->string ); if( it == NULL ) it = DMap_Insert( markers, & unit->marker->string, 0 ); it->value.pInt += 1; } } DaoLexer_Reset( self->wlexer ); rb = -1; if( tok->string.size == 3 ){ switch( chs[1] ){ case '(': rb = DaoParser_FindPair( self, "'('", "')'", i, to ); break; case '[': rb = DaoParser_FindPair( self, "'['", "']'", i, to ); break; case '{': rb = DaoParser_FindPair( self, "'{'", "'}'", i, to ); break; default : break; } } if( rb >= 0 ){ grp = DMacroGroup_New(); grp->parent = group; grp->repeat = DMACRO_AUTO; DArray_Append( group->units, (void*)grp ); if( DaoParser_MakeMacroGroup( self, grp, parent, i+1, rb, vars, markers ) == 0 ) return 0; i = rb; continue; } } i ++; } return 1; }