// // Parse a subchain of input delimited by matching pairs: [],(),{},<> // [WARNING: no other subchain types are recognized!]. This function expects // g_cxx.pToken to point to the initial token of the pair ([{<. // It will parse input until the matching terminator token is found. // Inner parsing is done by cxxParserParseAndCondenseSubchainsUpToOneOf() // so this is actually a recursive subchain nesting algorithm. // boolean cxxParserParseAndCondenseCurrentSubchain( unsigned int uInitialSubchainMarkerTypes, boolean bAcceptEOF ) { CXXTokenChain * pCurrentChain = g_cxx.pTokenChain; g_cxx.pTokenChain = cxxTokenChainCreate(); CXXToken * pInitial = cxxTokenChainTakeLast(pCurrentChain); cxxTokenChainAppend(g_cxx.pTokenChain,pInitial); CXXToken * pChainToken = cxxTokenCreate(); pChainToken->iLineNumber = pInitial->iLineNumber; pChainToken->oFilePosition = pInitial->oFilePosition; // see the declaration of CXXTokenType enum. // Shifting by 8 gives the corresponding chain marker pChainToken->eType = (enum CXXTokenType)(g_cxx.pToken->eType << 8); pChainToken->pChain = g_cxx.pTokenChain; cxxTokenChainAppend(pCurrentChain,pChainToken); // see the declaration of CXXTokenType enum. // Shifting by 4 gives the corresponding closing token type unsigned int uTokenTypes = g_cxx.pToken->eType << 4; if(bAcceptEOF) uTokenTypes |= CXXTokenTypeEOF; boolean bRet = cxxParserParseAndCondenseSubchainsUpToOneOf( uTokenTypes, uInitialSubchainMarkerTypes ); g_cxx.pTokenChain = pCurrentChain; g_cxx.pToken = pCurrentChain->pTail; return bRet; }
void cxxTokenChainCondense(CXXTokenChain * tc,unsigned int uFlags) { if(!tc) return; if(tc->iCount <= 1) return; CXXToken * pCondensed = cxxTokenCreate(); pCondensed->eType = CXXTokenTypeUnknown; pCondensed->iLineNumber = tc->pHead->iLineNumber; pCondensed->oFilePosition = tc->pHead->oFilePosition; while(tc->iCount > 0) { CXXToken * t = cxxTokenChainTakeFirst(tc); cxxTokenAppendToString(pCondensed->pszWord,t); if( (!(uFlags & CXXTokenChainCondenseNoTrailingSpaces)) && t->bFollowedBySpace ) vStringPut (pCondensed->pszWord, ' '); pCondensed->bFollowedBySpace = t->bFollowedBySpace; cxxTokenDestroy(t); } cxxTokenChainAppend(tc,pCondensed); }
CXXToken * cxxTokenCreateAnonymousIdentifier(enum CXXTagKind k) { g_uNextAnonumousIdentiferId++; char szNum[32]; CXXToken * t = cxxTokenCreate(); t->eType = CXXTokenTypeIdentifier; vStringCopyS(t->pszWord, "__anon"); // The names of anonymous structures depend on the name of // the file being processed so the identifiers won't collide across // multiple ctags runs with different file sets. unsigned int uHash = hash((const unsigned char *)getInputFileName()); sprintf(szNum,"%08x%02x%02x",uHash,g_uNextAnonumousIdentiferId, k); vStringCatS(t->pszWord,szNum); t->bFollowedBySpace = TRUE; t->iLineNumber = getInputLineNumber(); t->oFilePosition = getInputFilePosition(); return t; }
CXXToken * cxxTokenChainExtractIndexRange( CXXTokenChain * tc, int iFirstIndex, int iLastIndex, unsigned int uFlags ) { if(!tc) return NULL; if(iFirstIndex < 0) return NULL; if(iFirstIndex >= tc->iCount) return NULL; CXXToken * pToken = tc->pHead; int idx = 0; while(pToken && (idx < iFirstIndex)) { idx++; pToken = pToken->pNext; } if(!pToken) return NULL; CXXToken * pRet = cxxTokenCreate(); pRet->iLineNumber = pToken->iLineNumber; pRet->oFilePosition = pToken->oFilePosition; pRet->eType = pToken->eType; cxxTokenAppendToString(pRet->pszWord,pToken); if( (!(uFlags & CXXTokenChainExtractRangeNoTrailingSpaces)) && pToken->bFollowedBySpace ) vStringPut (pRet->pszWord, ' '); pRet->bFollowedBySpace = pToken->bFollowedBySpace; while(idx < iLastIndex) { pToken = pToken->pNext; if(!pToken) return pRet; cxxTokenAppendToString(pRet->pszWord,pToken); if( (!(uFlags & CXXTokenChainExtractRangeNoTrailingSpaces)) && pToken->bFollowedBySpace ) vStringPut (pRet->pszWord, ' '); pRet->bFollowedBySpace = pToken->bFollowedBySpace; idx++; } return pRet; }
CXXToken * cxxTokenCreateAnonymousIdentifier(enum CXXTagKind k) { CXXToken * t = cxxTokenCreate(); anonGenerate (t->pszWord, "__anon", k); t->bFollowedBySpace = TRUE; t->iLineNumber = getInputLineNumber(); t->oFilePosition = getInputFilePosition(); return t; }
CXXToken * cxxTokenCreateKeyword(int iLineNumber,MIOPos oFilePosition,enum CXXKeyword eKeyword) { CXXToken * pToken = cxxTokenCreate(); pToken->iLineNumber = iLineNumber; pToken->oFilePosition = oFilePosition; pToken->eType = CXXTokenTypeKeyword; pToken->eKeyword = eKeyword; pToken->bFollowedBySpace = TRUE; vStringCatS(pToken->pszWord,cxxKeywordName(eKeyword)); return pToken; }
CXXToken * cxxTokenChainExtractRange( CXXToken * from, CXXToken * to, unsigned int uFlags ) { if(!from) return NULL; CXXToken * pToken = from; CXXToken * pRet = cxxTokenCreate(); pRet->iLineNumber = pToken->iLineNumber; pRet->oFilePosition = pToken->oFilePosition; pRet->eType = pToken->eType; cxxTokenAppendToString(pRet->pszWord,pToken); if( (!(uFlags & CXXTokenChainExtractRangeNoTrailingSpaces)) && pToken->bFollowedBySpace ) vStringPut (pRet->pszWord, ' '); pRet->bFollowedBySpace = pToken->bFollowedBySpace; while(pToken != to) { pToken = pToken->pNext; if(!pToken) return pRet; cxxTokenAppendToString(pRet->pszWord,pToken); if( (!(uFlags & CXXTokenChainExtractRangeNoTrailingSpaces)) && pToken->bFollowedBySpace ) vStringPut (pRet->pszWord, ' '); pRet->bFollowedBySpace = pToken->bFollowedBySpace; } return pRet; }
bool cxxParserParseNextToken(void) { CXXToken * t = cxxTokenCreate(); // The token chain should not be allowed to grow arbitrairly large. // The token structures are quite big and it's easy to grow up to // 5-6GB or memory usage. However this limit should be large enough // to accomodate all the reasonable statements that could have some // information in them. This includes multiple function prototypes // in a single statement (ImageMagick has some examples) but probably // does NOT include large data tables. if(g_cxx.pTokenChain->iCount > 16384) cxxTokenChainDestroyLast(g_cxx.pTokenChain); cxxTokenChainAppend(g_cxx.pTokenChain,t); g_cxx.pToken = t; cxxParserSkipToNonWhiteSpace(); // FIXME: this cpp handling is kind of broken: // it works only because the moon is in the correct phase. cppBeginStatement(); // This must be done after getting char from input t->iLineNumber = getInputLineNumber(); t->oFilePosition = getInputFilePosition(); if(g_cxx.iChar == EOF) { t->eType = CXXTokenTypeEOF; t->bFollowedBySpace = false; return false; } unsigned int uInfo = UINFO(g_cxx.iChar); //fprintf(stderr,"Char %c %02x info %u\n",g_cxx.iChar,g_cxx.iChar,uInfo); if(uInfo & CXXCharTypeStartOfIdentifier) { // word t->eType = CXXTokenTypeIdentifier; t->bFollowedBySpace = false; vStringPut(t->pszWord,g_cxx.iChar); // special case for tile, which may actually be an operator if(g_cxx.iChar == '~') { // may be followed by space! g_cxx.iChar = cppGetc(); if(isspace(g_cxx.iChar)) { t->bFollowedBySpace = true; g_cxx.iChar = cppGetc(); while(isspace(g_cxx.iChar)) g_cxx.iChar = cppGetc(); } // non space uInfo = UINFO(g_cxx.iChar); if(!(uInfo & CXXCharTypeStartOfIdentifier)) { // this is not an identifier after all t->eType = CXXTokenTypeOperator; if((!t->bFollowedBySpace) && g_cxx.iChar == '=') { // make ~= single token so it's not handled as // a separate assignment vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); t->bFollowedBySpace = isspace(g_cxx.iChar); } return true; } } else { g_cxx.iChar = cppGetc(); } for(;;) { uInfo = UINFO(g_cxx.iChar); if(!(uInfo & CXXCharTypePartOfIdentifier)) break; vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); } int iCXXKeyword = lookupKeyword(t->pszWord->buffer,g_cxx.eLanguage); if(iCXXKeyword >= 0) { if( ( (iCXXKeyword == CXXKeywordFINAL) && (!g_cxx.bParsingClassStructOrUnionDeclaration) ) || ( ( (iCXXKeyword == CXXKeywordPUBLIC) || (iCXXKeyword == CXXKeywordPROTECTED) || (iCXXKeyword == CXXKeywordPRIVATE) ) && (!g_cxx.bEnablePublicProtectedPrivateKeywords) ) ) { t->eType = CXXTokenTypeIdentifier; } else { t->eType = CXXTokenTypeKeyword; t->eKeyword = (enum CXXKeyword)iCXXKeyword; if(iCXXKeyword == CXXKeyword__ATTRIBUTE__) { // special handling for __attribute__ return cxxParserParseNextTokenCondenseAttribute(); } } } else { const cppMacroInfo * pMacro = cppFindMacro(vStringValue(t->pszWord)); if(pMacro) { CXX_DEBUG_PRINT("Macro %s",vStringValue(t->pszWord)); cxxTokenChainDestroyLast(g_cxx.pTokenChain); CXXToken * pParameterChain = NULL; if(pMacro->hasParameterList) { CXX_DEBUG_PRINT("Macro has parameter list"); if(!cxxParserParseNextTokenSkipMacroParenthesis(&pParameterChain)) return false; } // This is used to avoid infinite recursion in substitution // (things like -D foo=foo or similar) static int iReplacementRecursionCount = 0; if(pMacro->replacements) { CXX_DEBUG_PRINT("The token has replacements: applying"); if(iReplacementRecursionCount < 1024) { // unget last char cppUngetc(g_cxx.iChar); // unget the replacement cxxParserParseNextTokenApplyReplacement( pMacro, pParameterChain ); g_cxx.iChar = cppGetc(); } } if(pParameterChain) cxxTokenDestroy(pParameterChain); iReplacementRecursionCount++; // Have no token to return: parse it CXX_DEBUG_PRINT("Parse inner token"); bool bRet = cxxParserParseNextToken(); CXX_DEBUG_PRINT("Parsed inner token: %s type %d",g_cxx.pToken->pszWord->buffer,g_cxx.pToken->eType); iReplacementRecursionCount--; return bRet; } } t->bFollowedBySpace = isspace(g_cxx.iChar); return true; } if(g_cxx.iChar == '-') { // special case for pointer vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); if(g_cxx.iChar == '>') { t->eType = CXXTokenTypePointerOperator; vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); } else { t->eType = CXXTokenTypeOperator; if(g_cxx.iChar == '-') { vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); } } t->bFollowedBySpace = isspace(g_cxx.iChar); return true; } #if 0 // As long as we use cppGetc() we don't need this if(g_cxx.iChar == '"') { // special case for strings t->eType = CXXTokenTypeStringConstant; vStringPut(t->pszWord,g_cxx.iChar); // We don't even care of storing the other chars: we don't need // them for parsing // FIXME: We might need them in signature:() tag.. maybe add // them up to a certain length only? for(;;) { g_cxx.iChar = cppGetc(); if(g_cxx.iChar == EOF) { t->bFollowedBySpace = false; return true; } if(g_cxx.iChar == '\\') { // escape g_cxx.iChar = cppGetc(); if(g_cxx.iChar == EOF) { t->bFollowedBySpace = false; return true; } } else if(g_cxx.iChar == '"') { g_cxx.iChar = cppGetc(); break; } } t->bFollowedBySpace = isspace(g_cxx.iChar); return true; } #else if(g_cxx.iChar == STRING_SYMBOL) { t->eType = CXXTokenTypeStringConstant; vStringPut(t->pszWord,'"'); vStringPut(t->pszWord,'"'); g_cxx.iChar = cppGetc(); t->bFollowedBySpace = isspace(g_cxx.iChar); return true; } #endif #if 0 // As long as we use cppGetc() we don't need this if(g_cxx.iChar == '\'') { // special case for strings t->eType = CXXTokenTypeCharacterConstant; vStringPut(t->pszWord,g_cxx.iChar); // We don't even care storing the other chars: we don't // need them for parsing for(;;) { g_cxx.iChar = cppGetc(); if(g_cxx.iChar == EOF) { t->bFollowedBySpace = false; return true; } if(g_cxx.iChar == '\\') { // escape g_cxx.iChar = cppGetc(); if(g_cxx.iChar == EOF) { t->bFollowedBySpace = false; return true; } } else if(g_cxx.iChar == '\'') { g_cxx.iChar = cppGetc(); break; } } t->bFollowedBySpace = isspace(g_cxx.iChar); return true; } #else if(g_cxx.iChar == CHAR_SYMBOL) { t->eType = CXXTokenTypeCharacterConstant; vStringPut(t->pszWord,'\''); vStringPut(t->pszWord,'\''); g_cxx.iChar = cppGetc(); t->bFollowedBySpace = isspace(g_cxx.iChar); return true; } #endif if(uInfo & CXXCharTypeDecimalDigit) { // number t->eType = CXXTokenTypeNumber; vStringPut(t->pszWord,g_cxx.iChar); for(;;) { g_cxx.iChar = cppGetc(); uInfo = UINFO(g_cxx.iChar); if(!(uInfo & CXXCharTypeValidInNumber)) break; vStringPut(t->pszWord,g_cxx.iChar); } t->bFollowedBySpace = isspace(g_cxx.iChar); return true; } if(uInfo & CXXCharTypeNamedSingleOrRepeatedCharToken) { t->eType = g_aCharTable[g_cxx.iChar].uSingleTokenType; vStringPut(t->pszWord,g_cxx.iChar); int iChar = g_cxx.iChar; g_cxx.iChar = cppGetc(); if(g_cxx.iChar == iChar) { t->eType = g_aCharTable[g_cxx.iChar].uMultiTokenType; // We could signal a syntax error with more than two colons // or equal signs...but we're tolerant do { vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); } while(g_cxx.iChar == iChar); } t->bFollowedBySpace = isspace(g_cxx.iChar); return true; } if(uInfo & CXXCharTypeNamedSingleOrOperatorToken) { t->eType = g_aCharTable[g_cxx.iChar].uSingleTokenType; vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); uInfo = UINFO(g_cxx.iChar); if(uInfo & (CXXCharTypeOperator | CXXCharTypeNamedSingleOrOperatorToken)) { t->eType = CXXTokenTypeOperator; do { vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); uInfo = UINFO(g_cxx.iChar); } while( uInfo & (CXXCharTypeOperator | CXXCharTypeNamedSingleOrOperatorToken) ); } t->bFollowedBySpace = isspace(g_cxx.iChar); return true; } if(uInfo & CXXCharTypeNamedSingleCharToken) { t->eType = g_aCharTable[g_cxx.iChar].uSingleTokenType; vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); t->bFollowedBySpace = isspace(g_cxx.iChar); return true; } if(uInfo & CXXCharTypeOperator) { t->eType = CXXTokenTypeOperator; vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); uInfo = UINFO(g_cxx.iChar); while(uInfo & CXXCharTypeOperator) { vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); uInfo = UINFO(g_cxx.iChar); } t->bFollowedBySpace = isspace(g_cxx.iChar); return true; } t->eType = CXXTokenTypeUnknown; vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); t->bFollowedBySpace = isspace(g_cxx.iChar); return true; }
CXXToken * cxxTokenChainExtractRangeFilterTypeName( CXXToken * from, CXXToken * to ) { if(!from) return NULL; CXXToken * pToken = from; for(;;) { if(!cxxTokenTypeIs(pToken,CXXTokenTypeKeyword)) break; if(!cxxKeywordExcludeFromTypeNames(pToken->eKeyword)) break; // must be excluded if(pToken == to) return NULL; // only excluded keywords pToken = pToken->pNext; if(!pToken) return NULL; // ... bug? } // Got at least one non-excluded keyword CXXToken * pRet = cxxTokenCreate(); pRet->iLineNumber = pToken->iLineNumber; pRet->oFilePosition = pToken->oFilePosition; pRet->eType = pToken->eType; cxxTokenAppendToString(pRet->pszWord,pToken); if(pToken->bFollowedBySpace) vStringPut (pRet->pszWord, ' '); pRet->bFollowedBySpace = pToken->bFollowedBySpace; while(pToken != to) { pToken = pToken->pNext; if(!pToken) return pRet; // ... bug? for(;;) { if(!cxxTokenTypeIs(pToken,CXXTokenTypeKeyword)) break; if(!cxxKeywordExcludeFromTypeNames(pToken->eKeyword)) break; // must be excluded if(pToken == to) return pRet; pToken = pToken->pNext; if(!pToken) return pRet; // ... bug? } cxxTokenAppendToString(pRet->pszWord,pToken); if(pToken->bFollowedBySpace) vStringPut (pRet->pszWord, ' '); pRet->bFollowedBySpace = pToken->bFollowedBySpace; } return pRet; }
boolean cxxParserParseNextToken(void) { CXXToken * t = cxxTokenCreate(); // The token chain should not be allowed to grow arbitrairly large. // The token structures are quite big and it's easy to grow up to // 5-6GB or memory usage. However this limit should be large enough // to accomodate all the reasonable statements that could have some // information in them. This includes multiple function prototypes // in a single statement (ImageMagick has some examples) but probably // does NOT include large data tables. if(g_cxx.pTokenChain->iCount > 16384) cxxTokenChainDestroyLast(g_cxx.pTokenChain); cxxTokenChainAppend(g_cxx.pTokenChain,t); g_cxx.pToken = t; cxxParserSkipToNonWhiteSpace(); // FIXME: this cpp handling is kind of broken: // it works only because the moon is in the correct phase. cppBeginStatement(); // This must be done after getting char from input t->iLineNumber = getInputLineNumber(); t->oFilePosition = getInputFilePosition(); if(g_cxx.iChar == EOF) { t->eType = CXXTokenTypeEOF; t->bFollowedBySpace = FALSE; return FALSE; } unsigned int uInfo = (g_cxx.iChar < 0x80) ? g_aCharTable[g_cxx.iChar].uType : 0; //printf("Char %c %02x info %u\n",g_cxx.iChar,g_cxx.iChar,uInfo); if(uInfo & CXXCharTypeStartOfIdentifier) { // word t->eType = CXXTokenTypeIdentifier; t->bFollowedBySpace = FALSE; vStringPut(t->pszWord,g_cxx.iChar); // special case for tile, which may actually be an operator if(g_cxx.iChar == '~') { // may be followed by space! g_cxx.iChar = cppGetc(); if(isspace(g_cxx.iChar)) { t->bFollowedBySpace = TRUE; g_cxx.iChar = cppGetc(); while(isspace(g_cxx.iChar)) g_cxx.iChar = cppGetc(); } // non space uInfo = (g_cxx.iChar < 0x80) ? g_aCharTable[g_cxx.iChar].uType : 0; if(!(uInfo & CXXCharTypeStartOfIdentifier)) { // this is not an identifier after all t->eType = CXXTokenTypeOperator; if((!t->bFollowedBySpace) && g_cxx.iChar == '=') { // make ~= single token so it's not handled as // a separate assignment vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); t->bFollowedBySpace = isspace(g_cxx.iChar); } return TRUE; } } else { g_cxx.iChar = cppGetc(); } for(;;) { uInfo = (g_cxx.iChar < 0x80) ? g_aCharTable[g_cxx.iChar].uType : 0; if(!(uInfo & CXXCharTypePartOfIdentifier)) break; vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); } int iCXXKeyword = lookupKeyword(t->pszWord->buffer,g_cxx.eLanguage); if(iCXXKeyword >= 0) { if( ( (iCXXKeyword == CXXKeywordFINAL) && (!g_cxx.bParsingClassStructOrUnionDeclaration) ) || ( ( (iCXXKeyword == CXXKeywordPUBLIC) || (iCXXKeyword == CXXKeywordPROTECTED) || (iCXXKeyword == CXXKeywordPRIVATE) ) && (!g_cxx.bEnablePublicProtectedPrivateKeywords) ) ) { t->eType = CXXTokenTypeIdentifier; } else { t->eType = CXXTokenTypeKeyword; t->eKeyword = (enum CXXKeyword)iCXXKeyword; if(iCXXKeyword == CXXKeyword__ATTRIBUTE__) { // special handling for __attribute__ return cxxParserParseNextTokenCondenseAttribute(); } } } else { boolean bIgnoreParens = FALSE; const char * szReplacement = NULL; if(isIgnoreToken( vStringValue(t->pszWord), &bIgnoreParens, &szReplacement )) { CXX_DEBUG_PRINT("Ignore token %s",vStringValue(t->pszWord)); // FIXME: Handle ignore parens! if(szReplacement && *szReplacement) { vStringClear(t->pszWord); vStringCatS(t->pszWord,szReplacement); } else { // skip cxxTokenChainDestroyLast(g_cxx.pTokenChain); return cxxParserParseNextToken(); } } } t->bFollowedBySpace = t->bFollowedBySpace | isspace(g_cxx.iChar); return TRUE; } if(g_cxx.iChar == '-') { // special case for pointer vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); if(g_cxx.iChar == '>') { t->eType = CXXTokenTypePointerOperator; vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); } else { t->eType = CXXTokenTypeOperator; if(g_cxx.iChar == '-') { vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); } } t->bFollowedBySpace = isspace(g_cxx.iChar); return TRUE; } #if 0 // As long as we use cppGetc() we don't need this if(g_cxx.iChar == '"') { // special case for strings t->eType = CXXTokenTypeStringConstant; vStringPut(t->pszWord,g_cxx.iChar); // We don't even care of storing the other chars: we don't need // them for parsing // FIXME: We might need them in signature:() tag.. maybe add // them up to a certain length only? for(;;) { g_cxx.iChar = cppGetc(); if(g_cxx.iChar == EOF) { t->bFollowedBySpace = FALSE; return TRUE; } if(g_cxx.iChar == '\\') { // escape g_cxx.iChar = cppGetc(); if(g_cxx.iChar == EOF) { t->bFollowedBySpace = FALSE; return TRUE; } } else if(g_cxx.iChar == '"') { g_cxx.iChar = cppGetc(); break; } } t->bFollowedBySpace = isspace(g_cxx.iChar); return TRUE; } #else if(g_cxx.iChar == STRING_SYMBOL) { t->eType = CXXTokenTypeStringConstant; vStringPut(t->pszWord,'"'); vStringPut(t->pszWord,'"'); g_cxx.iChar = cppGetc(); t->bFollowedBySpace = isspace(g_cxx.iChar); return TRUE; } #endif #if 0 // As long as we use cppGetc() we don't need this if(g_cxx.iChar == '\'') { // special case for strings t->eType = CXXTokenTypeCharacterConstant; vStringPut(t->pszWord,g_cxx.iChar); // We don't even care storing the other chars: we don't // need them for parsing for(;;) { g_cxx.iChar = cppGetc(); if(g_cxx.iChar == EOF) { t->bFollowedBySpace = FALSE; return TRUE; } if(g_cxx.iChar == '\\') { // escape g_cxx.iChar = cppGetc(); if(g_cxx.iChar == EOF) { t->bFollowedBySpace = FALSE; return TRUE; } } else if(g_cxx.iChar == '\'') { g_cxx.iChar = cppGetc(); break; } } t->bFollowedBySpace = isspace(g_cxx.iChar); return TRUE; } #else if(g_cxx.iChar == CHAR_SYMBOL) { t->eType = CXXTokenTypeCharacterConstant; vStringPut(t->pszWord,'\''); vStringPut(t->pszWord,'\''); g_cxx.iChar = cppGetc(); t->bFollowedBySpace = isspace(g_cxx.iChar); return TRUE; } #endif if(uInfo & CXXCharTypeDecimalDigit) { // number t->eType = CXXTokenTypeNumber; vStringPut(t->pszWord,g_cxx.iChar); for(;;) { g_cxx.iChar = cppGetc(); uInfo = (g_cxx.iChar < 0x80) ? g_aCharTable[g_cxx.iChar].uType : 0; if(!(uInfo & CXXCharTypeValidInNumber)) break; vStringPut(t->pszWord,g_cxx.iChar); } t->bFollowedBySpace = isspace(g_cxx.iChar); return TRUE; } if(uInfo & CXXCharTypeNamedSingleOrRepeatedCharToken) { t->eType = g_aCharTable[g_cxx.iChar].uSingleTokenType; vStringPut(t->pszWord,g_cxx.iChar); int iChar = g_cxx.iChar; g_cxx.iChar = cppGetc(); if(g_cxx.iChar == iChar) { t->eType = g_aCharTable[g_cxx.iChar].uMultiTokenType; // We could signal a syntax error with more than two colons // or equal signs...but we're tolerant do { vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); } while(g_cxx.iChar == iChar); } t->bFollowedBySpace = isspace(g_cxx.iChar); return TRUE; } if(uInfo & CXXCharTypeNamedSingleOrOperatorToken) { t->eType = g_aCharTable[g_cxx.iChar].uSingleTokenType; vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); uInfo = (g_cxx.iChar < 0x80) ? g_aCharTable[g_cxx.iChar].uType : 0; if(uInfo & (CXXCharTypeOperator | CXXCharTypeNamedSingleOrOperatorToken)) { t->eType = CXXTokenTypeOperator; do { vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); uInfo = (g_cxx.iChar < 0x80) ? g_aCharTable[g_cxx.iChar].uType : 0; } while( uInfo & (CXXCharTypeOperator | CXXCharTypeNamedSingleOrOperatorToken) ); } t->bFollowedBySpace = isspace(g_cxx.iChar); return TRUE; } if(uInfo & CXXCharTypeNamedSingleCharToken) { t->eType = g_aCharTable[g_cxx.iChar].uSingleTokenType; vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); t->bFollowedBySpace = isspace(g_cxx.iChar); return TRUE; } if(uInfo & CXXCharTypeOperator) { t->eType = CXXTokenTypeOperator; vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); uInfo = (g_cxx.iChar < 0x80) ? g_aCharTable[g_cxx.iChar].uType : 0; while(uInfo & CXXCharTypeOperator) { vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); uInfo = (g_cxx.iChar < 0x80) ? g_aCharTable[g_cxx.iChar].uType : 0; } t->bFollowedBySpace = isspace(g_cxx.iChar); return TRUE; } t->eType = CXXTokenTypeUnknown; vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); t->bFollowedBySpace = isspace(g_cxx.iChar); return TRUE; }
// In case of a lambda without parentheses this is the capture list token. boolean cxxParserHandleLambda(CXXToken * pParenthesis) { CXX_DEBUG_ENTER(); CXXToken * pIdentifier = cxxTokenCreateAnonymousIdentifier(CXXTagKindFUNCTION); CXXTokenChain * pSave = g_cxx.pTokenChain; CXXTokenChain * pNew = cxxTokenChainCreate(); g_cxx.pTokenChain = pNew; tagEntryInfo * tag = cxxTagBegin(CXXTagKindFUNCTION,pIdentifier); CXXToken * pAfterParenthesis = pParenthesis ? pParenthesis->pNext : NULL; CXXToken * pCaptureList = NULL; if(pParenthesis) { if(cxxTokenTypeIs(pParenthesis,CXXTokenTypeSquareParenthesisChain)) { // form (4) of lambda (see cxxParserOpeningBracketIsLambda()). pCaptureList = pParenthesis; } else if( pParenthesis->pPrev && cxxTokenTypeIs(pParenthesis->pPrev,CXXTokenTypeSquareParenthesisChain) ) { // other forms of lambda (see cxxParserOpeningBracketIsLambda()). pCaptureList = pParenthesis->pPrev; } } if( pAfterParenthesis && cxxTokenTypeIs(pAfterParenthesis,CXXTokenTypeKeyword) && (pAfterParenthesis->eKeyword == CXXKeywordCONST) ) pAfterParenthesis = pAfterParenthesis->pNext; CXXToken * pTypeStart = NULL; CXXToken * pTypeEnd; if( pAfterParenthesis && cxxTokenTypeIs(pAfterParenthesis,CXXTokenTypePointerOperator) && pAfterParenthesis->pNext && !cxxTokenTypeIs(pAfterParenthesis->pNext,CXXTokenTypeOpeningBracket) ) { pTypeStart = pAfterParenthesis->pNext; pTypeEnd = pTypeStart; while( pTypeEnd->pNext && (!cxxTokenTypeIs(pTypeEnd->pNext,CXXTokenTypeOpeningBracket)) ) pTypeEnd = pTypeEnd->pNext; while( (pTypeStart != pTypeEnd) && cxxTokenTypeIs(pTypeStart,CXXTokenTypeKeyword) && cxxKeywordExcludeFromTypeNames(pTypeStart->eKeyword) ) pTypeStart = pTypeStart->pNext; } int iCorkQueueIndex = CORK_NIL; if(tag) { tag->isFileScope = TRUE; CXXToken * pTypeName; if(pTypeStart) pTypeName = cxxTagSetTypeField(pTypeStart,pTypeEnd); else pTypeName = NULL; if(pCaptureList && cxxTagCPPFieldEnabled(CXXTagCPPFieldLambdaCaptureList)) { CXX_DEBUG_ASSERT(pCaptureList->pChain,"The capture list must be a chain"); cxxTokenChainCondense(pCaptureList->pChain,0); CXX_DEBUG_ASSERT( cxxTokenChainFirst(pCaptureList->pChain), "Condensation should have created a single token in the chain" ); cxxTagSetCPPField( CXXTagCPPFieldLambdaCaptureList, vStringValue(cxxTokenChainFirst(pCaptureList->pChain)->pszWord) ); } // FIXME: Properties? vString * pszSignature = NULL; if(cxxTokenTypeIs(pParenthesis,CXXTokenTypeParenthesisChain)) pszSignature = cxxTokenChainJoin(pParenthesis->pChain,NULL,0); if(pszSignature) tag->extensionFields.signature = vStringValue(pszSignature); iCorkQueueIndex = cxxTagCommit(); if(pTypeName) cxxTokenDestroy(pTypeName); if(pszSignature) vStringDelete(pszSignature); } cxxScopePush( pIdentifier, CXXTagKindFUNCTION, CXXScopeAccessUnknown ); if( pParenthesis && cxxTokenTypeIs(pParenthesis,CXXTokenTypeParenthesisChain) && cxxTagKindEnabled(CXXTagKindPARAMETER) ) { CXXFunctionParameterInfo oParamInfo; if(cxxParserTokenChainLooksLikeFunctionParameterList( pParenthesis->pChain,&oParamInfo )) cxxParserEmitFunctionParameterTags(&oParamInfo); } boolean bRet = cxxParserParseBlock(TRUE); if(iCorkQueueIndex > CORK_NIL) cxxParserMarkEndLineForTagInCorkQueue(iCorkQueueIndex); cxxScopePop(); pNew = g_cxx.pTokenChain; // May have been destroyed and re-created g_cxx.pTokenChain = pSave; g_cxx.pToken = pSave->pTail; // change the type of token so following parsing code is not confused too much g_cxx.pToken->eType = CXXTokenTypeAngleBracketChain; g_cxx.pToken->pChain = pNew; cxxTokenChainClear(pNew); CXXToken * t = cxxTokenCreate(); t->eType = CXXTokenTypeOpeningBracket; vStringCatS(t->pszWord,"{"); cxxTokenChainAppend(pNew,t); t = cxxTokenCreate(); t->eType = CXXTokenTypeClosingBracket; vStringCatS(t->pszWord,"}"); cxxTokenChainAppend(pNew,t); CXX_DEBUG_LEAVE(); return bRet; }
// // This is called after a full enum/struct/class/union declaration // that ends with a closing bracket. // static boolean cxxParserParseEnumStructClassOrUnionFullDeclarationTrailer( boolean bParsingTypedef, enum CXXKeyword eTagKeyword, enum CXXTagKind eTagKind, const char * szTypeName ) { CXX_DEBUG_ENTER(); cxxTokenChainClear(g_cxx.pTokenChain); CXX_DEBUG_PRINT( "Parse enum/struct/class/union trailer, typename is '%s'", szTypeName ); MIOPos oFilePosition = getInputFilePosition(); int iFileLine = getInputLineNumber(); if(!cxxParserParseUpToOneOf(CXXTokenTypeEOF | CXXTokenTypeSemicolon)) { CXX_DEBUG_LEAVE_TEXT("Failed to parse up to EOF/semicolon"); return FALSE; } if(cxxTokenTypeIs(g_cxx.pToken,CXXTokenTypeEOF)) { // It's a syntax error, but we can be tolerant here. CXX_DEBUG_LEAVE_TEXT("Got EOF after enum/class/struct/union block"); return TRUE; } if(g_cxx.pTokenChain->iCount < 2) { CXX_DEBUG_LEAVE_TEXT("Nothing interesting after enum/class/struct block"); return TRUE; } // fake the initial two tokens CXXToken * pIdentifier = cxxTokenCreate(); pIdentifier->oFilePosition = oFilePosition; pIdentifier->iLineNumber = iFileLine; pIdentifier->eType = CXXTokenTypeIdentifier; pIdentifier->bFollowedBySpace = TRUE; vStringCatS(pIdentifier->pszWord,szTypeName); cxxTokenChainPrepend(g_cxx.pTokenChain,pIdentifier); CXXToken * pKeyword = cxxTokenCreate(); pKeyword->oFilePosition = oFilePosition; pKeyword->iLineNumber = iFileLine; pKeyword->eType = CXXTokenTypeKeyword; pKeyword->eKeyword = eTagKeyword; pKeyword->bFollowedBySpace = TRUE; vStringCatS(pKeyword->pszWord,cxxTagGetKindOptions()[eTagKind].name); cxxTokenChainPrepend(g_cxx.pTokenChain,pKeyword); if(bParsingTypedef) cxxParserExtractTypedef(g_cxx.pTokenChain,TRUE); else cxxParserExtractVariableDeclarations(g_cxx.pTokenChain,0); CXX_DEBUG_LEAVE(); return TRUE; }