/** * Parse a method starting with the -/+. Return it in the method vString */ static void getSingleObjCMethod(vString *method) { int z; int skipNextIdent; const char *temp; vStringClear(method); z = cppGetc(); vStringPut(method, z); skipNextIdent = 0; while ((z = cppGetc()) != EOF && z != ';' && z != '{') { if (isspace(z)) continue; else if (z == '(') { cppUngetc(z); readToMatchingBrace(&z); } else if (z == ':') { vStringPut(method, z); skipNextIdent = 1; } else if (myIsIdentifier(z, 0)) { cppUngetc(z); temp = readToNonIdentifier(0); if (skipNextIdent) { skipNextIdent = 0; } else { vStringCatS(method, temp); } } } cppUngetc(z); vStringPut(method, 0); }
static void nextToken (void) { int c; repeat: /* * .proto files may contain C and C++ style comments and * quoted strings. cppGetc() takes care of them. */ c = cppGetc (); token.keyword = KEYWORD_NONE; if (c <= 0) token.type = TOKEN_EOF; else if (c == '{' || c == '}' || c == ';' || c == '.' || c == '=') token.type = c; else if (isalnum (c) || c == '_') { token.type = TOKEN_ID; vStringClear (token.value); while (c > 0 && (isalnum (c) || c == '_')) { vStringPut (token.value, c); c = cppGetc (); } token.keyword = lookupCaseKeyword (vStringValue (token.value), Lang_protobuf); cppUngetc (c); } else goto repeat; /* anything else is not important for this parser */ }
/** * This does a LOT of our parsing, you pass it a function * that returns false when it should stop and this takes * care of the vString stuff and checking for EOF etc.. */ static char *readToFalse(int (*shouldcontinue)(int c, int pos), int *ender) { static vString *wordBuffer = 0; int len = 0; int z; if (!wordBuffer) { wordBuffer = vStringNew(); } else { vStringClear(wordBuffer); } while ((z = cppGetc()) != EOF && shouldcontinue(z, len)) { vStringPut(wordBuffer, z); len++; } cppUngetc(z); if (ender) *ender = z; vStringPut(wordBuffer, 0); return vStringValue(wordBuffer); }
bool cxxParserParseNextToken(void) { CXXToken * t = cxxTokenCreate(); // The token chain should not be allowed to grow arbitrairly large. // The token structures are quite big and it's easy to grow up to // 5-6GB or memory usage. However this limit should be large enough // to accomodate all the reasonable statements that could have some // information in them. This includes multiple function prototypes // in a single statement (ImageMagick has some examples) but probably // does NOT include large data tables. if(g_cxx.pTokenChain->iCount > 16384) cxxTokenChainDestroyLast(g_cxx.pTokenChain); cxxTokenChainAppend(g_cxx.pTokenChain,t); g_cxx.pToken = t; cxxParserSkipToNonWhiteSpace(); // FIXME: this cpp handling is kind of broken: // it works only because the moon is in the correct phase. cppBeginStatement(); // This must be done after getting char from input t->iLineNumber = getInputLineNumber(); t->oFilePosition = getInputFilePosition(); if(g_cxx.iChar == EOF) { t->eType = CXXTokenTypeEOF; t->bFollowedBySpace = false; return false; } unsigned int uInfo = UINFO(g_cxx.iChar); //fprintf(stderr,"Char %c %02x info %u\n",g_cxx.iChar,g_cxx.iChar,uInfo); if(uInfo & CXXCharTypeStartOfIdentifier) { // word t->eType = CXXTokenTypeIdentifier; t->bFollowedBySpace = false; vStringPut(t->pszWord,g_cxx.iChar); // special case for tile, which may actually be an operator if(g_cxx.iChar == '~') { // may be followed by space! g_cxx.iChar = cppGetc(); if(isspace(g_cxx.iChar)) { t->bFollowedBySpace = true; g_cxx.iChar = cppGetc(); while(isspace(g_cxx.iChar)) g_cxx.iChar = cppGetc(); } // non space uInfo = UINFO(g_cxx.iChar); if(!(uInfo & CXXCharTypeStartOfIdentifier)) { // this is not an identifier after all t->eType = CXXTokenTypeOperator; if((!t->bFollowedBySpace) && g_cxx.iChar == '=') { // make ~= single token so it's not handled as // a separate assignment vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); t->bFollowedBySpace = isspace(g_cxx.iChar); } return true; } } else { g_cxx.iChar = cppGetc(); } for(;;) { uInfo = UINFO(g_cxx.iChar); if(!(uInfo & CXXCharTypePartOfIdentifier)) break; vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); } int iCXXKeyword = lookupKeyword(t->pszWord->buffer,g_cxx.eLanguage); if(iCXXKeyword >= 0) { if( ( (iCXXKeyword == CXXKeywordFINAL) && (!g_cxx.bParsingClassStructOrUnionDeclaration) ) || ( ( (iCXXKeyword == CXXKeywordPUBLIC) || (iCXXKeyword == CXXKeywordPROTECTED) || (iCXXKeyword == CXXKeywordPRIVATE) ) && (!g_cxx.bEnablePublicProtectedPrivateKeywords) ) ) { t->eType = CXXTokenTypeIdentifier; } else { t->eType = CXXTokenTypeKeyword; t->eKeyword = (enum CXXKeyword)iCXXKeyword; if(iCXXKeyword == CXXKeyword__ATTRIBUTE__) { // special handling for __attribute__ return cxxParserParseNextTokenCondenseAttribute(); } } } else { const cppMacroInfo * pMacro = cppFindMacro(vStringValue(t->pszWord)); if(pMacro) { CXX_DEBUG_PRINT("Macro %s",vStringValue(t->pszWord)); cxxTokenChainDestroyLast(g_cxx.pTokenChain); CXXToken * pParameterChain = NULL; if(pMacro->hasParameterList) { CXX_DEBUG_PRINT("Macro has parameter list"); if(!cxxParserParseNextTokenSkipMacroParenthesis(&pParameterChain)) return false; } // This is used to avoid infinite recursion in substitution // (things like -D foo=foo or similar) static int iReplacementRecursionCount = 0; if(pMacro->replacements) { CXX_DEBUG_PRINT("The token has replacements: applying"); if(iReplacementRecursionCount < 1024) { // unget last char cppUngetc(g_cxx.iChar); // unget the replacement cxxParserParseNextTokenApplyReplacement( pMacro, pParameterChain ); g_cxx.iChar = cppGetc(); } } if(pParameterChain) cxxTokenDestroy(pParameterChain); iReplacementRecursionCount++; // Have no token to return: parse it CXX_DEBUG_PRINT("Parse inner token"); bool bRet = cxxParserParseNextToken(); CXX_DEBUG_PRINT("Parsed inner token: %s type %d",g_cxx.pToken->pszWord->buffer,g_cxx.pToken->eType); iReplacementRecursionCount--; return bRet; } } t->bFollowedBySpace = isspace(g_cxx.iChar); return true; } if(g_cxx.iChar == '-') { // special case for pointer vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); if(g_cxx.iChar == '>') { t->eType = CXXTokenTypePointerOperator; vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); } else { t->eType = CXXTokenTypeOperator; if(g_cxx.iChar == '-') { vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); } } t->bFollowedBySpace = isspace(g_cxx.iChar); return true; } #if 0 // As long as we use cppGetc() we don't need this if(g_cxx.iChar == '"') { // special case for strings t->eType = CXXTokenTypeStringConstant; vStringPut(t->pszWord,g_cxx.iChar); // We don't even care of storing the other chars: we don't need // them for parsing // FIXME: We might need them in signature:() tag.. maybe add // them up to a certain length only? for(;;) { g_cxx.iChar = cppGetc(); if(g_cxx.iChar == EOF) { t->bFollowedBySpace = false; return true; } if(g_cxx.iChar == '\\') { // escape g_cxx.iChar = cppGetc(); if(g_cxx.iChar == EOF) { t->bFollowedBySpace = false; return true; } } else if(g_cxx.iChar == '"') { g_cxx.iChar = cppGetc(); break; } } t->bFollowedBySpace = isspace(g_cxx.iChar); return true; } #else if(g_cxx.iChar == STRING_SYMBOL) { t->eType = CXXTokenTypeStringConstant; vStringPut(t->pszWord,'"'); vStringPut(t->pszWord,'"'); g_cxx.iChar = cppGetc(); t->bFollowedBySpace = isspace(g_cxx.iChar); return true; } #endif #if 0 // As long as we use cppGetc() we don't need this if(g_cxx.iChar == '\'') { // special case for strings t->eType = CXXTokenTypeCharacterConstant; vStringPut(t->pszWord,g_cxx.iChar); // We don't even care storing the other chars: we don't // need them for parsing for(;;) { g_cxx.iChar = cppGetc(); if(g_cxx.iChar == EOF) { t->bFollowedBySpace = false; return true; } if(g_cxx.iChar == '\\') { // escape g_cxx.iChar = cppGetc(); if(g_cxx.iChar == EOF) { t->bFollowedBySpace = false; return true; } } else if(g_cxx.iChar == '\'') { g_cxx.iChar = cppGetc(); break; } } t->bFollowedBySpace = isspace(g_cxx.iChar); return true; } #else if(g_cxx.iChar == CHAR_SYMBOL) { t->eType = CXXTokenTypeCharacterConstant; vStringPut(t->pszWord,'\''); vStringPut(t->pszWord,'\''); g_cxx.iChar = cppGetc(); t->bFollowedBySpace = isspace(g_cxx.iChar); return true; } #endif if(uInfo & CXXCharTypeDecimalDigit) { // number t->eType = CXXTokenTypeNumber; vStringPut(t->pszWord,g_cxx.iChar); for(;;) { g_cxx.iChar = cppGetc(); uInfo = UINFO(g_cxx.iChar); if(!(uInfo & CXXCharTypeValidInNumber)) break; vStringPut(t->pszWord,g_cxx.iChar); } t->bFollowedBySpace = isspace(g_cxx.iChar); return true; } if(uInfo & CXXCharTypeNamedSingleOrRepeatedCharToken) { t->eType = g_aCharTable[g_cxx.iChar].uSingleTokenType; vStringPut(t->pszWord,g_cxx.iChar); int iChar = g_cxx.iChar; g_cxx.iChar = cppGetc(); if(g_cxx.iChar == iChar) { t->eType = g_aCharTable[g_cxx.iChar].uMultiTokenType; // We could signal a syntax error with more than two colons // or equal signs...but we're tolerant do { vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); } while(g_cxx.iChar == iChar); } t->bFollowedBySpace = isspace(g_cxx.iChar); return true; } if(uInfo & CXXCharTypeNamedSingleOrOperatorToken) { t->eType = g_aCharTable[g_cxx.iChar].uSingleTokenType; vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); uInfo = UINFO(g_cxx.iChar); if(uInfo & (CXXCharTypeOperator | CXXCharTypeNamedSingleOrOperatorToken)) { t->eType = CXXTokenTypeOperator; do { vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); uInfo = UINFO(g_cxx.iChar); } while( uInfo & (CXXCharTypeOperator | CXXCharTypeNamedSingleOrOperatorToken) ); } t->bFollowedBySpace = isspace(g_cxx.iChar); return true; } if(uInfo & CXXCharTypeNamedSingleCharToken) { t->eType = g_aCharTable[g_cxx.iChar].uSingleTokenType; vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); t->bFollowedBySpace = isspace(g_cxx.iChar); return true; } if(uInfo & CXXCharTypeOperator) { t->eType = CXXTokenTypeOperator; vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); uInfo = UINFO(g_cxx.iChar); while(uInfo & CXXCharTypeOperator) { vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); uInfo = UINFO(g_cxx.iChar); } t->bFollowedBySpace = isspace(g_cxx.iChar); return true; } t->eType = CXXTokenTypeUnknown; vStringPut(t->pszWord,g_cxx.iChar); g_cxx.iChar = cppGetc(); t->bFollowedBySpace = isspace(g_cxx.iChar); return true; }