static bool readTagContent (tokenInfo *token, vString *text, long *line, long *lineOffset, int depth)
{
tokenType type;
readTokenText (token, text != NULL);
appendText (text, token->string);
do
{
*line = getInputLineNumber ();
*lineOffset = getInputLineOffset ();
readToken (token, false);
type = token->type;
if (type == TOKEN_TAG_START)
readTag (token, text, depth + 1);
if (type == TOKEN_COMMENT || type == TOKEN_TAG_START)
{
readTokenText (token, text != NULL);
appendText (text, token->string);
}
}
while (type == TOKEN_COMMENT || type == TOKEN_TAG_START);
return type == TOKEN_TAG_START2;
}
CXXToken * cxxTokenCreateAnonymousIdentifier(enum CXXTagKind k)
{
g_uNextAnonumousIdentiferId++;
char szNum[32];
CXXToken * t = cxxTokenCreate();
t->eType = CXXTokenTypeIdentifier;
vStringCopyS(t->pszWord, "__anon");
// The names of anonymous structures depend on the name of
// the file being processed so the identifiers won't collide across
// multiple ctags runs with different file sets.
unsigned int uHash = hash((const unsigned char *)getInputFileName());
sprintf(szNum,"%08x%02x%02x",uHash,g_uNextAnonumousIdentiferId, k);
vStringCatS(t->pszWord,szNum);
t->bFollowedBySpace = TRUE;
t->iLineNumber = getInputLineNumber();
t->oFilePosition = getInputFilePosition();
return t;
}
static NestingLevel *getNestingLevel(const int kind)
{
NestingLevel *nl;
tagEntryInfo *e;
int d = 0;
if (kind > K_EOF)
{
d++;
/* 1. we want the line before the '---' underline chars */
d++;
/* 2. we want the line before the next section/chapter title. */
}
while (1)
{
nl = nestingLevelsGetCurrent(nestingLevels);
e = getEntryOfNestingLevel (nl);
if ((nl && (e == NULL)) || (e && (e->kind - RstKinds) >= kind))
{
if (e)
e->extensionFields.endLine = (getInputLineNumber() - d);
nestingLevelsPop(nestingLevels);
}
else
break;
}
return nl;
}
void tracePrintPrefix(const char * szFunction)
{
if (!isTraced())
return;
debugIndent();
fprintf(stderr,"[%s][at %lu] ",szFunction,getInputLineNumber());
}
static tokenInfo *newToken (void)
{
tokenInfo *const token = xMalloc (1, tokenInfo);
token->type = TOKEN_NONE;
token->keyword = KEYWORD_NONE;
token->string = vStringNew ();
token->lineNumber = getInputLineNumber ();
token->filePosition = getInputFilePosition ();
return token;
}
CXXToken * cxxTokenCreateAnonymousIdentifier(enum CXXTagKind k)
{
CXXToken * t = cxxTokenCreate();
anonGenerate (t->pszWord, "__anon", k);
t->bFollowedBySpace = TRUE;
t->iLineNumber = getInputLineNumber();
t->oFilePosition = getInputFilePosition();
return t;
}
void tracePrint(const char * szFunction,const char * szFormat,...)
{
debugIndent();
fprintf(stderr,"[%s][at %lu] ",szFunction,getInputLineNumber());
va_list va;
va_start(va,szFormat);
vfprintf(stderr,szFormat,va);
va_end(va);
fprintf(stderr,"\n");
}
//
// Attach the current position of input file as "end" field of
// the specified tag in the cork queue
//
void cxxParserMarkEndLineForTagInCorkQueue(int iCorkQueueIndex)
{
CXX_DEBUG_ASSERT(iCorkQueueIndex > CORK_NIL,"The cork queue index is not valid");
char buf[16];
if(cxxParserCurrentLanguageIsCPP())
{
if(!cxxTagCPPFieldEnabled(CXXTagCPPFieldEndLine))
return;
sprintf(buf,"%ld",getInputLineNumber());
cxxTagSetCorkQueueCPPField(iCorkQueueIndex,CXXTagCPPFieldEndLine,buf);
return;
}
if(!cxxTagCFieldEnabled(CXXTagCFieldEndLine))
return;
sprintf(buf,"%ld",getInputLineNumber());
cxxTagSetCorkQueueCField(iCorkQueueIndex,CXXTagCFieldEndLine,buf);
}
static tokenInfo *newToken (void)
{
tokenInfo *const token = xMalloc (1, tokenInfo);
token->type = TOKEN_UNDEFINED;
token->scopeKind = TAG_NONE;
token->string = vStringNew ();
token->scope = vStringNew ();
token->lineNumber = getInputLineNumber ();
token->filePosition = getInputFilePosition ();
return token;
}
static void clearToken (void *data)
{
tokenInfo *token = data;
if (token->klass->clear)
token->klass->clear (token);
token->type = token->klass->typeForKeyword;
token->keyword = token->klass->keywordNone;
vStringClear (token->string);
token->lineNumber = getInputLineNumber ();
token->filePosition = getInputFilePosition ();
}
static void makeFunctionTag (vString * const name, const char *dbp, int scope_index)
{
functionName (name, dbp);
if (vStringLength (name) > 0 && ClojureKinds[K_FUNCTION].enabled)
{
tagEntryInfo e;
initTagEntry (&e, vStringValue (name), &(ClojureKinds[K_FUNCTION]));
e.lineNumber = getInputLineNumber ();
e.filePosition = getInputFilePosition ();
e.extensionFields.scopeIndex = scope_index;
makeTagEntry (&e);
}
}
static int makeNamespaceTag (vString * const name, const char *dbp)
{
functionName (name, dbp);
if (vStringLength (name) > 0 && ClojureKinds[K_NAMESPACE].enabled)
{
tagEntryInfo e;
initTagEntry (&e, vStringValue (name), &(ClojureKinds[K_NAMESPACE]));
e.lineNumber = getInputLineNumber ();
e.filePosition = getInputFilePosition ();
return makeTagEntry (&e);
}
else
return CORK_NIL;
}
static tokenInfo *newToken (void)
{
tokenInfo *const token = xMalloc (1, tokenInfo);
token->type = TOKEN_UNDEFINED;
token->keyword = KEYWORD_NONE;
token->string = vStringNew ();
token->scope = vStringNew ();
token->nestLevel = 0;
token->ignoreTag = FALSE;
token->lineNumber = getInputLineNumber ();
token->filePosition = getInputFilePosition ();
return token;
}
static bool readIdentifier (tokenInfo *const token, int c)
{
vStringClear (token->name);
if (isIdentifierCharacter (c))
{
while (isIdentifierCharacter (c))
{
vStringPut (token->name, c);
c = vGetc ();
}
vUngetc (c);
token->lineNumber = getInputLineNumber ();
token->filePosition = getInputFilePosition ();
}
return (bool)(vStringLength (token->name) > 0);
}
static tokenInfo *newToken (void)
{
tokenInfo *const token = xMalloc (1, tokenInfo);
token->kind = K_UNDEFINED;
token->name = vStringNew ();
token->lineNumber = getInputLineNumber ();
token->filePosition = getInputFilePosition ();
token->scope = NULL;
token->nestLevel = 0;
token->lastKind = K_UNDEFINED;
token->blockName = vStringNew ();
token->inheritance = vStringNew ();
token->prototype = false;
token->classScope = false;
return token;
}
extern void debugAssert (const char *assertion, const char *file, unsigned int line, const char *function)
{
fprintf(stderr, "ctags: %s:%u: %s%sAssertion `%s' failed.\n",
file, line,
function ? function : "", function ? ": " : "",
assertion);
if (getInputFileName())
{
fprintf(stderr, "ctags: %s:%u: parsing %s:%lu as %s\n",
file, line,
getInputFileName(), getInputLineNumber(),
getInputLanguageName());
}
fflush(stderr);
abort();
}
static void matchTagPattern (const vString* const line,
const regexPattern* const patbuf,
const regmatch_t* const pmatch)
{
vString *const name = substitute (vStringValue (line),
patbuf->u.tag.name_pattern, BACK_REFERENCE_COUNT, pmatch);
vStringStripLeading (name);
vStringStripTrailing (name);
if (vStringLength (name) > 0)
makeRegexTag (name, &patbuf->u.tag.kind);
else
error (WARNING, "%s:%ld: null expansion of name pattern \"%s\"",
getInputFileName (), getInputLineNumber (),
patbuf->u.tag.name_pattern);
vStringDelete (name);
}
void traceLeave(const char * szFunction,const char * szFormat,...)
{
if (!isTraced())
return;
debugDec();
debugIndent ();
fprintf(stderr,"[<< %s][at %lu] ",szFunction,getInputLineNumber());
va_list va;
va_start(va,szFormat);
vfprintf(stderr,szFormat,va);
va_end(va);
fprintf(stderr,"\n");
}
static void found_macro_cb (const char *line,
const regexMatch *matches,
unsigned int count,
void *uesrData)
{
struct macro_cb_data *data = uesrData;
if (count > 0)
{
vString *signature = ((count > 1) && (matches[2].length > 0))? vStringNew(): NULL;
vString *name = vStringNew ();
tagEntryInfo tag;
if (signature)
vStringNCopyS (signature, line + matches[2].start, matches[2].length);
vStringNCopyS (name, line + matches[1].start, matches[1].length);
if (data->rindex == ROLE_INDEX_DEFINITION)
initTagEntry (&tag, vStringValue (name), &(RpmSpecKinds[data->kindex]));
else
initRefTagEntry (&tag, vStringValue (name), &(RpmSpecKinds[data->kindex]), data->rindex);
if (signature)
tag.extensionFields.signature = vStringValue (signature);
/* Skip the definition */
while (line && is_line_continued (line))
{
rejecting = TRUE;
line = (const char *)readLineFromInputFile ();
}
rejecting = FALSE;
tag.extensionFields.endLine = getInputLineNumber();
makeTagEntry (&tag);
vStringDelete (name);
if (signature)
vStringDelete (signature);
}
}
extern void makeFileTag (const char *const fileName)
{
boolean via_line_directive = (strcmp (fileName, getInputFileName()) != 0);
xtagType xtag = XTAG_UNKNOWN;
if (isXtagEnabled(XTAG_FILE_NAMES))
xtag = XTAG_FILE_NAMES;
if (isXtagEnabled(XTAG_FILE_NAMES_WITH_TOTAL_LINES))
xtag = XTAG_FILE_NAMES_WITH_TOTAL_LINES;
if (xtag != XTAG_UNKNOWN)
{
tagEntryInfo tag;
kindOption *kind;
kind = getInputLanguageFileKind();
Assert (kind);
kind->enabled = isXtagEnabled(XTAG_FILE_NAMES);
/* TODO: you can return here if enabled == FALSE. */
initTagEntry (&tag, baseFilename (fileName), kind);
tag.isFileEntry = TRUE;
tag.lineNumberEntry = TRUE;
markTagExtraBit (&tag, xtag);
if (via_line_directive || (!isXtagEnabled(XTAG_FILE_NAMES_WITH_TOTAL_LINES)))
{
tag.lineNumber = 1;
}
else
{
while (readLineFromInputFile () != NULL)
; /* Do nothing */
tag.lineNumber = getInputLineNumber ();
}
makeTagEntry (&tag);
}
}
static void skipToMatch (const char *const pair)
{
const int begin = pair [0], end = pair [1];
const unsigned long inputLineNumber = getInputLineNumber ();
int matchLevel = 1;
int c = '\0';
while (matchLevel > 0)
{
c = nextChar ();
if (c == begin)
++matchLevel;
else if (c == end)
--matchLevel;
else if (c == '\n')
break;
}
if (c == EOF)
verbose ("%s: failed to find match for '%c' at line %lu\n",
getInputFileName (), begin, inputLineNumber);
}
static bool skipScriptContent (tokenInfo *token, long *line, long *lineOffset)
{
bool found_start = false;
bool found_script = false;
long line_tmp[2];
long lineOffset_tmp[2];
tokenType type;
do
{
line_tmp[0] = getInputLineNumber ();
lineOffset_tmp[0] = getInputLineOffset ();
readToken (token, false);
type = token->type;
if (type == TOKEN_TAG_START2)
{
found_start = true;
line_tmp[1] = line_tmp[0];
lineOffset_tmp[1] = lineOffset_tmp[0];
}
else if (found_start
&& type == TOKEN_NAME
&& lookupKeyword (vStringValue (token->string), Lang_html) == KEYWORD_script)
{
found_script = true;
*line = line_tmp[1];
*lineOffset = lineOffset_tmp[1];
}
else
found_start = false;
}
while ((type != TOKEN_EOF) && (!found_script));
return found_script;
}
static void readToken (tokenInfo *const token)
{
int c;
token->type = TOKEN_UNDEFINED;
token->keyword = KEYWORD_NONE;
vStringClear (token->string);
getNextChar:
if (! InPhp)
{
c = findPhpStart ();
if (c != EOF)
InPhp = TRUE;
}
else
c = getcFromInputFile ();
c = skipWhitespaces (c);
token->lineNumber = getInputLineNumber ();
token->filePosition = getInputFilePosition ();
switch (c)
{
case EOF: token->type = TOKEN_EOF; break;
case '(': token->type = TOKEN_OPEN_PAREN; break;
case ')': token->type = TOKEN_CLOSE_PAREN; break;
case ';': token->type = TOKEN_SEMICOLON; break;
case ',': token->type = TOKEN_COMMA; break;
case '.': token->type = TOKEN_PERIOD; break;
case ':': token->type = TOKEN_COLON; break;
case '{': token->type = TOKEN_OPEN_CURLY; break;
case '}': token->type = TOKEN_CLOSE_CURLY; break;
case '[': token->type = TOKEN_OPEN_SQUARE; break;
case ']': token->type = TOKEN_CLOSE_SQUARE; break;
case '&': token->type = TOKEN_AMPERSAND; break;
case '\\': token->type = TOKEN_BACKSLASH; break;
case '=':
{
int d = getcFromInputFile ();
if (d == '=' || d == '>')
token->type = TOKEN_OPERATOR;
else
{
ungetcToInputFile (d);
token->type = TOKEN_EQUAL_SIGN;
}
break;
}
case '\'':
case '"':
token->type = TOKEN_STRING;
parseString (token->string, c);
token->lineNumber = getInputLineNumber ();
token->filePosition = getInputFilePosition ();
break;
case '<':
{
int d = getcFromInputFile ();
if (d == '/')
{
/* </script[:white:]*> */
if (tolower ((d = getcFromInputFile ())) == 's' &&
tolower ((d = getcFromInputFile ())) == 'c' &&
tolower ((d = getcFromInputFile ())) == 'r' &&
tolower ((d = getcFromInputFile ())) == 'i' &&
tolower ((d = getcFromInputFile ())) == 'p' &&
tolower ((d = getcFromInputFile ())) == 't' &&
(d = skipWhitespaces (getcFromInputFile ())) == '>')
{
InPhp = FALSE;
goto getNextChar;
}
else
{
ungetcToInputFile (d);
token->type = TOKEN_UNDEFINED;
}
}
else if (d == '<' && (d = getcFromInputFile ()) == '<')
{
token->type = TOKEN_STRING;
parseHeredoc (token->string);
}
else
{
ungetcToInputFile (d);
token->type = TOKEN_UNDEFINED;
}
break;
}
case '#': /* comment */
skipSingleComment ();
goto getNextChar;
break;
case '+':
case '-':
case '*':
case '%':
{
int d = getcFromInputFile ();
if (d != '=' && ! (c == '-' && d == '>'))
ungetcToInputFile (d);
token->type = TOKEN_OPERATOR;
break;
}
case '/': /* division or comment start */
{
int d = getcFromInputFile ();
if (d == '/') /* single-line comment */
{
skipSingleComment ();
goto getNextChar;
}
else if (d == '*')
{
do
{
c = skipToCharacter ('*');
if (c != EOF)
{
c = getcFromInputFile ();
if (c == '/')
break;
else
ungetcToInputFile (c);
}
} while (c != EOF && c != '\0');
goto getNextChar;
}
else
{
if (d != '=')
ungetcToInputFile (d);
token->type = TOKEN_OPERATOR;
}
break;
}
case '$': /* variable start */
{
int d = getcFromInputFile ();
if (! isIdentChar (d))
{
ungetcToInputFile (d);
token->type = TOKEN_UNDEFINED;
}
else
{
parseIdentifier (token->string, d);
token->type = TOKEN_VARIABLE;
}
break;
}
case '?': /* maybe the end of the PHP chunk */
{
int d = getcFromInputFile ();
if (d == '>')
{
InPhp = FALSE;
goto getNextChar;
}
else
{
ungetcToInputFile (d);
token->type = TOKEN_UNDEFINED;
}
break;
}
default:
if (! isIdentChar (c))
token->type = TOKEN_UNDEFINED;
else
{
parseIdentifier (token->string, c);
token->keyword = analyzeToken (token->string, getInputLanguage ());
if (token->keyword == KEYWORD_NONE)
token->type = TOKEN_IDENTIFIER;
else
token->type = TOKEN_KEYWORD;
}
break;
}
if (token->type == TOKEN_SEMICOLON ||
token->type == TOKEN_OPEN_CURLY ||
token->type == TOKEN_CLOSE_CURLY)
{
/* reset current statement details on statement end, and when entering
* a deeper scope.
* it is a bit ugly to do this in readToken(), but it makes everything
* a lot simpler. */
CurrentStatement.access = ACCESS_UNDEFINED;
CurrentStatement.impl = IMPL_UNDEFINED;
}
}
/* Algorithm adapted from from GNU etags.
* Perl support by Bart Robinson <*****@*****.**>
* Perl sub names: look for /^ [ \t\n]sub [ \t\n]+ [^ \t\n{ (]+/
*/
static void findPerlTags (void)
{
vString *name = vStringNew ();
vString *package = NULL;
bool skipPodDoc = false;
const unsigned char *line;
unsigned long podStart = 0UL;
/* Core modules AutoLoader and SelfLoader support delayed compilation
* by allowing Perl code that follows __END__ and __DATA__ tokens,
* respectively. When we detect that one of these modules is used
* in the file, we continue processing even after we see the
* corresponding token that would usually terminate parsing of the
* file.
*/
enum {
RESPECT_END = (1 << 0),
RESPECT_DATA = (1 << 1),
} respect_token = RESPECT_END | RESPECT_DATA;
while ((line = readLineFromInputFile ()) != NULL)
{
bool spaceRequired = false;
bool qualified = false;
const unsigned char *cp = line;
perlKind kind = K_NONE;
tagEntryInfo e;
if (skipPodDoc)
{
if (strncmp ((const char*) line, "=cut", (size_t) 4) == 0)
{
skipPodDoc = false;
if (podStart != 0UL)
{
makePromise ("Pod",
podStart, 0,
getInputLineNumber(), 0,
getSourceLineNumber());
podStart = 0UL;
}
}
continue;
}
else if (line [0] == '=')
{
skipPodDoc = isPodWord ((const char*)line + 1);
if (skipPodDoc)
podStart = getSourceLineNumber ();
continue;
}
else if (strcmp ((const char*) line, "__DATA__") == 0)
{
if (respect_token & RESPECT_DATA)
break;
else
continue;
}
else if (strcmp ((const char*) line, "__END__") == 0)
{
if (respect_token & RESPECT_END)
break;
else
continue;
}
else if (line [0] == '#')
continue;
while (isspace (*cp))
cp++;
if (strncmp((const char*) cp, "sub", (size_t) 3) == 0)
{
TRACE("this looks like a sub\n");
cp += 3;
kind = K_SUBROUTINE;
spaceRequired = true;
qualified = true;
}
else if (strncmp((const char*) cp, "use", (size_t) 3) == 0)
{
cp += 3;
if (!isspace(*cp))
continue;
while (*cp && isspace (*cp))
++cp;
if (strncmp((const char*) cp, "AutoLoader", (size_t) 10) == 0) {
respect_token &= ~RESPECT_END;
continue;
}
if (strncmp((const char*) cp, "SelfLoader", (size_t) 10) == 0) {
respect_token &= ~RESPECT_DATA;
continue;
}
if (strncmp((const char*) cp, "constant", (size_t) 8) != 0)
continue;
cp += 8;
/* Skip up to the first non-space character, skipping empty
* and comment lines.
*/
while (isspace(*cp))
cp++;
while (!*cp || '#' == *cp) {
cp = readLineFromInputFile ();
if (!cp)
goto END_MAIN_WHILE;
while (isspace (*cp))
cp++;
}
if ('{' == *cp) {
++cp;
if (0 == parseConstantsFromHashRef(cp, name, package)) {
vStringClear(name);
continue;
} else
goto END_MAIN_WHILE;
}
kind = K_CONSTANT;
spaceRequired = false;
qualified = true;
}
else if (strncmp((const char*) cp, "package", (size_t) 7) == 0 &&
('\0' == cp[7] || isspace(cp[7])))
{
cp += 7;
while (isspace (*cp))
cp++;
while (!*cp || '#' == *cp) {
cp = readLineFromInputFile ();
if (!cp)
goto END_MAIN_WHILE;
while (isspace (*cp))
cp++;
}
if (package == NULL)
package = vStringNew ();
else
vStringClear (package);
const unsigned char *const first = cp;
while (*cp && (int) *cp != ';' && !isspace ((int) *cp))
{
vStringPut (package, (int) *cp);
cp++;
}
vStringCatS (package, "::");
cp = first; /* Rewind */
kind = K_PACKAGE;
spaceRequired = false;
qualified = true;
}
else if (strncmp((const char*) cp, "format", (size_t) 6) == 0)
{
cp += 6;
kind = K_FORMAT;
spaceRequired = true;
qualified = true;
}
else
{
if (isIdentifier1 (*cp))
{
const unsigned char *p = cp;
while (isIdentifier (*p))
++p;
while (isspace (*p))
++p;
if ((int) *p == ':' && (int) *(p + 1) != ':')
kind = K_LABEL;
}
}
if (kind != K_NONE)
{
TRACE("cp0: %s\n", (const char *) cp);
if (spaceRequired && *cp && !isspace (*cp))
continue;
TRACE("cp1: %s\n", (const char *) cp);
while (isspace (*cp))
cp++;
while (!*cp || '#' == *cp) { /* Gobble up empty lines
and comments */
cp = readLineFromInputFile ();
if (!cp)
goto END_MAIN_WHILE;
while (isspace (*cp))
cp++;
}
while (isIdentifier (*cp) || (K_PACKAGE == kind && ':' == *cp))
{
vStringPut (name, (int) *cp);
cp++;
}
if (K_FORMAT == kind &&
vStringLength (name) == 0 && /* cp did not advance */
'=' == *cp)
{
/* format's name is optional. If it's omitted, 'STDOUT'
is assumed. */
vStringCatS (name, "STDOUT");
}
TRACE("name: %s\n", name->buffer);
if (0 == vStringLength(name)) {
vStringClear(name);
continue;
}
if (K_SUBROUTINE == kind)
{
/*
* isSubroutineDeclaration() may consume several lines. So
* we record line positions.
*/
initTagEntry(&e, vStringValue(name), NULL);
if (true == isSubroutineDeclaration(cp)) {
if (true == PerlKinds[K_SUBROUTINE_DECLARATION].enabled) {
kind = K_SUBROUTINE_DECLARATION;
} else {
vStringClear (name);
continue;
}
} else if (! PerlKinds[kind].enabled) {
continue;
}
e.kind = &(PerlKinds[kind]);
makeTagEntry(&e);
if (isXtagEnabled (XTAG_QUALIFIED_TAGS) && qualified &&
package != NULL && vStringLength (package) > 0)
{
vString *const qualifiedName = vStringNew ();
vStringCopy (qualifiedName, package);
vStringCat (qualifiedName, name);
e.name = vStringValue(qualifiedName);
markTagExtraBit (&e, XTAG_QUALIFIED_TAGS);
makeTagEntry(&e);
vStringDelete (qualifiedName);
}
} else if (vStringLength (name) > 0)
{
makeSimpleTag (name, PerlKinds, kind);
if (isXtagEnabled(XTAG_QUALIFIED_TAGS) && qualified &&
K_PACKAGE != kind &&
package != NULL && vStringLength (package) > 0)
{
tagEntryInfo fqe;
vString *const qualifiedName = vStringNew ();
vStringCopy (qualifiedName, package);
vStringCat (qualifiedName, name);
initTagEntry (&fqe, vStringValue (qualifiedName),
PerlKinds + kind);
markTagExtraBit (&fqe, XTAG_QUALIFIED_TAGS);
vStringDelete (qualifiedName);
}
}
vStringClear (name);
}
}
END_MAIN_WHILE:
vStringDelete (name);
if (package != NULL)
vStringDelete (package);
}
boolean cxxParserParseNextToken(void)
{
CXXToken * t = cxxTokenCreate();
// The token chain should not be allowed to grow arbitrairly large.
// The token structures are quite big and it's easy to grow up to
// 5-6GB or memory usage. However this limit should be large enough
// to accomodate all the reasonable statements that could have some
// information in them. This includes multiple function prototypes
// in a single statement (ImageMagick has some examples) but probably
// does NOT include large data tables.
if(g_cxx.pTokenChain->iCount > 16384)
cxxTokenChainDestroyLast(g_cxx.pTokenChain);
cxxTokenChainAppend(g_cxx.pTokenChain,t);
g_cxx.pToken = t;
cxxParserSkipToNonWhiteSpace();
// FIXME: this cpp handling is kind of broken:
// it works only because the moon is in the correct phase.
cppBeginStatement();
// This must be done after getting char from input
t->iLineNumber = getInputLineNumber();
t->oFilePosition = getInputFilePosition();
if(g_cxx.iChar == EOF)
{
t->eType = CXXTokenTypeEOF;
t->bFollowedBySpace = FALSE;
return FALSE;
}
unsigned int uInfo = (g_cxx.iChar < 0x80) ? g_aCharTable[g_cxx.iChar].uType : 0;
//printf("Char %c %02x info %u\n",g_cxx.iChar,g_cxx.iChar,uInfo);
if(uInfo & CXXCharTypeStartOfIdentifier)
{
// word
t->eType = CXXTokenTypeIdentifier;
t->bFollowedBySpace = FALSE;
vStringPut(t->pszWord,g_cxx.iChar);
// special case for tile, which may actually be an operator
if(g_cxx.iChar == '~')
{
// may be followed by space!
g_cxx.iChar = cppGetc();
if(isspace(g_cxx.iChar))
{
t->bFollowedBySpace = TRUE;
g_cxx.iChar = cppGetc();
while(isspace(g_cxx.iChar))
g_cxx.iChar = cppGetc();
}
// non space
uInfo = (g_cxx.iChar < 0x80) ? g_aCharTable[g_cxx.iChar].uType : 0;
if(!(uInfo & CXXCharTypeStartOfIdentifier))
{
// this is not an identifier after all
t->eType = CXXTokenTypeOperator;
if((!t->bFollowedBySpace) && g_cxx.iChar == '=')
{
// make ~= single token so it's not handled as
// a separate assignment
vStringPut(t->pszWord,g_cxx.iChar);
g_cxx.iChar = cppGetc();
t->bFollowedBySpace = isspace(g_cxx.iChar);
}
return TRUE;
}
} else {
g_cxx.iChar = cppGetc();
}
for(;;)
{
uInfo = (g_cxx.iChar < 0x80) ? g_aCharTable[g_cxx.iChar].uType : 0;
if(!(uInfo & CXXCharTypePartOfIdentifier))
break;
vStringPut(t->pszWord,g_cxx.iChar);
g_cxx.iChar = cppGetc();
}
int iCXXKeyword = lookupKeyword(t->pszWord->buffer,g_cxx.eLanguage);
if(iCXXKeyword >= 0)
{
if(
(
(iCXXKeyword == CXXKeywordFINAL) &&
(!g_cxx.bParsingClassStructOrUnionDeclaration)
) || (
(
(iCXXKeyword == CXXKeywordPUBLIC) ||
(iCXXKeyword == CXXKeywordPROTECTED) ||
(iCXXKeyword == CXXKeywordPRIVATE)
) &&
(!g_cxx.bEnablePublicProtectedPrivateKeywords)
)
)
{
t->eType = CXXTokenTypeIdentifier;
} else {
t->eType = CXXTokenTypeKeyword;
t->eKeyword = (enum CXXKeyword)iCXXKeyword;
if(iCXXKeyword == CXXKeyword__ATTRIBUTE__)
{
// special handling for __attribute__
return cxxParserParseNextTokenCondenseAttribute();
}
}
} else {
boolean bIgnoreParens = FALSE;
const char * szReplacement = NULL;
if(isIgnoreToken(
vStringValue(t->pszWord),
&bIgnoreParens,
&szReplacement
))
{
CXX_DEBUG_PRINT("Ignore token %s",vStringValue(t->pszWord));
// FIXME: Handle ignore parens!
if(szReplacement && *szReplacement)
{
vStringClear(t->pszWord);
vStringCatS(t->pszWord,szReplacement);
} else {
// skip
cxxTokenChainDestroyLast(g_cxx.pTokenChain);
return cxxParserParseNextToken();
}
}
}
t->bFollowedBySpace = t->bFollowedBySpace | isspace(g_cxx.iChar);
return TRUE;
}
if(g_cxx.iChar == '-')
{
// special case for pointer
vStringPut(t->pszWord,g_cxx.iChar);
g_cxx.iChar = cppGetc();
if(g_cxx.iChar == '>')
{
t->eType = CXXTokenTypePointerOperator;
vStringPut(t->pszWord,g_cxx.iChar);
g_cxx.iChar = cppGetc();
} else {
t->eType = CXXTokenTypeOperator;
if(g_cxx.iChar == '-')
{
vStringPut(t->pszWord,g_cxx.iChar);
g_cxx.iChar = cppGetc();
}
}
t->bFollowedBySpace = isspace(g_cxx.iChar);
return TRUE;
}
#if 0
// As long as we use cppGetc() we don't need this
if(g_cxx.iChar == '"')
{
// special case for strings
t->eType = CXXTokenTypeStringConstant;
vStringPut(t->pszWord,g_cxx.iChar);
// We don't even care of storing the other chars: we don't need
// them for parsing
// FIXME: We might need them in signature:() tag.. maybe add
// them up to a certain length only?
for(;;)
{
g_cxx.iChar = cppGetc();
if(g_cxx.iChar == EOF)
{
t->bFollowedBySpace = FALSE;
return TRUE;
}
if(g_cxx.iChar == '\\')
{
// escape
g_cxx.iChar = cppGetc();
if(g_cxx.iChar == EOF)
{
t->bFollowedBySpace = FALSE;
return TRUE;
}
} else if(g_cxx.iChar == '"')
{
g_cxx.iChar = cppGetc();
break;
}
}
t->bFollowedBySpace = isspace(g_cxx.iChar);
return TRUE;
}
#else
if(g_cxx.iChar == STRING_SYMBOL)
{
t->eType = CXXTokenTypeStringConstant;
vStringPut(t->pszWord,'"');
vStringPut(t->pszWord,'"');
g_cxx.iChar = cppGetc();
t->bFollowedBySpace = isspace(g_cxx.iChar);
return TRUE;
}
#endif
#if 0
// As long as we use cppGetc() we don't need this
if(g_cxx.iChar == '\'')
{
// special case for strings
t->eType = CXXTokenTypeCharacterConstant;
vStringPut(t->pszWord,g_cxx.iChar);
// We don't even care storing the other chars: we don't
// need them for parsing
for(;;)
{
g_cxx.iChar = cppGetc();
if(g_cxx.iChar == EOF)
{
t->bFollowedBySpace = FALSE;
return TRUE;
}
if(g_cxx.iChar == '\\')
{
// escape
g_cxx.iChar = cppGetc();
if(g_cxx.iChar == EOF)
{
t->bFollowedBySpace = FALSE;
return TRUE;
}
} else if(g_cxx.iChar == '\'')
{
g_cxx.iChar = cppGetc();
break;
}
}
t->bFollowedBySpace = isspace(g_cxx.iChar);
return TRUE;
}
#else
if(g_cxx.iChar == CHAR_SYMBOL)
{
t->eType = CXXTokenTypeCharacterConstant;
vStringPut(t->pszWord,'\'');
vStringPut(t->pszWord,'\'');
g_cxx.iChar = cppGetc();
t->bFollowedBySpace = isspace(g_cxx.iChar);
return TRUE;
}
#endif
if(uInfo & CXXCharTypeDecimalDigit)
{
// number
t->eType = CXXTokenTypeNumber;
vStringPut(t->pszWord,g_cxx.iChar);
for(;;)
{
g_cxx.iChar = cppGetc();
uInfo = (g_cxx.iChar < 0x80) ? g_aCharTable[g_cxx.iChar].uType : 0;
if(!(uInfo & CXXCharTypeValidInNumber))
break;
vStringPut(t->pszWord,g_cxx.iChar);
}
t->bFollowedBySpace = isspace(g_cxx.iChar);
return TRUE;
}
if(uInfo & CXXCharTypeNamedSingleOrRepeatedCharToken)
{
t->eType = g_aCharTable[g_cxx.iChar].uSingleTokenType;
vStringPut(t->pszWord,g_cxx.iChar);
int iChar = g_cxx.iChar;
g_cxx.iChar = cppGetc();
if(g_cxx.iChar == iChar)
{
t->eType = g_aCharTable[g_cxx.iChar].uMultiTokenType;
// We could signal a syntax error with more than two colons
// or equal signs...but we're tolerant
do {
vStringPut(t->pszWord,g_cxx.iChar);
g_cxx.iChar = cppGetc();
} while(g_cxx.iChar == iChar);
}
t->bFollowedBySpace = isspace(g_cxx.iChar);
return TRUE;
}
if(uInfo & CXXCharTypeNamedSingleOrOperatorToken)
{
t->eType = g_aCharTable[g_cxx.iChar].uSingleTokenType;
vStringPut(t->pszWord,g_cxx.iChar);
g_cxx.iChar = cppGetc();
uInfo = (g_cxx.iChar < 0x80) ? g_aCharTable[g_cxx.iChar].uType : 0;
if(uInfo & (CXXCharTypeOperator | CXXCharTypeNamedSingleOrOperatorToken))
{
t->eType = CXXTokenTypeOperator;
do {
vStringPut(t->pszWord,g_cxx.iChar);
g_cxx.iChar = cppGetc();
uInfo = (g_cxx.iChar < 0x80) ? g_aCharTable[g_cxx.iChar].uType : 0;
} while(
uInfo &
(CXXCharTypeOperator | CXXCharTypeNamedSingleOrOperatorToken)
);
}
t->bFollowedBySpace = isspace(g_cxx.iChar);
return TRUE;
}
if(uInfo & CXXCharTypeNamedSingleCharToken)
{
t->eType = g_aCharTable[g_cxx.iChar].uSingleTokenType;
vStringPut(t->pszWord,g_cxx.iChar);
g_cxx.iChar = cppGetc();
t->bFollowedBySpace = isspace(g_cxx.iChar);
return TRUE;
}
if(uInfo & CXXCharTypeOperator)
{
t->eType = CXXTokenTypeOperator;
vStringPut(t->pszWord,g_cxx.iChar);
g_cxx.iChar = cppGetc();
uInfo = (g_cxx.iChar < 0x80) ? g_aCharTable[g_cxx.iChar].uType : 0;
while(uInfo & CXXCharTypeOperator)
{
vStringPut(t->pszWord,g_cxx.iChar);
g_cxx.iChar = cppGetc();
uInfo = (g_cxx.iChar < 0x80) ? g_aCharTable[g_cxx.iChar].uType : 0;
}
t->bFollowedBySpace = isspace(g_cxx.iChar);
return TRUE;
}
t->eType = CXXTokenTypeUnknown;
vStringPut(t->pszWord,g_cxx.iChar);
g_cxx.iChar = cppGetc();
t->bFollowedBySpace = isspace(g_cxx.iChar);
return TRUE;
}
//
// This is called after a full enum/struct/class/union declaration
// that ends with a closing bracket.
//
static boolean cxxParserParseEnumStructClassOrUnionFullDeclarationTrailer(
boolean bParsingTypedef,
enum CXXKeyword eTagKeyword,
enum CXXTagKind eTagKind,
const char * szTypeName
)
{
CXX_DEBUG_ENTER();
cxxTokenChainClear(g_cxx.pTokenChain);
CXX_DEBUG_PRINT(
"Parse enum/struct/class/union trailer, typename is '%s'",
szTypeName
);
MIOPos oFilePosition = getInputFilePosition();
int iFileLine = getInputLineNumber();
if(!cxxParserParseUpToOneOf(CXXTokenTypeEOF | CXXTokenTypeSemicolon))
{
CXX_DEBUG_LEAVE_TEXT("Failed to parse up to EOF/semicolon");
return FALSE;
}
if(cxxTokenTypeIs(g_cxx.pToken,CXXTokenTypeEOF))
{
// It's a syntax error, but we can be tolerant here.
CXX_DEBUG_LEAVE_TEXT("Got EOF after enum/class/struct/union block");
return TRUE;
}
if(g_cxx.pTokenChain->iCount < 2)
{
CXX_DEBUG_LEAVE_TEXT("Nothing interesting after enum/class/struct block");
return TRUE;
}
// fake the initial two tokens
CXXToken * pIdentifier = cxxTokenCreate();
pIdentifier->oFilePosition = oFilePosition;
pIdentifier->iLineNumber = iFileLine;
pIdentifier->eType = CXXTokenTypeIdentifier;
pIdentifier->bFollowedBySpace = TRUE;
vStringCatS(pIdentifier->pszWord,szTypeName);
cxxTokenChainPrepend(g_cxx.pTokenChain,pIdentifier);
CXXToken * pKeyword = cxxTokenCreate();
pKeyword->oFilePosition = oFilePosition;
pKeyword->iLineNumber = iFileLine;
pKeyword->eType = CXXTokenTypeKeyword;
pKeyword->eKeyword = eTagKeyword;
pKeyword->bFollowedBySpace = TRUE;
vStringCatS(pKeyword->pszWord,cxxTagGetKindOptions()[eTagKind].name);
cxxTokenChainPrepend(g_cxx.pTokenChain,pKeyword);
if(bParsingTypedef)
cxxParserExtractTypedef(g_cxx.pTokenChain,TRUE);
else
cxxParserExtractVariableDeclarations(g_cxx.pTokenChain,0);
CXX_DEBUG_LEAVE();
return TRUE;
}
static void readToken (tokenInfo *const token)
{
int c;
static tokenType lastTokenType = TOKEN_NONE;
boolean firstWhitespace = TRUE;
boolean whitespace;
token->type = TOKEN_NONE;
token->keyword = KEYWORD_NONE;
vStringClear (token->string);
getNextChar:
do
{
c = getcFromInputFile ();
token->lineNumber = getInputLineNumber ();
token->filePosition = getInputFilePosition ();
if (c == '\n' && (lastTokenType == TOKEN_IDENTIFIER ||
lastTokenType == TOKEN_STRING ||
lastTokenType == TOKEN_OTHER ||
lastTokenType == TOKEN_CLOSE_PAREN ||
lastTokenType == TOKEN_CLOSE_CURLY ||
lastTokenType == TOKEN_CLOSE_SQUARE))
{
c = ';'; // semicolon injection
}
whitespace = c == '\t' || c == ' ' || c == '\r' || c == '\n';
if (signature && whitespace && firstWhitespace && vStringLength (signature) < MAX_SIGNATURE_LENGTH)
{
firstWhitespace = FALSE;
vStringPut(signature, ' ');
}
}
while (whitespace);
switch (c)
{
case EOF:
token->type = TOKEN_EOF;
break;
case ';':
token->type = TOKEN_SEMICOLON;
break;
case '/':
{
boolean hasNewline = FALSE;
int d = getcFromInputFile ();
switch (d)
{
case '/':
skipToCharacterInInputFile ('\n');
/* Line comments start with the
* character sequence // and
* continue through the next
* newline. A line comment acts
* like a newline. */
ungetcToInputFile ('\n');
goto getNextChar;
case '*':
do
{
do
{
d = getcFromInputFile ();
if (d == '\n')
{
hasNewline = TRUE;
}
} while (d != EOF && d != '*');
c = getcFromInputFile ();
if (c == '/')
break;
else
ungetcToInputFile (c);
} while (c != EOF && c != '\0');
ungetcToInputFile (hasNewline ? '\n' : ' ');
goto getNextChar;
default:
token->type = TOKEN_OTHER;
ungetcToInputFile (d);
break;
}
}
break;
case '"':
case '\'':
case '`':
token->type = TOKEN_STRING;
parseString (token->string, c);
token->lineNumber = getInputLineNumber ();
token->filePosition = getInputFilePosition ();
break;
case '<':
{
int d = getcFromInputFile ();
if (d == '-')
token->type = TOKEN_LEFT_ARROW;
else
{
ungetcToInputFile (d);
token->type = TOKEN_OTHER;
}
}
break;
case '(':
token->type = TOKEN_OPEN_PAREN;
break;
case ')':
token->type = TOKEN_CLOSE_PAREN;
break;
case '{':
token->type = TOKEN_OPEN_CURLY;
break;
case '}':
token->type = TOKEN_CLOSE_CURLY;
break;
case '[':
token->type = TOKEN_OPEN_SQUARE;
break;
case ']':
token->type = TOKEN_CLOSE_SQUARE;
break;
case '*':
token->type = TOKEN_STAR;
break;
case '.':
token->type = TOKEN_DOT;
break;
case ',':
token->type = TOKEN_COMMA;
break;
default:
if (isStartIdentChar (c))
{
parseIdentifier (token->string, c);
token->lineNumber = getInputLineNumber ();
token->filePosition = getInputFilePosition ();
token->keyword = lookupKeyword (vStringValue (token->string), Lang_go);
if (isKeyword (token, KEYWORD_NONE))
token->type = TOKEN_IDENTIFIER;
else
token->type = TOKEN_KEYWORD;
}
else
token->type = TOKEN_OTHER;
break;
}
if (signature && vStringLength (signature) < MAX_SIGNATURE_LENGTH)
{
if (token->type == TOKEN_LEFT_ARROW)
vStringCatS(signature, "<-");
else if (token->type == TOKEN_STRING)
{
// only struct member annotations can appear in function prototypes
// so only `` type strings are possible
vStringPut(signature, '`');
vStringCat(signature, token->string);
vStringPut(signature, '`');
}
else if (token->type == TOKEN_IDENTIFIER || token->type == TOKEN_KEYWORD)
vStringCat(signature, token->string);
else if (c != EOF)
vStringPut(signature, c);
}
lastTokenType = token->type;
}
/* Advances the parser one token, optionally skipping whitespace
* (otherwise it is concatenated and returned as a single whitespace token).
* Whitespace is needed to properly render function signatures. Unrecognized
* token starts are stored literally, e.g. token may equal to a character '#'. */
static int advanceToken (lexerState *lexer, boolean skip_whitspace)
{
boolean have_whitespace = FALSE;
lexer->line = getInputLineNumber();
lexer->pos = getInputFilePosition();
while (lexer->cur_c != EOF)
{
if (isWhitespace(lexer->cur_c))
{
scanWhitespace(lexer);
have_whitespace = TRUE;
}
else if (lexer->cur_c == '/' && (lexer->next_c == '/' || lexer->next_c == '*'))
{
scanComments(lexer);
have_whitespace = TRUE;
}
else
{
if (have_whitespace && !skip_whitspace)
return lexer->cur_token = TOKEN_WHITESPACE;
break;
}
}
lexer->line = getInputLineNumber();
lexer->pos = getInputFilePosition();
while (lexer->cur_c != EOF)
{
if (lexer->cur_c == '"')
{
scanString(lexer);
return lexer->cur_token = TOKEN_STRING;
}
else if (lexer->cur_c == 'r' && (lexer->next_c == '#' || lexer->next_c == '"'))
{
scanRawString(lexer);
return lexer->cur_token = TOKEN_STRING;
}
else if (lexer->cur_c == '\'')
{
scanCharacterOrLifetime(lexer);
return lexer->cur_token = TOKEN_STRING;
}
else if (isIdentifierStart(lexer->cur_c))
{
scanIdentifier(lexer);
return lexer->cur_token = TOKEN_IDENT;
}
/* These shift tokens aren't too important for tag-generation per se,
* but they confuse the skipUntil code which tracks the <> pairs. */
else if (lexer->cur_c == '>' && lexer->next_c == '>')
{
advanceNChar(lexer, 2);
return lexer->cur_token = TOKEN_RSHIFT;
}
else if (lexer->cur_c == '<' && lexer->next_c == '<')
{
advanceNChar(lexer, 2);
return lexer->cur_token = TOKEN_LSHIFT;
}
else if (lexer->cur_c == '-' && lexer->next_c == '>')
{
advanceNChar(lexer, 2);
return lexer->cur_token = TOKEN_RARROW;
}
else
{
int c = lexer->cur_c;
advanceChar(lexer);
return lexer->cur_token = c;
}
}
return lexer->cur_token = TOKEN_EOF;
}
bool cxxParserParseNextToken(void)
{
CXXToken * t = cxxTokenCreate();
// The token chain should not be allowed to grow arbitrairly large.
// The token structures are quite big and it's easy to grow up to
// 5-6GB or memory usage. However this limit should be large enough
// to accomodate all the reasonable statements that could have some
// information in them. This includes multiple function prototypes
// in a single statement (ImageMagick has some examples) but probably
// does NOT include large data tables.
if(g_cxx.pTokenChain->iCount > 16384)
cxxTokenChainDestroyLast(g_cxx.pTokenChain);
cxxTokenChainAppend(g_cxx.pTokenChain,t);
g_cxx.pToken = t;
cxxParserSkipToNonWhiteSpace();
// FIXME: this cpp handling is kind of broken:
// it works only because the moon is in the correct phase.
cppBeginStatement();
// This must be done after getting char from input
t->iLineNumber = getInputLineNumber();
t->oFilePosition = getInputFilePosition();
if(g_cxx.iChar == EOF)
{
t->eType = CXXTokenTypeEOF;
t->bFollowedBySpace = false;
return false;
}
unsigned int uInfo = UINFO(g_cxx.iChar);
//fprintf(stderr,"Char %c %02x info %u\n",g_cxx.iChar,g_cxx.iChar,uInfo);
if(uInfo & CXXCharTypeStartOfIdentifier)
{
// word
t->eType = CXXTokenTypeIdentifier;
t->bFollowedBySpace = false;
vStringPut(t->pszWord,g_cxx.iChar);
// special case for tile, which may actually be an operator
if(g_cxx.iChar == '~')
{
// may be followed by space!
g_cxx.iChar = cppGetc();
if(isspace(g_cxx.iChar))
{
t->bFollowedBySpace = true;
g_cxx.iChar = cppGetc();
while(isspace(g_cxx.iChar))
g_cxx.iChar = cppGetc();
}
// non space
uInfo = UINFO(g_cxx.iChar);
if(!(uInfo & CXXCharTypeStartOfIdentifier))
{
// this is not an identifier after all
t->eType = CXXTokenTypeOperator;
if((!t->bFollowedBySpace) && g_cxx.iChar == '=')
{
// make ~= single token so it's not handled as
// a separate assignment
vStringPut(t->pszWord,g_cxx.iChar);
g_cxx.iChar = cppGetc();
t->bFollowedBySpace = isspace(g_cxx.iChar);
}
return true;
}
} else {
g_cxx.iChar = cppGetc();
}
for(;;)
{
uInfo = UINFO(g_cxx.iChar);
if(!(uInfo & CXXCharTypePartOfIdentifier))
break;
vStringPut(t->pszWord,g_cxx.iChar);
g_cxx.iChar = cppGetc();
}
int iCXXKeyword = lookupKeyword(t->pszWord->buffer,g_cxx.eLanguage);
if(iCXXKeyword >= 0)
{
if(
(
(iCXXKeyword == CXXKeywordFINAL) &&
(!g_cxx.bParsingClassStructOrUnionDeclaration)
) || (
(
(iCXXKeyword == CXXKeywordPUBLIC) ||
(iCXXKeyword == CXXKeywordPROTECTED) ||
(iCXXKeyword == CXXKeywordPRIVATE)
) &&
(!g_cxx.bEnablePublicProtectedPrivateKeywords)
)
)
{
t->eType = CXXTokenTypeIdentifier;
} else {
t->eType = CXXTokenTypeKeyword;
t->eKeyword = (enum CXXKeyword)iCXXKeyword;
if(iCXXKeyword == CXXKeyword__ATTRIBUTE__)
{
// special handling for __attribute__
return cxxParserParseNextTokenCondenseAttribute();
}
}
} else {
const cppMacroInfo * pMacro = cppFindMacro(vStringValue(t->pszWord));
if(pMacro)
{
CXX_DEBUG_PRINT("Macro %s",vStringValue(t->pszWord));
cxxTokenChainDestroyLast(g_cxx.pTokenChain);
CXXToken * pParameterChain = NULL;
if(pMacro->hasParameterList)
{
CXX_DEBUG_PRINT("Macro has parameter list");
if(!cxxParserParseNextTokenSkipMacroParenthesis(&pParameterChain))
return false;
}
// This is used to avoid infinite recursion in substitution
// (things like -D foo=foo or similar)
static int iReplacementRecursionCount = 0;
if(pMacro->replacements)
{
CXX_DEBUG_PRINT("The token has replacements: applying");
if(iReplacementRecursionCount < 1024)
{
// unget last char
cppUngetc(g_cxx.iChar);
// unget the replacement
cxxParserParseNextTokenApplyReplacement(
pMacro,
pParameterChain
);
g_cxx.iChar = cppGetc();
}
}
if(pParameterChain)
cxxTokenDestroy(pParameterChain);
iReplacementRecursionCount++;
// Have no token to return: parse it
CXX_DEBUG_PRINT("Parse inner token");
bool bRet = cxxParserParseNextToken();
CXX_DEBUG_PRINT("Parsed inner token: %s type %d",g_cxx.pToken->pszWord->buffer,g_cxx.pToken->eType);
iReplacementRecursionCount--;
return bRet;
}
}
t->bFollowedBySpace = isspace(g_cxx.iChar);
return true;
}
if(g_cxx.iChar == '-')
{
// special case for pointer
vStringPut(t->pszWord,g_cxx.iChar);
g_cxx.iChar = cppGetc();
if(g_cxx.iChar == '>')
{
t->eType = CXXTokenTypePointerOperator;
vStringPut(t->pszWord,g_cxx.iChar);
g_cxx.iChar = cppGetc();
} else {
t->eType = CXXTokenTypeOperator;
if(g_cxx.iChar == '-')
{
vStringPut(t->pszWord,g_cxx.iChar);
g_cxx.iChar = cppGetc();
}
}
t->bFollowedBySpace = isspace(g_cxx.iChar);
return true;
}
#if 0
// As long as we use cppGetc() we don't need this
if(g_cxx.iChar == '"')
{
// special case for strings
t->eType = CXXTokenTypeStringConstant;
vStringPut(t->pszWord,g_cxx.iChar);
// We don't even care of storing the other chars: we don't need
// them for parsing
// FIXME: We might need them in signature:() tag.. maybe add
// them up to a certain length only?
for(;;)
{
g_cxx.iChar = cppGetc();
if(g_cxx.iChar == EOF)
{
t->bFollowedBySpace = false;
return true;
}
if(g_cxx.iChar == '\\')
{
// escape
g_cxx.iChar = cppGetc();
if(g_cxx.iChar == EOF)
{
t->bFollowedBySpace = false;
return true;
}
} else if(g_cxx.iChar == '"')
{
g_cxx.iChar = cppGetc();
break;
}
}
t->bFollowedBySpace = isspace(g_cxx.iChar);
return true;
}
#else
if(g_cxx.iChar == STRING_SYMBOL)
{
t->eType = CXXTokenTypeStringConstant;
vStringPut(t->pszWord,'"');
vStringPut(t->pszWord,'"');
g_cxx.iChar = cppGetc();
t->bFollowedBySpace = isspace(g_cxx.iChar);
return true;
}
#endif
#if 0
// As long as we use cppGetc() we don't need this
if(g_cxx.iChar == '\'')
{
// special case for strings
t->eType = CXXTokenTypeCharacterConstant;
vStringPut(t->pszWord,g_cxx.iChar);
// We don't even care storing the other chars: we don't
// need them for parsing
for(;;)
{
g_cxx.iChar = cppGetc();
if(g_cxx.iChar == EOF)
{
t->bFollowedBySpace = false;
return true;
}
if(g_cxx.iChar == '\\')
{
// escape
g_cxx.iChar = cppGetc();
if(g_cxx.iChar == EOF)
{
t->bFollowedBySpace = false;
return true;
}
} else if(g_cxx.iChar == '\'')
{
g_cxx.iChar = cppGetc();
break;
}
}
t->bFollowedBySpace = isspace(g_cxx.iChar);
return true;
}
#else
if(g_cxx.iChar == CHAR_SYMBOL)
{
t->eType = CXXTokenTypeCharacterConstant;
vStringPut(t->pszWord,'\'');
vStringPut(t->pszWord,'\'');
g_cxx.iChar = cppGetc();
t->bFollowedBySpace = isspace(g_cxx.iChar);
return true;
}
#endif
if(uInfo & CXXCharTypeDecimalDigit)
{
// number
t->eType = CXXTokenTypeNumber;
vStringPut(t->pszWord,g_cxx.iChar);
for(;;)
{
g_cxx.iChar = cppGetc();
uInfo = UINFO(g_cxx.iChar);
if(!(uInfo & CXXCharTypeValidInNumber))
break;
vStringPut(t->pszWord,g_cxx.iChar);
}
t->bFollowedBySpace = isspace(g_cxx.iChar);
return true;
}
if(uInfo & CXXCharTypeNamedSingleOrRepeatedCharToken)
{
t->eType = g_aCharTable[g_cxx.iChar].uSingleTokenType;
vStringPut(t->pszWord,g_cxx.iChar);
int iChar = g_cxx.iChar;
g_cxx.iChar = cppGetc();
if(g_cxx.iChar == iChar)
{
t->eType = g_aCharTable[g_cxx.iChar].uMultiTokenType;
// We could signal a syntax error with more than two colons
// or equal signs...but we're tolerant
do {
vStringPut(t->pszWord,g_cxx.iChar);
g_cxx.iChar = cppGetc();
} while(g_cxx.iChar == iChar);
}
t->bFollowedBySpace = isspace(g_cxx.iChar);
return true;
}
if(uInfo & CXXCharTypeNamedSingleOrOperatorToken)
{
t->eType = g_aCharTable[g_cxx.iChar].uSingleTokenType;
vStringPut(t->pszWord,g_cxx.iChar);
g_cxx.iChar = cppGetc();
uInfo = UINFO(g_cxx.iChar);
if(uInfo & (CXXCharTypeOperator | CXXCharTypeNamedSingleOrOperatorToken))
{
t->eType = CXXTokenTypeOperator;
do {
vStringPut(t->pszWord,g_cxx.iChar);
g_cxx.iChar = cppGetc();
uInfo = UINFO(g_cxx.iChar);
} while(
uInfo &
(CXXCharTypeOperator | CXXCharTypeNamedSingleOrOperatorToken)
);
}
t->bFollowedBySpace = isspace(g_cxx.iChar);
return true;
}
if(uInfo & CXXCharTypeNamedSingleCharToken)
{
t->eType = g_aCharTable[g_cxx.iChar].uSingleTokenType;
vStringPut(t->pszWord,g_cxx.iChar);
g_cxx.iChar = cppGetc();
t->bFollowedBySpace = isspace(g_cxx.iChar);
return true;
}
if(uInfo & CXXCharTypeOperator)
{
t->eType = CXXTokenTypeOperator;
vStringPut(t->pszWord,g_cxx.iChar);
g_cxx.iChar = cppGetc();
uInfo = UINFO(g_cxx.iChar);
while(uInfo & CXXCharTypeOperator)
{
vStringPut(t->pszWord,g_cxx.iChar);
g_cxx.iChar = cppGetc();
uInfo = UINFO(g_cxx.iChar);
}
t->bFollowedBySpace = isspace(g_cxx.iChar);
return true;
}
t->eType = CXXTokenTypeUnknown;
vStringPut(t->pszWord,g_cxx.iChar);
g_cxx.iChar = cppGetc();
t->bFollowedBySpace = isspace(g_cxx.iChar);
return true;
}
static void readTag (tokenInfo *token, vString *text, int depth)
{
bool textCreated = false;
readToken (token, true);
if (token->type == TOKEN_NAME)
{
keywordId startTag;
bool isHeading;
bool isVoid;
startTag = lookupKeyword (vStringValue (token->string), Lang_html);
isHeading = (startTag == KEYWORD_h1 || startTag == KEYWORD_h2 || startTag == KEYWORD_h3);
isVoid = (startTag >= KEYWORD_area && startTag <= KEYWORD_wbr);
if (text == NULL && isHeading)
{
text = vStringNew ();
textCreated = true;
}
do
{
readToken (token, true);
if (startTag == KEYWORD_a && token->type == TOKEN_NAME)
{
keywordId attribute = lookupKeyword (vStringValue (token->string), Lang_html);
if (attribute == KEYWORD_name)
{
readToken (token, true);
if (token->type == TOKEN_EQUAL)
{
readToken (token, true);
if (token->type == TOKEN_STRING || token->type == TOKEN_NAME)
makeSimpleTag (token->string, HtmlKinds, K_ANCHOR);
}
}
}
}
while (token->type != TOKEN_TAG_END && token->type != TOKEN_TAG_END2 &&
token->type != TOKEN_EOF);
if (!isVoid && token->type == TOKEN_TAG_END && depth < MAX_DEPTH)
{
long startSourceLineNumber = getSourceLineNumber ();
long startLineNumber = getInputLineNumber ();
long startLineOffset = getInputLineOffset ();
long endLineNumber;
long endLineOffset;
bool tag_start2;
if (startTag == KEYWORD_script)
{
bool script = skipScriptContent (token, &endLineNumber, &endLineOffset);
if (script)
makePromise ("JavaScript", startLineNumber, startLineOffset,
endLineNumber, endLineOffset, startSourceLineNumber);
readToken (token, true);
goto out;
}
tag_start2 = readTagContent (token, text, &endLineNumber, &endLineOffset, depth);
if (tag_start2)
{
readToken (token, true);
if (isHeading && textCreated && vStringLength (text) > 0)
{
keywordId endTag = lookupKeyword (vStringValue (token->string), Lang_html);
if (startTag == endTag)
{
htmlKind headingKind;
if (startTag == KEYWORD_h1)
headingKind = K_HEADING1;
else if (startTag == KEYWORD_h2)
headingKind = K_HEADING2;
else
headingKind = K_HEADING3;
vStringStripTrailing (text);
makeSimpleTag (text, HtmlKinds, headingKind);
}
}
else if (startTag == KEYWORD_style)
{
keywordId endTag = lookupKeyword (vStringValue (token->string), Lang_html);
if (startTag == endTag)
makePromise ("CSS", startLineNumber, startLineOffset,
endLineNumber, endLineOffset, startSourceLineNumber);
}
readToken (token, true);
}
}
}
out:
if (textCreated)
vStringDelete (text);
}
