static void readAndEmitTag (const unsigned char** cp, rubyKind expected_kind)
{
if (isspace (**cp))
{
vString *name = vStringNew ();
vString *chunk = vStringNew ();
rubyKind actual_kind;
unsigned int i = 0;
/* parse the identifier, allowing scoping like "class Foo::Bar::Baz" */
while (1)
{
actual_kind = parseIdentifier (cp, chunk, expected_kind);
if (i++ > 0)
vStringPut (name, '.');
vStringCat (name, chunk);
vStringClear (chunk);
if (actual_kind != K_UNDEFINED && (*cp)[0] == ':' && (*cp)[1] == ':')
*cp += 2;
else
break;
}
vStringDelete (chunk);
vStringTerminate (name);
if (actual_kind == K_UNDEFINED || vStringLength (name) == 0)
{
/*
* What kind of tags should we create for code like this?
*
* %w(self.clfloor clfloor).each do |name|
* module_eval <<-"end;"
* def #{name}(x, y=1)
* q, r = x.divmod(y)
* q = q.to_i
* return q, r
* end
* end;
* end
*
* Or this?
*
* class << HTTP
*
* For now, we don't create any.
*/
}
else
{
emitRubyTag (name, actual_kind);
}
vStringDelete (name);
}
}
static void appendText (vString *text, vString *appendedText)
{
if (text != NULL && vStringLength (appendedText) > 0)
{
if (vStringLength (text) > 0 && vStringLast (text) == ' ' &&
vStringLength (appendedText) > 0 && vStringChar (appendedText, 0) == ' ')
{
vStringStripTrailing (text);
}
vStringCat (text, appendedText);
}
}
static void readQualifiedName (tokenInfo *const token, vString *name,
tokenInfo *const lastToken)
{
while (token->type == TOKEN_IDENTIFIER || token->type == TOKEN_BACKSLASH)
{
if (token->type == TOKEN_BACKSLASH)
vStringPut (name, '\\');
else
vStringCat (name, token->string);
copyToken (lastToken, token, TRUE);
readToken (token);
}
}
static void addToScope (tokenInfo *const token, const vString *const extra,
int kindOfUpperScope)
{
if (vStringLength (token->scope) > 0)
{
const char* sep;
sep = phpScopeSeparatorFor(token->parentKind,
kindOfUpperScope);
vStringCatS (token->scope, sep);
}
vStringCat (token->scope, extra);
vStringTerminate(token->scope);
}
static void plistFindTagsUnderKey (xmlNode *node,
const struct sTagXpathRecurSpec *spec,
xmlXPathContext *ctx,
void *userData)
{
xmlNode *current;
xmlNode *prev;
stringList *queue;
vString* path;
vString* v;
int c;
queue = stringListNew ();
current = node;
for (current = node; current; current = current->parent)
{
if (isCompoundElement (current)
&& (prev = getPrevKeyElement (current)))
{
char* parent = (char *)xmlNodeGetContent (prev);
if (parent)
{
v = vStringNewInit (parent);
stringListAdd (queue, v);
xmlFree (parent);
}
}
}
path = vStringNew ();
while ((c = stringListCount (queue)) > 0)
{
v = stringListLast (queue);
vStringCat (path, v);
vStringDelete (v);
stringListRemoveLast (queue);
if (c != 1)
vStringPut (path, '.');
}
stringListDelete (queue);
findXMLTags (ctx, node,
plistXpathTableTable + TABLE_TEXT,
PlistKinds,
(vStringLength (path) > 0)? vStringValue (path): NULL);
vStringDelete (path);
}
void cxxTokenAppendToString(vString * s,CXXToken * t)
{
switch(t->eType)
{
case CXXTokenTypeParenthesisChain:
case CXXTokenTypeSquareParenthesisChain:
case CXXTokenTypeBracketChain:
case CXXTokenTypeAngleBracketChain:
CXX_DEBUG_ASSERT(t->pChain,"This token should have a nested chain!");
cxxTokenChainJoinInString(t->pChain,s,NULL,0);
break;
default:
vStringCat(s,t->pszWord);
break;
}
}
/* Essentially grabs the last ident before 'for', '<' and '{', which
* tends to correspond to what we want as the impl tag entry name */
static void parseQualifiedType (lexerState *lexer, vString* name)
{
while (lexer->cur_token != TOKEN_EOF)
{
if (lexer->cur_token == TOKEN_IDENT)
{
if (strcmp(lexer->token_str->buffer, "for") == 0)
break;
vStringClear(name);
vStringCat(name, lexer->token_str);
}
else if (lexer->cur_token == '<' || lexer->cur_token == '{')
{
break;
}
advanceToken(lexer, TRUE);
}
skipTypeBlock(lexer);
}
static void createContext (tokenInfo *const scope)
{
if (scope)
{
vString *contextName = vStringNew ();
verbose ("Creating new context %s\n", vStringValue (scope->name));
/* Determine full context name */
if (currentContext->kind != K_UNDEFINED)
{
vStringCopy (contextName, currentContext->name);
vStringCatS (contextName, ".");
}
vStringCat (contextName, scope->name);
/* Create context */
currentContext = pushToken (currentContext, scope);
vStringCopy (currentContext->name, contextName);
vStringDelete (contextName);
}
}
static void makeVhdlTag (tokenInfo * const token, const vhdlKind kind)
{
if (VhdlKinds[kind].enabled)
{
/*
* If a scope has been added to the token, change the token
* string to include the scope when making the tag.
*/
if (vStringLength (token->scope) > 0)
{
vString *fulltag = vStringNew ();
vStringCopy (fulltag, token->scope);
vStringPut (fulltag, '.');
vStringCat (fulltag, token->string);
vStringCopy (token->string, fulltag);
vStringDelete (fulltag);
}
makeConstTag (token, kind);
}
}
static void parseMethodsImplemName (vString * const ident, objcToken what)
{
switch (what)
{
case Tok_PARL:
toDoNext = &tillToken;
comeAfter = &parseMethodsImplemName;
waitedToken = Tok_PARR;
break;
case Tok_dpoint:
vStringCat (fullMethodName, prevIdent);
vStringCatS (fullMethodName, ":");
vStringClear (prevIdent);
break;
case ObjcIDENTIFIER:
vStringCopy (prevIdent, ident);
break;
case Tok_CurlL:
case Tok_semi:
/* method name is not simple */
if (vStringLength (fullMethodName) != '\0')
{
addTag (fullMethodName, methodKind);
vStringClear (fullMethodName);
}
else
addTag (prevIdent, methodKind);
toDoNext = &parseImplemMethods;
parseImplemMethods (ident, what);
vStringClear (prevIdent);
break;
default:
break;
}
}
static void makeTag (tokenInfo *const token, const goKind kind,
tokenInfo *const parent_token, const goKind parent_kind,
const char *argList, const char *varType)
{
const char *const name = vStringValue (token->string);
tagEntryInfo e;
initTagEntry (&e, name);
if (!GoKinds [kind].enabled)
return;
e.lineNumber = token->lineNumber;
e.filePosition = token->filePosition;
e.kindName = GoKinds [kind].name;
e.kind = GoKinds [kind].letter;
if (argList)
e.extensionFields.arglist = argList;
if (varType)
e.extensionFields.varType = varType;
if (parent_kind != GOTAG_UNDEFINED && parent_token != NULL)
{
e.extensionFields.scope[0] = GoKinds[parent_kind].name;
e.extensionFields.scope[1] = vStringValue (parent_token->string);
}
makeTagEntry (&e);
if (scope && Option.include.qualifiedTags)
{
vString *qualifiedName = vStringNew ();
vStringCopy (qualifiedName, scope);
vStringCatS (qualifiedName, ".");
vStringCat (qualifiedName, token->string);
e.name = vStringValue (qualifiedName);
makeTagEntry (&e);
vStringDelete (qualifiedName);
}
}
/* parses namespace declarations
* namespace Foo {}
* namespace Foo\Bar {}
* namespace Foo;
* namespace Foo\Bar;
* namespace;
* napespace {} */
static boolean parseNamespace (tokenInfo *const token)
{
tokenInfo *nsToken = newToken ();
vStringClear (CurrentNamesapce);
copyToken (nsToken, token, FALSE);
do
{
readToken (token);
if (token->type == TOKEN_IDENTIFIER)
{
if (vStringLength (CurrentNamesapce) > 0)
{
const char *sep;
sep = phpScopeSeparatorFor(K_NAMESPACE,
K_NAMESPACE);
vStringCatS (CurrentNamesapce, sep);
}
vStringCat (CurrentNamesapce, token->string);
}
}
while (token->type != TOKEN_EOF &&
token->type != TOKEN_SEMICOLON &&
token->type != TOKEN_OPEN_CURLY);
vStringTerminate (CurrentNamesapce);
if (vStringLength (CurrentNamesapce) > 0)
makeNamespacePhpTag (nsToken, CurrentNamesapce);
if (token->type == TOKEN_OPEN_CURLY)
enterScope (token, NULL, -1);
deleteToken (nsToken);
return TRUE;
}
static void makeEiffelLocalTag (tokenInfo *const token)
{
if (EiffelKinds [EKIND_LOCAL].enabled && isXtagEnabled(XTAG_FILE_SCOPE))
{
const char *const name = vStringValue (token->string);
vString* scope = vStringNew ();
tagEntryInfo e;
initTagEntry (&e, name, &(EiffelKinds [EKIND_LOCAL]));
e.isFileScope = TRUE;
vStringCopy (scope, token->className);
vStringPut (scope, '.');
vStringCat (scope, token->featureName);
e.extensionFields.scopeKind = &(EiffelKinds [EKIND_FEATURE]);
e.extensionFields.scopeName = vStringValue (scope);
makeTagEntry (&e);
vStringDelete (scope);
}
}
/* Push a new context, handle null string */
static void pushContext (contextKind kind, contextType type, parseNext after,
vString const *contextName)
{
int parentIndex;
if (stackIndex >= OCAML_MAX_STACK_SIZE)
{
verbose ("OCaml Maximum depth reached");
return;
}
stack[stackIndex].kind = kind;
stack[stackIndex].type = type;
stack[stackIndex].callback = after;
parentIndex = getLastNamedIndex ();
if (contextName == NULL)
{
vStringClear (stack[stackIndex++].contextName);
return;
}
if (parentIndex >= 0)
{
vStringCopy (stack[stackIndex].contextName,
stack[parentIndex].contextName);
vStringPut (stack[stackIndex].contextName,
contextTypeSuffix (stack[parentIndex].type));
vStringCat (stack[stackIndex].contextName, contextName);
}
else
vStringCopy (stack[stackIndex].contextName, contextName);
stackIndex++;
}
static void makeTag (tokenInfo *const token, const goKind kind,
tokenInfo *const parent_token, const goKind parent_kind,
const char *argList)
{
const char *const name = vStringValue (token->string);
tagEntryInfo e;
initTagEntry (&e, name, &(GoKinds [kind]));
if (!GoKinds [kind].enabled)
return;
e.lineNumber = token->lineNumber;
e.filePosition = token->filePosition;
if (argList)
e.extensionFields.signature = argList;
if (parent_kind != GOTAG_UNDEFINED && parent_token != NULL)
{
e.extensionFields.scopeKind = &(GoKinds[parent_kind]);
e.extensionFields.scopeName = vStringValue (parent_token->string);
}
makeTagEntry (&e);
if (scope && isXtagEnabled(XTAG_QUALIFIED_TAGS))
{
vString *qualifiedName = vStringNew ();
vStringCopy (qualifiedName, scope);
vStringCatS (qualifiedName, ".");
vStringCat (qualifiedName, token->string);
e.name = vStringValue (qualifiedName);
markTagExtraBit (&e, XTAG_QUALIFIED_TAGS);
makeTagEntry (&e);
vStringDelete (qualifiedName);
}
}
static void readToken (tokenInfo *const token)
{
int c;
static tokenType lastTokenType = TOKEN_NONE;
boolean firstWhitespace = TRUE;
boolean whitespace;
token->type = TOKEN_NONE;
token->keyword = KEYWORD_NONE;
vStringClear (token->string);
getNextChar:
do
{
c = getcFromInputFile ();
token->lineNumber = getInputLineNumber ();
token->filePosition = getInputFilePosition ();
if (c == '\n' && (lastTokenType == TOKEN_IDENTIFIER ||
lastTokenType == TOKEN_STRING ||
lastTokenType == TOKEN_OTHER ||
lastTokenType == TOKEN_CLOSE_PAREN ||
lastTokenType == TOKEN_CLOSE_CURLY ||
lastTokenType == TOKEN_CLOSE_SQUARE))
{
c = ';'; // semicolon injection
}
whitespace = c == '\t' || c == ' ' || c == '\r' || c == '\n';
if (signature && whitespace && firstWhitespace && vStringLength (signature) < MAX_SIGNATURE_LENGTH)
{
firstWhitespace = FALSE;
vStringPut(signature, ' ');
}
}
while (whitespace);
switch (c)
{
case EOF:
token->type = TOKEN_EOF;
break;
case ';':
token->type = TOKEN_SEMICOLON;
break;
case '/':
{
boolean hasNewline = FALSE;
int d = getcFromInputFile ();
switch (d)
{
case '/':
skipToCharacterInInputFile ('\n');
/* Line comments start with the
* character sequence // and
* continue through the next
* newline. A line comment acts
* like a newline. */
ungetcToInputFile ('\n');
goto getNextChar;
case '*':
do
{
do
{
d = getcFromInputFile ();
if (d == '\n')
{
hasNewline = TRUE;
}
} while (d != EOF && d != '*');
c = getcFromInputFile ();
if (c == '/')
break;
else
ungetcToInputFile (c);
} while (c != EOF && c != '\0');
ungetcToInputFile (hasNewline ? '\n' : ' ');
goto getNextChar;
default:
token->type = TOKEN_OTHER;
ungetcToInputFile (d);
break;
}
}
break;
case '"':
case '\'':
case '`':
token->type = TOKEN_STRING;
parseString (token->string, c);
token->lineNumber = getInputLineNumber ();
token->filePosition = getInputFilePosition ();
break;
case '<':
{
int d = getcFromInputFile ();
if (d == '-')
token->type = TOKEN_LEFT_ARROW;
else
{
ungetcToInputFile (d);
token->type = TOKEN_OTHER;
}
}
break;
case '(':
token->type = TOKEN_OPEN_PAREN;
break;
case ')':
token->type = TOKEN_CLOSE_PAREN;
break;
case '{':
token->type = TOKEN_OPEN_CURLY;
break;
case '}':
token->type = TOKEN_CLOSE_CURLY;
break;
case '[':
token->type = TOKEN_OPEN_SQUARE;
break;
case ']':
token->type = TOKEN_CLOSE_SQUARE;
break;
case '*':
token->type = TOKEN_STAR;
break;
case '.':
token->type = TOKEN_DOT;
break;
case ',':
token->type = TOKEN_COMMA;
break;
default:
if (isStartIdentChar (c))
{
parseIdentifier (token->string, c);
token->lineNumber = getInputLineNumber ();
token->filePosition = getInputFilePosition ();
token->keyword = lookupKeyword (vStringValue (token->string), Lang_go);
if (isKeyword (token, KEYWORD_NONE))
token->type = TOKEN_IDENTIFIER;
else
token->type = TOKEN_KEYWORD;
}
else
token->type = TOKEN_OTHER;
break;
}
if (signature && vStringLength (signature) < MAX_SIGNATURE_LENGTH)
{
if (token->type == TOKEN_LEFT_ARROW)
vStringCatS(signature, "<-");
else if (token->type == TOKEN_STRING)
{
// only struct member annotations can appear in function prototypes
// so only `` type strings are possible
vStringPut(signature, '`');
vStringCat(signature, token->string);
vStringPut(signature, '`');
}
else if (token->type == TOKEN_IDENTIFIER || token->type == TOKEN_KEYWORD)
vStringCat(signature, token->string);
else if (c != EOF)
vStringPut(signature, c);
}
lastTokenType = token->type;
}
/* parse a function
*
* function myfunc($foo, $bar) {}
*/
static bool parseFunction (tokenInfo *const token)
{
bool readNext = true;
tokenInfo *nameFree = NULL;
const char *access;
readToken (token);
if (token->type != TOKEN_IDENTIFIER)
return false;
access = parsePowerShellScope (token);
nameFree = newToken ();
copyToken (nameFree, token, true);
readToken (token);
if (token->type == TOKEN_OPEN_PAREN)
{
vString *arglist = vStringNew ();
int depth = 1;
vStringPut (arglist, '(');
do
{
readToken (token);
switch (token->type)
{
case TOKEN_OPEN_PAREN: depth++; break;
case TOKEN_CLOSE_PAREN: depth--; break;
default: break;
}
/* display part */
switch (token->type)
{
case TOKEN_CLOSE_CURLY: vStringPut (arglist, '}'); break;
case TOKEN_CLOSE_PAREN: vStringPut (arglist, ')'); break;
case TOKEN_CLOSE_SQUARE: vStringPut (arglist, ']'); break;
case TOKEN_COLON: vStringPut (arglist, ':'); break;
case TOKEN_COMMA: vStringCatS (arglist, ", "); break;
case TOKEN_EQUAL_SIGN: vStringCatS (arglist, " = "); break;
case TOKEN_OPEN_CURLY: vStringPut (arglist, '{'); break;
case TOKEN_OPEN_PAREN: vStringPut (arglist, '('); break;
case TOKEN_OPEN_SQUARE: vStringPut (arglist, '['); break;
case TOKEN_PERIOD: vStringPut (arglist, '.'); break;
case TOKEN_SEMICOLON: vStringPut (arglist, ';'); break;
case TOKEN_STRING: vStringCatS (arglist, "'...'"); break;
case TOKEN_IDENTIFIER:
case TOKEN_KEYWORD:
case TOKEN_VARIABLE:
{
switch (vStringLast (arglist))
{
case 0:
case ' ':
case '{':
case '(':
case '[':
case '.':
/* no need for a space between those and the identifier */
break;
default:
vStringPut (arglist, ' ');
break;
}
if (token->type == TOKEN_VARIABLE)
vStringPut (arglist, '$');
vStringCat (arglist, token->string);
break;
}
default: break;
}
}
while (token->type != TOKEN_EOF && depth > 0);
makeFunctionTag (nameFree, arglist, access);
vStringDelete (arglist);
readToken (token);
}
else if (token->type == TOKEN_OPEN_CURLY)
{ /* filters doesn't need to have an arglist */
makeFunctionTag (nameFree, NULL, access);
}
if (token->type == TOKEN_OPEN_CURLY)
enterScope (token, nameFree->string, K_FUNCTION);
else
readNext = false;
if (nameFree)
deleteToken (nameFree);
return readNext;
}
static void readTokenFull (tokenInfo *const token, boolean include_newlines, vString *const repr)
{
int c;
int i;
token->type = TOKEN_UNDEFINED;
token->keyword = KEYWORD_NONE;
vStringClear (token->string);
getNextChar:
i = 0;
do
{
c = fileGetc ();
i++;
}
while (c == '\t' || c == ' ' ||
((c == '\r' || c == '\n') && ! include_newlines));
token->lineNumber = getSourceLineNumber ();
token->filePosition = getInputFilePosition ();
if (repr)
{
if (i > 1)
vStringPut (repr, ' ');
vStringPut (repr, c);
}
switch (c)
{
case EOF: token->type = TOKEN_EOF; break;
case '(': token->type = TOKEN_OPEN_PAREN; break;
case ')': token->type = TOKEN_CLOSE_PAREN; break;
case ';': token->type = TOKEN_SEMICOLON; break;
case ',': token->type = TOKEN_COMMA; break;
case '.': token->type = TOKEN_PERIOD; break;
case ':': token->type = TOKEN_COLON; break;
case '{': token->type = TOKEN_OPEN_CURLY; break;
case '}': token->type = TOKEN_CLOSE_CURLY; break;
case '=': token->type = TOKEN_EQUAL_SIGN; break;
case '[': token->type = TOKEN_OPEN_SQUARE; break;
case ']': token->type = TOKEN_CLOSE_SQUARE; break;
case '+':
case '-':
{
int d = fileGetc ();
if (d == c) /* ++ or -- */
token->type = TOKEN_POSTFIX_OPERATOR;
else
{
fileUngetc (d);
token->type = TOKEN_BINARY_OPERATOR;
}
break;
}
case '*':
case '%':
case '?':
case '>':
case '<':
case '^':
case '|':
case '&':
token->type = TOKEN_BINARY_OPERATOR;
break;
case '\r':
case '\n':
/* This isn't strictly correct per the standard, but following the
* real rules means understanding all statements, and that's not
* what the parser currently does. What we do here is a guess, by
* avoiding inserting semicolons that would make the statement on
* the left invalid. Hopefully this should not have false negatives
* (e.g. should not miss insertion of a semicolon) but might have
* false positives (e.g. it will wrongfully emit a semicolon for the
* newline in "foo\n+bar").
* This should however be mostly harmless as we only deal with
* newlines in specific situations where we know a false positive
* wouldn't hurt too bad. */
switch (LastTokenType)
{
/* these cannot be the end of a statement, so hold the newline */
case TOKEN_EQUAL_SIGN:
case TOKEN_COLON:
case TOKEN_PERIOD:
case TOKEN_FORWARD_SLASH:
case TOKEN_BINARY_OPERATOR:
/* and these already end one, no need to duplicate it */
case TOKEN_SEMICOLON:
case TOKEN_COMMA:
case TOKEN_CLOSE_CURLY:
case TOKEN_OPEN_CURLY:
include_newlines = FALSE; /* no need to recheck */
goto getNextChar;
break;
default:
token->type = TOKEN_SEMICOLON;
}
break;
case '\'':
case '"':
token->type = TOKEN_STRING;
parseString (token->string, c);
token->lineNumber = getSourceLineNumber ();
token->filePosition = getInputFilePosition ();
if (repr)
{
vStringCat (repr, token->string);
vStringPut (repr, c);
}
break;
case '\\':
c = fileGetc ();
if (c != '\\' && c != '"' && !isspace (c))
fileUngetc (c);
token->type = TOKEN_CHARACTER;
token->lineNumber = getSourceLineNumber ();
token->filePosition = getInputFilePosition ();
break;
case '/':
{
int d = fileGetc ();
if ( (d != '*') && /* is this the start of a comment? */
(d != '/') ) /* is a one line comment? */
{
fileUngetc (d);
switch (LastTokenType)
{
case TOKEN_CHARACTER:
case TOKEN_IDENTIFIER:
case TOKEN_STRING:
case TOKEN_CLOSE_CURLY:
case TOKEN_CLOSE_PAREN:
case TOKEN_CLOSE_SQUARE:
token->type = TOKEN_FORWARD_SLASH;
break;
default:
token->type = TOKEN_REGEXP;
parseRegExp ();
token->lineNumber = getSourceLineNumber ();
token->filePosition = getInputFilePosition ();
break;
}
}
else
{
if (repr) /* remove the / we added */
repr->buffer[--repr->length] = 0;
if (d == '*')
{
do
{
skipToCharacter ('*');
c = fileGetc ();
if (c == '/')
break;
else
fileUngetc (c);
} while (c != EOF && c != '\0');
goto getNextChar;
}
else if (d == '/') /* is this the start of a comment? */
{
skipToCharacter ('\n');
/* if we care about newlines, put it back so it is seen */
if (include_newlines)
fileUngetc ('\n');
goto getNextChar;
}
}
break;
}
case '#':
/* skip shebang in case of e.g. Node.js scripts */
if (token->lineNumber > 1)
token->type = TOKEN_UNDEFINED;
else if ((c = fileGetc ()) != '!')
{
fileUngetc (c);
token->type = TOKEN_UNDEFINED;
}
else
{
skipToCharacter ('\n');
goto getNextChar;
}
break;
default:
if (! isIdentChar (c))
token->type = TOKEN_UNDEFINED;
else
{
parseIdentifier (token->string, c);
token->lineNumber = getSourceLineNumber ();
token->filePosition = getInputFilePosition ();
token->keyword = analyzeToken (token->string);
if (isKeyword (token, KEYWORD_NONE))
token->type = TOKEN_IDENTIFIER;
else
token->type = TOKEN_KEYWORD;
if (repr && vStringLength (token->string) > 1)
vStringCatS (repr, vStringValue (token->string) + 1);
}
break;
}
LastTokenType = token->type;
}
/* Algorithm adapted from from GNU etags.
* Perl support by Bart Robinson <*****@*****.**>
* Perl sub names: look for /^ [ \t\n]sub [ \t\n]+ [^ \t\n{ (]+/
*/
static void findPerlTags (void)
{
vString *name = vStringNew ();
vString *package = NULL;
boolean skipPodDoc = FALSE;
const unsigned char *line;
/* Core modules AutoLoader and SelfLoader support delayed compilation
* by allowing Perl code that follows __END__ and __DATA__ tokens,
* respectively. When we detect that one of these modules is used
* in the file, we continue processing even after we see the
* corresponding token that would usually terminate parsing of the
* file.
*/
enum {
RESPECT_END = (1 << 0),
RESPECT_DATA = (1 << 1),
} respect_token = RESPECT_END | RESPECT_DATA;
while ((line = fileReadLine ()) != NULL)
{
boolean spaceRequired = FALSE;
boolean qualified = FALSE;
const unsigned char *cp = line;
perlKind kind = K_NONE;
tagEntryInfo e;
if (skipPodDoc)
{
if (strncmp ((const char*) line, "=cut", (size_t) 4) == 0)
skipPodDoc = FALSE;
continue;
}
else if (line [0] == '=')
{
skipPodDoc = isPodWord ((const char*)line + 1);
continue;
}
else if (strcmp ((const char*) line, "__DATA__") == 0)
{
if (respect_token & RESPECT_DATA)
break;
else
continue;
}
else if (strcmp ((const char*) line, "__END__") == 0)
{
if (respect_token & RESPECT_END)
break;
else
continue;
}
else if (line [0] == '#')
continue;
while (isspace (*cp))
cp++;
if (strncmp((const char*) cp, "sub", (size_t) 3) == 0)
{
TRACE("this looks like a sub\n");
cp += 3;
kind = K_SUBROUTINE;
spaceRequired = TRUE;
qualified = TRUE;
}
else if (strncmp((const char*) cp, "use", (size_t) 3) == 0)
{
cp += 3;
if (!isspace(*cp))
continue;
while (*cp && isspace (*cp))
++cp;
if (strncmp((const char*) cp, "AutoLoader", (size_t) 10) == 0) {
respect_token &= ~RESPECT_END;
continue;
}
if (strncmp((const char*) cp, "SelfLoader", (size_t) 10) == 0) {
respect_token &= ~RESPECT_DATA;
continue;
}
if (strncmp((const char*) cp, "constant", (size_t) 8) != 0)
continue;
cp += 8;
/* Skip up to the first non-space character, skipping empty
* and comment lines.
*/
while (isspace(*cp))
cp++;
while (!*cp || '#' == *cp) {
cp = fileReadLine ();
if (!cp)
goto END_MAIN_WHILE;
while (isspace (*cp))
cp++;
}
if ('{' == *cp) {
++cp;
if (0 == parseConstantsFromHashRef(cp, name, package)) {
vStringClear(name);
continue;
} else
goto END_MAIN_WHILE;
}
kind = K_CONSTANT;
spaceRequired = FALSE;
qualified = TRUE;
}
else if (strncmp((const char*) cp, "package", (size_t) 7) == 0 &&
('\0' == cp[7] || isspace(cp[7])))
{
cp += 7;
while (isspace (*cp))
cp++;
while (!*cp || '#' == *cp) {
cp = fileReadLine ();
if (!cp)
goto END_MAIN_WHILE;
while (isspace (*cp))
cp++;
}
if (package == NULL)
package = vStringNew ();
else
vStringClear (package);
const unsigned char *const first = cp;
while (*cp && (int) *cp != ';' && !isspace ((int) *cp))
{
vStringPut (package, (int) *cp);
cp++;
}
vStringCatS (package, "::");
cp = first; /* Rewind */
kind = K_PACKAGE;
spaceRequired = FALSE;
qualified = TRUE;
}
else if (strncmp((const char*) cp, "format", (size_t) 6) == 0)
{
cp += 6;
kind = K_FORMAT;
spaceRequired = TRUE;
qualified = TRUE;
}
else
{
if (isIdentifier1 (*cp))
{
const unsigned char *p = cp;
while (isIdentifier (*p))
++p;
while (isspace (*p))
++p;
if ((int) *p == ':' && (int) *(p + 1) != ':')
kind = K_LABEL;
}
}
if (kind != K_NONE)
{
TRACE("cp0: %s\n", (const char *) cp);
if (spaceRequired && *cp && !isspace (*cp))
continue;
TRACE("cp1: %s\n", (const char *) cp);
while (isspace (*cp))
cp++;
while (!*cp || '#' == *cp) { /* Gobble up empty lines
and comments */
cp = fileReadLine ();
if (!cp)
goto END_MAIN_WHILE;
while (isspace (*cp))
cp++;
}
while (isIdentifier (*cp) || (K_PACKAGE == kind && ':' == *cp))
{
vStringPut (name, (int) *cp);
cp++;
}
if (K_FORMAT == kind &&
vStringLength (name) == 0 && /* cp did not advance */
'=' == *cp)
{
/* format's name is optional. If it's omitted, 'STDOUT'
is assumed. */
vStringCatS (name, "STDOUT");
}
vStringTerminate (name);
TRACE("name: %s\n", name->buffer);
if (0 == vStringLength(name)) {
vStringClear(name);
continue;
}
if (K_SUBROUTINE == kind)
{
/*
* isSubroutineDeclaration() may consume several lines. So
* we record line positions.
*/
initTagEntry(&e, vStringValue(name), NULL);
if (TRUE == isSubroutineDeclaration(cp)) {
if (TRUE == PerlKinds[K_SUBROUTINE_DECLARATION].enabled) {
kind = K_SUBROUTINE_DECLARATION;
} else {
vStringClear (name);
continue;
}
} else if (! PerlKinds[kind].enabled) {
continue;
}
e.kind = &(PerlKinds[kind]);
makeTagEntry(&e);
if (Option.include.qualifiedTags && qualified &&
package != NULL && vStringLength (package) > 0)
{
vString *const qualifiedName = vStringNew ();
vStringCopy (qualifiedName, package);
vStringCat (qualifiedName, name);
e.name = vStringValue(qualifiedName);
makeTagEntry(&e);
vStringDelete (qualifiedName);
}
} else if (vStringLength (name) > 0)
{
makeSimpleTag (name, PerlKinds, kind);
if (Option.include.qualifiedTags && qualified &&
K_PACKAGE != kind &&
package != NULL && vStringLength (package) > 0)
{
vString *const qualifiedName = vStringNew ();
vStringCopy (qualifiedName, package);
vStringCat (qualifiedName, name);
makeSimpleTag (qualifiedName, PerlKinds, kind);
vStringDelete (qualifiedName);
}
}
vStringClear (name);
}
}
END_MAIN_WHILE:
vStringDelete (name);
if (package != NULL)
vStringDelete (package);
}
/* Adds a name to the end of the scope string */
static void addToScope (vString *scope, vString *name)
{
if (vStringLength(scope) > 0)
vStringCatS(scope, "::");
vStringCat(scope, name);
}
/* Structs and enums are very similar syntax-wise.
* It is possible to parse variants a bit more cleverly (e.g. make tuple variants functions and
* struct variants structs) but it'd be too clever and the signature wouldn't make too much sense without
* the enum's definition (e.g. for the type bounds)
*
* Struct/Enum format:
* "struct/enum" <ident>[<type_bounds>] "{" [<ident>,]+ "}"
* "struct/enum" <ident>[<type_bounds>] ";"
* */
static void parseStructOrEnum (lexerState *lexer, vString *scope, int parent_kind, boolean is_struct)
{
int kind = is_struct ? K_STRUCT : K_ENUM;
int field_kind = is_struct ? K_FIELD : K_VARIANT;
int goal_tokens1[] = {';', '{'};
advanceToken(lexer, TRUE);
if (lexer->cur_token != TOKEN_IDENT)
return;
addTag(lexer->token_str, NULL, kind, lexer->line, lexer->pos, scope, parent_kind);
addToScope(scope, lexer->token_str);
skipUntil(lexer, goal_tokens1, 2);
if (lexer->cur_token == '{')
{
vString *field_name = vStringNew();
while (lexer->cur_token != TOKEN_EOF)
{
int goal_tokens2[] = {'}', ','};
/* Skip attributes. Format:
* #[..] or #![..]
* */
if (lexer->cur_token == '#')
{
advanceToken(lexer, TRUE);
if (lexer->cur_token == '!')
advanceToken(lexer, TRUE);
if (lexer->cur_token == '[')
{
/* It's an attribute, skip it. */
skipUntil(lexer, NULL, 0);
}
else
{
/* Something's up with this field, skip to the next one */
skipUntil(lexer, goal_tokens2, 2);
continue;
}
}
if (lexer->cur_token == TOKEN_IDENT)
{
if (strcmp(lexer->token_str->buffer, "priv") == 0
|| strcmp(lexer->token_str->buffer, "pub") == 0)
{
advanceToken(lexer, TRUE);
if (lexer->cur_token != TOKEN_IDENT)
{
/* Something's up with this field, skip to the next one */
skipUntil(lexer, goal_tokens2, 2);
continue;
}
}
vStringClear(field_name);
vStringCat(field_name, lexer->token_str);
addTag(field_name, NULL, field_kind, lexer->line, lexer->pos, scope, kind);
skipUntil(lexer, goal_tokens2, 2);
}
if (lexer->cur_token == '}')
{
advanceToken(lexer, TRUE);
break;
}
advanceToken(lexer, TRUE);
}
vStringDelete(field_name);
}
}
static void readTokenFull (tokenInfo *const token, boolean include_newlines, vString *const repr)
{
int c;
int i;
boolean newline_encountered = FALSE;
/* if we've got a token held back, emit it */
if (NextToken)
{
copyToken (token, NextToken, FALSE);
deleteToken (NextToken);
NextToken = NULL;
return;
}
token->type = TOKEN_UNDEFINED;
token->keyword = KEYWORD_NONE;
vStringClear (token->string);
getNextChar:
i = 0;
do
{
c = fileGetc ();
if (include_newlines && (c == '\r' || c == '\n'))
newline_encountered = TRUE;
i++;
}
while (c == '\t' || c == ' ' || c == '\r' || c == '\n');
token->lineNumber = getSourceLineNumber ();
token->filePosition = getInputFilePosition ();
if (repr)
{
if (i > 1)
vStringPut (repr, ' ');
vStringPut (repr, c);
}
switch (c)
{
case EOF: token->type = TOKEN_EOF; break;
case '(': token->type = TOKEN_OPEN_PAREN; break;
case ')': token->type = TOKEN_CLOSE_PAREN; break;
case ';': token->type = TOKEN_SEMICOLON; break;
case ',': token->type = TOKEN_COMMA; break;
case '.': token->type = TOKEN_PERIOD; break;
case ':': token->type = TOKEN_COLON; break;
case '{': token->type = TOKEN_OPEN_CURLY; break;
case '}': token->type = TOKEN_CLOSE_CURLY; break;
case '=': token->type = TOKEN_EQUAL_SIGN; break;
case '[': token->type = TOKEN_OPEN_SQUARE; break;
case ']': token->type = TOKEN_CLOSE_SQUARE; break;
case '+':
case '-':
{
int d = fileGetc ();
if (d == c) /* ++ or -- */
token->type = TOKEN_POSTFIX_OPERATOR;
else
{
fileUngetc (d);
token->type = TOKEN_BINARY_OPERATOR;
}
break;
}
case '*':
case '%':
case '?':
case '>':
case '<':
case '^':
case '|':
case '&':
token->type = TOKEN_BINARY_OPERATOR;
break;
case '\'':
case '"':
token->type = TOKEN_STRING;
parseString (token->string, c);
token->lineNumber = getSourceLineNumber ();
token->filePosition = getInputFilePosition ();
if (repr)
{
vStringCat (repr, token->string);
vStringPut (repr, c);
}
break;
case '`':
token->type = TOKEN_TEMPLATE_STRING;
parseTemplateString (token->string);
token->lineNumber = getSourceLineNumber ();
token->filePosition = getInputFilePosition ();
if (repr)
{
vStringCat (repr, token->string);
vStringPut (repr, c);
}
break;
case '\\':
c = fileGetc ();
if (c != '\\' && c != '"' && !isspace (c))
fileUngetc (c);
token->type = TOKEN_CHARACTER;
token->lineNumber = getSourceLineNumber ();
token->filePosition = getInputFilePosition ();
break;
case '/':
{
int d = fileGetc ();
if ( (d != '*') && /* is this the start of a comment? */
(d != '/') ) /* is a one line comment? */
{
fileUngetc (d);
switch (LastTokenType)
{
case TOKEN_CHARACTER:
case TOKEN_IDENTIFIER:
case TOKEN_STRING:
case TOKEN_TEMPLATE_STRING:
case TOKEN_CLOSE_CURLY:
case TOKEN_CLOSE_PAREN:
case TOKEN_CLOSE_SQUARE:
token->type = TOKEN_BINARY_OPERATOR;
break;
default:
token->type = TOKEN_REGEXP;
parseRegExp ();
token->lineNumber = getSourceLineNumber ();
token->filePosition = getInputFilePosition ();
break;
}
}
else
{
if (repr) /* remove the / we added */
repr->buffer[--repr->length] = 0;
if (d == '*')
{
do
{
fileSkipToCharacter ('*');
c = fileGetc ();
if (c == '/')
break;
else
fileUngetc (c);
} while (c != EOF && c != '\0');
goto getNextChar;
}
else if (d == '/') /* is this the start of a comment? */
{
fileSkipToCharacter ('\n');
/* if we care about newlines, put it back so it is seen */
if (include_newlines)
fileUngetc ('\n');
goto getNextChar;
}
}
break;
}
case '#':
/* skip shebang in case of e.g. Node.js scripts */
if (token->lineNumber > 1)
token->type = TOKEN_UNDEFINED;
else if ((c = fileGetc ()) != '!')
{
fileUngetc (c);
token->type = TOKEN_UNDEFINED;
}
else
{
fileSkipToCharacter ('\n');
goto getNextChar;
}
break;
default:
if (! isIdentChar (c))
token->type = TOKEN_UNDEFINED;
else
{
parseIdentifier (token->string, c);
token->lineNumber = getSourceLineNumber ();
token->filePosition = getInputFilePosition ();
token->keyword = analyzeToken (token->string, Lang_js);
if (isKeyword (token, KEYWORD_NONE))
token->type = TOKEN_IDENTIFIER;
else
token->type = TOKEN_KEYWORD;
if (repr && vStringLength (token->string) > 1)
vStringCatS (repr, vStringValue (token->string) + 1);
}
break;
}
if (include_newlines && newline_encountered)
{
/* This isn't strictly correct per the standard, but following the
* real rules means understanding all statements, and that's not
* what the parser currently does. What we do here is a guess, by
* avoiding inserting semicolons that would make the statement on
* the left or right obviously invalid. Hopefully this should not
* have false negatives (e.g. should not miss insertion of a semicolon)
* but might have false positives (e.g. it will wrongfully emit a
* semicolon sometimes, i.e. for the newline in "foo\n(bar)").
* This should however be mostly harmless as we only deal with
* newlines in specific situations where we know a false positive
* wouldn't hurt too bad. */
/* these already end a statement, so no need to duplicate it */
#define IS_STMT_SEPARATOR(t) ((t) == TOKEN_SEMICOLON || \
(t) == TOKEN_EOF || \
(t) == TOKEN_COMMA || \
(t) == TOKEN_CLOSE_CURLY || \
(t) == TOKEN_OPEN_CURLY)
/* these cannot be the start or end of a statement */
#define IS_BINARY_OPERATOR(t) ((t) == TOKEN_EQUAL_SIGN || \
(t) == TOKEN_COLON || \
(t) == TOKEN_PERIOD || \
(t) == TOKEN_BINARY_OPERATOR)
if (! IS_STMT_SEPARATOR(LastTokenType) &&
! IS_STMT_SEPARATOR(token->type) &&
! IS_BINARY_OPERATOR(LastTokenType) &&
! IS_BINARY_OPERATOR(token->type) &&
/* these cannot be followed by a semicolon */
! (LastTokenType == TOKEN_OPEN_PAREN ||
LastTokenType == TOKEN_OPEN_SQUARE))
{
/* hold the token... */
Assert (NextToken == NULL);
NextToken = newToken ();
copyToken (NextToken, token, FALSE);
/* ...and emit a semicolon instead */
token->type = TOKEN_SEMICOLON;
token->keyword = KEYWORD_NONE;
vStringClear (token->string);
if (repr)
vStringPut (token->string, '\n');
}
#undef IS_STMT_SEPARATOR
#undef IS_BINARY_OPERATOR
}
LastTokenType = token->type;
}
/* Algorithm adapted from from GNU etags.
* Perl support by Bart Robinson <*****@*****.**>
* Perl sub names: look for /^ [ \t\n]sub [ \t\n]+ [^ \t\n{ (]+/
*/
static void findPerlTags (void)
{
vString *name = vStringNew ();
vString *package = NULL;
boolean skipPodDoc = FALSE;
const unsigned char *line;
while ((line = fileReadLine ()) != NULL)
{
boolean spaceRequired = FALSE;
boolean qualified = FALSE;
const unsigned char *cp = line;
perlKind kind = K_NONE;
tagEntryInfo e;
if (skipPodDoc)
{
if (strncmp ((const char*) line, "=cut", (size_t) 4) == 0)
skipPodDoc = FALSE;
continue;
}
else if (line [0] == '=')
{
skipPodDoc = isPodWord ((const char*)line + 1);
continue;
}
else if (strcmp ((const char*) line, "__DATA__") == 0)
break;
else if (strcmp ((const char*) line, "__END__") == 0)
break;
else if (line [0] == '#')
continue;
while (isspace (*cp))
cp++;
if (strncmp((const char*) cp, "sub", (size_t) 3) == 0)
{
TRACE("this looks like a sub\n");
cp += 3;
kind = K_SUBROUTINE;
spaceRequired = TRUE;
qualified = TRUE;
}
else if (strncmp((const char*) cp, "use", (size_t) 3) == 0)
{
cp += 3;
if (!isspace(*cp))
continue;
while (*cp && isspace (*cp))
++cp;
if (strncmp((const char*) cp, "constant", (size_t) 8) != 0)
continue;
cp += 8;
/* <<<<<<< HEAD */
/* kind = K_CONSTANT; */
/* spaceRequired = TRUE; */
/* ======= */
/* Skip up to the first non-space character, skipping empty
* and comment lines.
*/
while (isspace(*cp))
cp++;
while (!*cp || '#' == *cp) {
cp = fileReadLine ();
if (!cp)
goto END_MAIN_WHILE;
while (isspace (*cp))
cp++;
}
if ('{' == *cp) {
++cp;
if (0 == parseConstantsFromHashRef(cp, name, package)) {
vStringClear(name);
continue;
} else
goto END_MAIN_WHILE;
}
kind = K_CONSTANT;
spaceRequired = FALSE;
/* >>>>>>> f039977edb44f3d46aacd26cc486ce153b879b0e */
qualified = TRUE;
}
else if (strncmp((const char*) cp, "package", (size_t) 7) == 0)
{
/* This will point to space after 'package' so that a tag
can be made */
const unsigned char *space = cp += 7;
if (package == NULL)
package = vStringNew ();
else
vStringClear (package);
while (isspace (*cp))
cp++;
/* <<<<<<< HEAD */
/* while ((int) *cp != ';' && !isspace ((int) *cp)) */
/* ======= */
while (*cp && (int) *cp != ';' && !isspace ((int) *cp))
/* >>>>>>> f039977edb44f3d46aacd26cc486ce153b879b0e */
{
vStringPut (package, (int) *cp);
cp++;
}
vStringCatS (package, "::");
cp = space; /* Rewind */
kind = K_PACKAGE;
spaceRequired = TRUE;
qualified = TRUE;
}
else if (strncmp((const char*) cp, "format", (size_t) 6) == 0)
{
cp += 6;
kind = K_FORMAT;
spaceRequired = TRUE;
qualified = TRUE;
}
else
{
if (isIdentifier1 (*cp))
{
const unsigned char *p = cp;
while (isIdentifier (*p))
++p;
while (isspace (*p))
++p;
if ((int) *p == ':' && (int) *(p + 1) != ':')
kind = K_LABEL;
}
}
if (kind != K_NONE)
{
TRACE("cp0: %s\n", (const char *) cp);
if (spaceRequired && *cp && !isspace (*cp))
continue;
TRACE("cp1: %s\n", (const char *) cp);
while (isspace (*cp))
cp++;
while (!*cp || '#' == *cp) { /* Gobble up empty lines
and comments */
cp = fileReadLine ();
if (!cp)
goto END_MAIN_WHILE;
while (isspace (*cp))
cp++;
}
while (isIdentifier (*cp) || (K_PACKAGE == kind && ':' == *cp))
{
vStringPut (name, (int) *cp);
cp++;
}
if (K_FORMAT == kind &&
vStringLength (name) == 0 && /* cp did not advance */
'=' == *cp)
{
/* format's name is optional. If it's omitted, 'STDOUT'
is assumed. */
vStringCatS (name, "STDOUT");
}
vStringTerminate (name);
TRACE("name: %s\n", name->buffer);
if (0 == vStringLength(name)) {
vStringClear(name);
continue;
}
if (K_SUBROUTINE == kind)
{
/*
* isSubroutineDeclaration() may consume several lines. So
* we record line positions.
*/
initTagEntry(&e, vStringValue(name));
if (TRUE == isSubroutineDeclaration(cp)) {
if (TRUE == PerlKinds[K_SUBROUTINE_DECLARATION].enabled) {
kind = K_SUBROUTINE_DECLARATION;
} else {
vStringClear (name);
continue;
}
}
e.kind = PerlKinds[kind].letter;
e.kindName = PerlKinds[kind].name;
makeTagEntry(&e);
if (Option.include.qualifiedTags && qualified &&
package != NULL && vStringLength (package) > 0)
{
vString *const qualifiedName = vStringNew ();
vStringCopy (qualifiedName, package);
vStringCat (qualifiedName, name);
e.name = vStringValue(qualifiedName);
makeTagEntry(&e);
vStringDelete (qualifiedName);
}
} else if (vStringLength (name) > 0)
{
makeSimpleTag (name, PerlKinds, kind);
if (Option.include.qualifiedTags && qualified &&
K_PACKAGE != kind &&
package != NULL && vStringLength (package) > 0)
{
vString *const qualifiedName = vStringNew ();
vStringCopy (qualifiedName, package);
vStringCat (qualifiedName, name);
makeSimpleTag (qualifiedName, PerlKinds, kind);
vStringDelete (qualifiedName);
}
}
vStringClear (name);
}
}
END_MAIN_WHILE:
vStringDelete (name);
if (package != NULL)
vStringDelete (package);
}
/* Algorithm adapted from from GNU etags.
* Perl support by Bart Robinson <*****@*****.**>
* Perl sub names: look for /^ [ \t\n]sub [ \t\n]+ [^ \t\n{ (]+/
*/
static void findPerlTags (void)
{
vString *name = vStringNew ();
vString *package = NULL;
bool skipPodDoc = false;
const unsigned char *line;
while ((line = readLineFromInputFile ()) != NULL)
{
bool spaceRequired = false;
bool qualified = false;
const unsigned char *cp = line;
perlKind kind = K_NONE;
tagEntryInfo e;
if (skipPodDoc)
{
if (strncmp ((const char*) line, "=cut", (size_t) 4) == 0)
skipPodDoc = false;
continue;
}
else if (line [0] == '=')
{
skipPodDoc = isPodWord ((const char*)line + 1);
continue;
}
else if (strcmp ((const char*) line, "__DATA__") == 0)
break;
else if (strcmp ((const char*) line, "__END__") == 0)
break;
else if (line [0] == '#')
continue;
while (isspace (*cp))
cp++;
if (strncmp((const char*) cp, "sub", (size_t) 3) == 0)
{
TRACE("this looks like a sub\n");
cp += 3;
kind = K_SUBROUTINE;
spaceRequired = true;
qualified = true;
}
else if (strncmp((const char*) cp, "use", (size_t) 3) == 0)
{
cp += 3;
if (!isspace(*cp))
continue;
while (*cp && isspace (*cp))
++cp;
if (strncmp((const char*) cp, "constant", (size_t) 8) != 0)
continue;
cp += 8;
kind = K_CONSTANT;
spaceRequired = true;
qualified = true;
}
else if (strncmp((const char*) cp, "package", (size_t) 7) == 0)
{
/* This will point to space after 'package' so that a tag
can be made */
const unsigned char *space = cp += 7;
if (package == NULL)
package = vStringNew ();
else
vStringClear (package);
while (isspace (*cp))
cp++;
while ((int) *cp != ';' && !isspace ((int) *cp))
{
vStringPut (package, (int) *cp);
cp++;
}
vStringCatS (package, "::");
cp = space; /* Rewind */
kind = K_PACKAGE;
spaceRequired = true;
qualified = true;
}
else if (strncmp((const char*) cp, "format", (size_t) 6) == 0)
{
cp += 6;
kind = K_FORMAT;
spaceRequired = true;
qualified = true;
}
else
{
if (isIdentifier1 (*cp))
{
const unsigned char *p = cp;
while (isIdentifier (*p))
++p;
while (isspace (*p))
++p;
if ((int) *p == ':' && (int) *(p + 1) != ':')
kind = K_LABEL;
}
}
if (kind != K_NONE)
{
TRACE("cp0: %s\n", (const char *) cp);
if (spaceRequired && *cp && !isspace (*cp))
continue;
TRACE("cp1: %s\n", (const char *) cp);
while (isspace (*cp))
cp++;
while (!*cp || '#' == *cp) { /* Gobble up empty lines
and comments */
cp = readLineFromInputFile ();
if (!cp)
goto END_MAIN_WHILE;
while (isspace (*cp))
cp++;
}
while (isIdentifier (*cp) || (K_PACKAGE == kind && ':' == *cp))
{
vStringPut (name, (int) *cp);
cp++;
}
if (K_FORMAT == kind &&
vStringLength (name) == 0 && /* cp did not advance */
'=' == *cp)
{
/* format's name is optional. If it's omitted, 'STDOUT'
is assumed. */
vStringCatS (name, "STDOUT");
}
TRACE("name: %s\n", name->buffer);
if (0 == vStringLength(name)) {
vStringClear(name);
continue;
}
if (K_SUBROUTINE == kind)
{
/*
* isSubroutineDeclaration() may consume several lines. So
* we record line positions.
*/
initTagEntry(&e, vStringValue(name), &(PerlKinds[kind]));
if (true == isSubroutineDeclaration(cp)) {
if (true == PerlKinds[K_SUBROUTINE_DECLARATION].enabled) {
kind = K_SUBROUTINE_DECLARATION;
} else {
vStringClear (name);
continue;
}
}
makeTagEntry(&e);
if (isXtagEnabled(XTAG_QUALIFIED_TAGS) && qualified &&
package != NULL && vStringLength (package) > 0)
{
vString *const qualifiedName = vStringNew ();
vStringCopy (qualifiedName, package);
vStringCat (qualifiedName, name);
e.name = vStringValue(qualifiedName);
makeTagEntry(&e);
vStringDelete (qualifiedName);
}
} else if (vStringLength (name) > 0)
{
makeSimpleTag (name, PerlKinds, kind);
if (isXtagEnabled(XTAG_QUALIFIED_TAGS) && qualified &&
K_PACKAGE != kind &&
package != NULL && vStringLength (package) > 0)
{
vString *const qualifiedName = vStringNew ();
vStringCopy (qualifiedName, package);
vStringCat (qualifiedName, name);
makeSimpleTag (qualifiedName, PerlKinds, kind);
vStringDelete (qualifiedName);
}
}
vStringClear (name);
}
}
END_MAIN_WHILE:
vStringDelete (name);
if (package != NULL)
vStringDelete (package);
}
/* parse a function
*
* if @name is NULL, parses a normal function
* function myfunc($foo, $bar) {}
* function &myfunc($foo, $bar) {}
* function myfunc($foo, $bar) : type {}
*
* if @name is not NULL, parses an anonymous function with name @name
* $foo = function($foo, $bar) {}
* $foo = function&($foo, $bar) {}
* $foo = function($foo, $bar) use ($x, &$y) {}
* $foo = function($foo, $bar) use ($x, &$y) : type {} */
static boolean parseFunction (tokenInfo *const token, const tokenInfo *name)
{
boolean readNext = TRUE;
accessType access = CurrentStatement.access;
implType impl = CurrentStatement.impl;
tokenInfo *nameFree = NULL;
readToken (token);
/* skip a possible leading ampersand (return by reference) */
if (token->type == TOKEN_AMPERSAND)
readToken (token);
if (! name)
{
if (token->type != TOKEN_IDENTIFIER && token->type != TOKEN_KEYWORD)
return FALSE;
name = nameFree = newToken ();
copyToken (nameFree, token, TRUE);
readToken (token);
}
if (token->type == TOKEN_OPEN_PAREN)
{
vString *arglist = vStringNew ();
int depth = 1;
vStringPut (arglist, '(');
do
{
readToken (token);
switch (token->type)
{
case TOKEN_OPEN_PAREN: depth++; break;
case TOKEN_CLOSE_PAREN: depth--; break;
default: break;
}
/* display part */
switch (token->type)
{
case TOKEN_AMPERSAND: vStringPut (arglist, '&'); break;
case TOKEN_CLOSE_CURLY: vStringPut (arglist, '}'); break;
case TOKEN_CLOSE_PAREN: vStringPut (arglist, ')'); break;
case TOKEN_CLOSE_SQUARE: vStringPut (arglist, ']'); break;
case TOKEN_COLON: vStringPut (arglist, ':'); break;
case TOKEN_COMMA: vStringCatS (arglist, ", "); break;
case TOKEN_EQUAL_SIGN: vStringCatS (arglist, " = "); break;
case TOKEN_OPEN_CURLY: vStringPut (arglist, '{'); break;
case TOKEN_OPEN_PAREN: vStringPut (arglist, '('); break;
case TOKEN_OPEN_SQUARE: vStringPut (arglist, '['); break;
case TOKEN_PERIOD: vStringPut (arglist, '.'); break;
case TOKEN_SEMICOLON: vStringPut (arglist, ';'); break;
case TOKEN_BACKSLASH: vStringPut (arglist, '\\'); break;
case TOKEN_STRING:
{
vStringCatS (arglist, "'");
vStringCat (arglist, token->string);
vStringCatS (arglist, "'");
break;
}
case TOKEN_IDENTIFIER:
case TOKEN_KEYWORD:
case TOKEN_VARIABLE:
{
switch (vStringLast (arglist))
{
case 0:
case ' ':
case '{':
case '(':
case '[':
case '.':
case '\\':
/* no need for a space between those and the identifier */
break;
default:
vStringPut (arglist, ' ');
break;
}
if (token->type == TOKEN_VARIABLE)
vStringPut (arglist, '$');
vStringCat (arglist, token->string);
break;
}
default: break;
}
}
while (token->type != TOKEN_EOF && depth > 0);
vStringTerminate (arglist);
makeFunctionTag (name, arglist, access, impl);
vStringDelete (arglist);
readToken (token); /* normally it's an open brace or "use" keyword */
}
/* skip use(...) */
if (token->type == TOKEN_KEYWORD && token->keyword == KEYWORD_use)
{
readToken (token);
skipOverParens (token);
}
/* PHP7 return type declaration or if parsing Zephir, skip function return
* type hint */
if ((getInputLanguage () == Lang_php && token->type == TOKEN_COLON) ||
(getInputLanguage () == Lang_zephir && token->type == TOKEN_OPERATOR))
{
do
readToken (token);
while (token->type == TOKEN_IDENTIFIER ||
token->type == TOKEN_BACKSLASH);
}
if (token->type == TOKEN_OPEN_CURLY)
enterScope (token, name->string, K_FUNCTION);
else
readNext = FALSE;
if (nameFree)
deleteToken (nameFree);
return readNext;
}
static void readToken (tokenInfo *const token)
{
int c;
token->type = TOKEN_UNDEFINED;
token->keyword = KEYWORD_NONE;
vStringClear (token->string);
getNextChar:
do
c = fileGetc ();
while (c == '\t' || c == ' ' || c == '\n');
switch (c)
{
case EOF:
longjmp (Exception, (int)ExceptionEOF);
break;
case '!':
token->type = TOKEN_BANG;
break;
case '$':
token->type = TOKEN_DOLLAR;
break;
case '(':
token->type = TOKEN_OPEN_PAREN;
break;
case ')':
token->type = TOKEN_CLOSE_PAREN;
break;
case ',':
token->type = TOKEN_COMMA;
break;
case '.':
token->type = TOKEN_DOT;
break;
case ';':
goto getNextChar;
case '[':
token->type = TOKEN_OPEN_BRACKET;
break;
case ']':
token->type = TOKEN_CLOSE_BRACKET;
break;
case '{':
token->type = TOKEN_OPEN_BRACE;
break;
case '}':
token->type = TOKEN_CLOSE_BRACE;
break;
case '~':
token->type = TOKEN_TILDE;
break;
case '+':
case '*':
case '^':
case '=':
token->type = TOKEN_OPERATOR;
break;
case '-':
c = fileGetc ();
if (c == '>')
token->type = TOKEN_CONSTRAINT;
else if (c == '-') /* is this the start of a comment? */
{
skipToCharacter ('\n');
goto getNextChar;
}
else
{
if (!isspace (c))
fileUngetc (c);
token->type = TOKEN_OPERATOR;
}
break;
case '?':
case ':':
c = fileGetc ();
if (c == '=')
token->type = TOKEN_OPERATOR;
else
{
token->type = TOKEN_COLON;
if (!isspace (c))
fileUngetc (c);
}
break;
case '<':
c = fileGetc ();
if (c != '=' && c != '>' && !isspace (c))
fileUngetc (c);
token->type = TOKEN_OPERATOR;
break;
case '>':
c = fileGetc ();
if (c != '=' && c != '>' && !isspace (c))
fileUngetc (c);
token->type = TOKEN_OPERATOR;
break;
case '/':
c = fileGetc ();
if (c != '/' && c != '=' && !isspace (c))
fileUngetc (c);
token->type = TOKEN_OPERATOR;
break;
case '\\':
c = fileGetc ();
if (c != '\\' && !isspace (c))
fileUngetc (c);
token->type = TOKEN_OPERATOR;
break;
case '"':
token->type = TOKEN_STRING;
parseString (token->string);
break;
case '\'':
token->type = TOKEN_CHARACTER;
parseCharacter ();
break;
default:
if (isalpha (c))
{
parseIdentifier (token->string, c);
token->keyword = analyzeToken (token->string);
if (isKeyword (token, KEYWORD_NONE))
token->type = TOKEN_IDENTIFIER;
else
token->type = TOKEN_KEYWORD;
}
else if (isdigit (c))
{
vStringCat (token->string, parseNumeric (c));
token->type = TOKEN_NUMERIC;
}
else if (isFreeOperatorChar (c))
{
parseFreeOperator (token->string, c);
token->type = TOKEN_OPERATOR;
}
else
{
token->type = TOKEN_UNDEFINED;
Assert (! isType (token, TOKEN_UNDEFINED));
}
break;
}
}
static void findCssTags (void)
{
bool readNextToken = true;
tokenInfo token;
token.string = vStringNew ();
do
{
if (readNextToken)
readToken (&token);
readNextToken = true;
if (token.type == '@')
{ /* At-rules, from the "@" to the next block or semicolon */
bool useContents;
readToken (&token);
useContents = (strcmp (vStringValue (token.string), "media") == 0 ||
strcmp (vStringValue (token.string), "supports") == 0);
while (token.type != TOKEN_EOF &&
token.type != ';' && token.type != '{')
{
readToken (&token);
}
/* HACK: we *eat* the opening '{' for medias and the like so that
* the content is parsed as if it was at the root */
readNextToken = useContents && token.type == '{';
}
else if (token.type == TOKEN_SELECTOR)
{ /* collect selectors and make a tag */
cssKind kind = K_SELECTOR;
MIOPos filePosition;
unsigned long lineNumber;
vString *selector = vStringNew ();
do
{
if (vStringLength (selector) > 0)
vStringPut (selector, ' ');
vStringCat (selector, token.string);
kind = classifySelector (token.string);
lineNumber = getInputLineNumber ();
filePosition = getInputFilePosition ();
readToken (&token);
/* handle attribute selectors */
if (token.type == '[')
{
int depth = 1;
while (depth > 0 && token.type != TOKEN_EOF)
{
vStringCat (selector, token.string);
readToken (&token);
if (token.type == '[')
depth++;
else if (token.type == ']')
depth--;
}
if (token.type != TOKEN_EOF)
vStringCat (selector, token.string);
readToken (&token);
}
}
while (token.type == TOKEN_SELECTOR);
/* we already consumed the next token, don't read it twice */
readNextToken = false;
if (CssKinds[kind].enabled)
{
tagEntryInfo e;
initTagEntry (&e, vStringValue (selector), &(CssKinds[kind]));
e.lineNumber = lineNumber;
e.filePosition = filePosition;
makeTagEntry (&e);
}
vStringDelete (selector);
}
else if (token.type == '{')
{ /* skip over { ... } */
int depth = 1;
while (depth > 0 && token.type != TOKEN_EOF)
{
readToken (&token);
if (token.type == '{')
depth++;
else if (token.type == '}')
depth--;
}
}
}
while (token.type != TOKEN_EOF);
vStringDelete (token.string);
}
static void readToken (tokenInfo *const token)
{
int c;
token->type = TOKEN_UNDEFINED;
token->keyword = KEYWORD_NONE;
vStringClear (token->string);
getNextChar:
do
c = getcFromInputFile ();
while (c == '\t' || c == ' ' || c == '\n');
switch (c)
{
case EOF: token->type = TOKEN_EOF; break;
case ';': token->type = TOKEN_SEMICOLON; break;
case '!': token->type = TOKEN_BANG; break;
case '}': token->type = TOKEN_CLOSE_BRACE; break;
case ']': token->type = TOKEN_CLOSE_BRACKET; break;
case ')': token->type = TOKEN_CLOSE_PAREN; break;
case ',': token->type = TOKEN_COMMA; break;
case '$': token->type = TOKEN_DOLLAR; break;
case '.': token->type = TOKEN_DOT; break;
case '{': token->type = TOKEN_OPEN_BRACE; break;
case '[': token->type = TOKEN_OPEN_BRACKET; break;
case '(': token->type = TOKEN_OPEN_PAREN; break;
case '~': token->type = TOKEN_TILDE; break;
case '+':
case '*':
case '^':
case '=': token->type = TOKEN_OPERATOR; break;
case '-':
c = getcFromInputFile ();
if (c == '>')
token->type = TOKEN_CONSTRAINT;
else if (c == '-') /* is this the start of a comment? */
{
skipToCharacter ('\n');
goto getNextChar;
}
else
{
if (!isspace (c))
ungetcToInputFile (c);
token->type = TOKEN_OPERATOR;
}
break;
case '?':
case ':':
{
int c2 = getcFromInputFile ();
if (c2 == '=')
token->type = TOKEN_OPERATOR;
else
{
if (!isspace (c2))
ungetcToInputFile (c2);
if (c == ':')
token->type = TOKEN_COLON;
else
token->type = TOKEN_QUESTION;
}
break;
}
case '<':
c = getcFromInputFile ();
if (c != '=' && c != '>' && !isspace (c))
ungetcToInputFile (c);
token->type = TOKEN_OPERATOR;
break;
case '>':
c = getcFromInputFile ();
if (c != '=' && c != '>' && !isspace (c))
ungetcToInputFile (c);
token->type = TOKEN_OPERATOR;
break;
case '/':
c = getcFromInputFile ();
if (c != '/' && c != '=' && !isspace (c))
ungetcToInputFile (c);
token->type = TOKEN_OPERATOR;
break;
case '\\':
c = getcFromInputFile ();
if (c != '\\' && !isspace (c))
ungetcToInputFile (c);
token->type = TOKEN_OPERATOR;
break;
case '"':
token->type = TOKEN_STRING;
parseString (token->string);
break;
case '\'':
token->type = TOKEN_CHARACTER;
parseCharacter ();
break;
default:
if (isalpha (c))
{
parseIdentifier (token->string, c);
token->keyword = analyzeToken (token->string, Lang_eiffel);
if (isKeyword (token, KEYWORD_NONE))
token->type = TOKEN_IDENTIFIER;
else
token->type = TOKEN_KEYWORD;
}
else if (isdigit (c))
{
vString* numeric = parseNumeric (c);
vStringCat (token->string, numeric);
vStringDelete (numeric);
token->type = TOKEN_NUMERIC;
}
else if (isFreeOperatorChar (c))
{
parseFreeOperator (token->string, c);
token->type = TOKEN_OPERATOR;
}
else
token->type = TOKEN_UNDEFINED;
break;
}
}