static vString* extractEmacsModeLanguageAtEOF (FILE* input) { vString* mode; /* "48.2.4.1 Specifying File Variables" of Emacs info: --------------------------------------------------- you can define file local variables using a "local variables list" near the end of the file. The start of the local variables list should be no more than 3000 characters from the end of the file, */ fseek(input, -3000, SEEK_END); mode = determineEmacsModeAtEOF (input); if (mode && (vStringLength (mode) == 0)) { vStringDelete (mode); mode = NULL; } return mode; }
static void parseMethodsImplemName (vString * const ident, objcToken what) { switch (what) { case Tok_PARL: toDoNext = &tillToken; comeAfter = &parseMethodsImplemName; waitedToken = Tok_PARR; break; case Tok_dpoint: vStringCat (fullMethodName, prevIdent); vStringCatS (fullMethodName, ":"); vStringClear (prevIdent); break; case ObjcIDENTIFIER: vStringCopy (prevIdent, ident); break; case Tok_CurlL: case Tok_semi: /* method name is not simple */ if (vStringLength (fullMethodName) != '\0') { addTag (fullMethodName, methodKind); vStringClear (fullMethodName); } else addTag (prevIdent, methodKind); toDoNext = &parseImplemMethods; parseImplemMethods (ident, what); vStringClear (prevIdent); break; default: break; } }
static int makeClassTag (const tokenInfo *const token, const vString *const inheritance, const vString *const decorators) { if (PythonKinds[K_CLASS].enabled) { tagEntryInfo e; initPythonEntry (&e, token, K_CLASS); e.extensionFields.inheritance = inheritance ? vStringValue (inheritance) : ""; if (decorators && vStringLength (decorators) > 0) { attachParserField (&e, PythonFields[F_DECORATORS].ftype, vStringValue (decorators)); } return makeTagEntry (&e); } return CORK_NIL; }
static void makeJsonTag (tokenInfo *const token, const jsonKind kind) { tagEntryInfo e; if (! JsonKinds[kind].enabled) return; initTagEntry (&e, vStringValue (token->string), &(JsonKinds[kind])); e.lineNumber = token->lineNumber; e.filePosition = token->filePosition; if (vStringLength (token->scope) > 0) { Assert (token->scopeKind > TAG_NONE && token->scopeKind < TAG_COUNT); e.extensionFields.scopeKind = &(JsonKinds[token->scopeKind]); e.extensionFields.scopeName = vStringValue (token->scope); } makeTagEntry (&e); }
/* * Emits a tag for the given 'name' of kind 'kind' at the current nesting. */ static void emitRubyTag (vString* name, rubyKind kind) { tagEntryInfo tag; vString* scope; vStringTerminate (name); scope = stringListToScope (nesting); initTagEntry (&tag, vStringValue (name)); if (vStringLength (scope) > 0) { tag.extensionFields.scope [0] = "class"; tag.extensionFields.scope [1] = vStringValue (scope); } tag.kindName = RubyKinds [kind].name; tag.kind = RubyKinds [kind].letter; makeTagEntry (&tag); stringListAdd (nesting, vStringNewCopy (name)); vStringClear (name); vStringDelete (scope); }
/* TODO: parse overlining & underlining as distinct sections. */ static void findRestTags (void) { vString *name = vStringNew (); const unsigned char *line; memset(kindchars, 0, sizeof kindchars); nestingLevels = nestingLevelsNew(); while ((line = readLineFromInputFile ()) != NULL) { int line_len = strlen((const char*) line); int name_len_bytes = vStringLength(name); int name_len = utf8_strlen(vStringValue(name), name_len_bytes); /* if the name doesn't look like UTF-8, assume one-byte charset */ if (name_len < 0) name_len = name_len_bytes; /* underlines must be the same length or more */ if (line_len >= name_len && name_len > 0 && ispunct(line[0]) && issame((const char*) line)) { char c = line[0]; int kind = get_kind(c); if (kind >= 0) { makeRestTag(name, kind); continue; } } vStringClear (name); if (! isspace(*line)) vStringCatS(name, (const char*) line); } vStringDelete (name); nestingLevelsFree(nestingLevels); }
/* follows PEP-8, and always reports single-underscores as protected * See: * - http://www.python.org/dev/peps/pep-0008/#method-names-and-instance-variables * - http://www.python.org/dev/peps/pep-0008/#designing-for-inheritance */ static pythonAccess accessFromIdentifier (const vString *const ident, pythonKind kind, boolean has_parent, boolean parent_is_class) { const char *const p = vStringValue (ident); const size_t len = vStringLength (ident); /* inside a function/method, private */ if (has_parent && !parent_is_class) return A_PRIVATE; /* not starting with "_", public */ else if (len < 1 || p[0] != '_') return A_PUBLIC; /* "__...__": magic methods */ else if (kind == K_METHOD && parent_is_class && len > 3 && p[1] == '_' && p[len - 2] == '_' && p[len - 1] == '_') return A_PUBLIC; /* "__...": name mangling */ else if (parent_is_class && len > 1 && p[1] == '_') return A_PRIVATE; /* "_...": suggested as non-public, but easily accessible */ else return A_PROTECTED; }
static int makeFunctionTag (const tokenInfo *const token, const vString *const arglist, const vString *const decorators) { if (PythonKinds[K_FUNCTION].enabled) { tagEntryInfo e; initPythonEntry (&e, token, K_FUNCTION); if (arglist) e.extensionFields.signature = vStringValue (arglist); if (decorators && vStringLength (decorators) > 0) { attachParserField (&e, PythonFields[F_DECORATORS].ftype, vStringValue (decorators)); } return makeTagEntry (&e); } return CORK_NIL; }
/* follows PEP-8, and always reports single-underscores as protected * See: * - http://www.python.org/dev/peps/pep-0008/#method-names-and-instance-variables * - http://www.python.org/dev/peps/pep-0008/#designing-for-inheritance */ static accessType accessFromIdentifier (const vString *const ident, pythonKind kind, int parentKind) { const char *const p = vStringValue (ident); const size_t len = vStringLength (ident); /* inside a function/method, private */ if (parentKind != -1 && parentKind != K_CLASS) return ACCESS_PRIVATE; /* not starting with "_", public */ else if (len < 1 || p[0] != '_') return ACCESS_PUBLIC; /* "__...__": magic methods */ else if (kind == K_FUNCTION && parentKind == K_CLASS && len > 3 && p[1] == '_' && p[len - 2] == '_' && p[len - 1] == '_') return ACCESS_PUBLIC; /* "__...": name mangling */ else if (parentKind == K_CLASS && len > 1 && p[1] == '_') return ACCESS_PRIVATE; /* "_...": suggested as non-public, but easily accessible */ else return ACCESS_PROTECTED; }
static void makeTexTag (tokenInfo *const token, texKind kind) { vString * fulltag; if (TexKinds [kind].enabled) { /* * If a scope has been added to the token, change the token * string to include the scope when making the tag. */ if ( vStringLength (token->scope) > 0 ) { fulltag = vStringNew (); vStringCopy (fulltag, token->scope); vStringCatS (fulltag, "."); vStringCatS (fulltag, vStringValue (token->string)); vStringTerminate (fulltag); vStringCopy (token->string, fulltag); vStringDelete (fulltag); } makeConstTag (token, kind); } }
static void makeRstTag(const vString* const name, const int kind, const fpos_t filepos) { const NestingLevel *const nl = getNestingLevel(kind); if (vStringLength (name) > 0) { tagEntryInfo e; initTagEntry (&e, vStringValue (name)); e.lineNumber--; /* we want the line before the '---' underline chars */ e.kindName = RstKinds [kind].name; e.kind = RstKinds [kind].letter; e.filePosition = filepos; if (nl && nl->type < kind) { e.extensionFields.scope [0] = RstKinds [nl->type].name; e.extensionFields.scope [1] = vStringValue (nl->name); } makeTagEntry (&e); } nestingLevelsPush(nestingLevels, name, kind); }
/* `end' points to the equal sign. Parse from right to left to get the * identifier. Assume we're dealing with something of form \s*\w+\s*=> */ static void makeTagFromLeftSide (const char *begin, const char *end, vString *name, vString *package) { tagEntryInfo entry; const char *b, *e; if (! PerlKinds[K_CONSTANT].enabled) return; for (e = end - 1; e > begin && isspace(*e); --e) ; if (e < begin) return; for (b = e; b >= begin && isIdentifier(*b); --b) ; /* Identifier must be either beginning of line of have some whitespace * on its left: */ if (b < begin || isspace(*b) || ',' == *b) ++b; else if (b != begin) return; Assert(e - b + 1 > 0); vStringClear(name); vStringNCatS(name, b, e - b + 1); initTagEntry(&entry, vStringValue(name)); entry.kind = PerlKinds[K_CONSTANT].letter; entry.kindName = PerlKinds[K_CONSTANT].name; makeTagEntry(&entry); if (Option.include.qualifiedTags && package && vStringLength(package)) { vStringClear(name); vStringCopy(name, package); vStringNCatS(name, b, e - b + 1); initTagEntry(&entry, vStringValue(name)); entry.kind = PerlKinds[K_CONSTANT].letter; entry.kindName = PerlKinds[K_CONSTANT].name; makeTagEntry(&entry); } }
static void readAndEmitTag (const unsigned char** cp, rubyKind expected_kind) { if (isspace (**cp)) { vString *name = vStringNew (); rubyKind actual_kind = parseIdentifier (cp, name, expected_kind); if (actual_kind == K_UNDEFINED || vStringLength (name) == 0) { /* * What kind of tags should we create for code like this? * * %w(self.clfloor clfloor).each do |name| * module_eval <<-"end;" * def #{name}(x, y=1) * q, r = x.divmod(y) * q = q.to_i * return q, r * end * end; * end * * Or this? * * class << HTTP * * For now, we don't create any. */ enterUnnamedScope (); } else { emitRubyTag (name, actual_kind); } vStringDelete (name); } }
static void processFunction (tokenInfo *const token) { int c; /* Search for function name * Last identifier found before a '(' or a ';' is the function name */ c = skipWhite (vGetc ()); do { readIdentifier (token, c); c = skipWhite (vGetc ()); } while (c != '(' && c != ';' && c != EOF); if ( vStringLength (token->name) > 0 ) { verbose ("Found function: %s\n", vStringValue (token->name)); /* Create tag */ createTag (token); /* Get port list from function */ processPortList (c); } }
static void makeTexTag (tokenInfo *const token, texKind kind) { if (TexKinds [kind].enabled) { const char *const name = vStringValue (token->string); vString *parentKind = vStringNew(); vString *parentName = vStringNew(); tagEntryInfo e; initTagEntry (&e, name, &(TexKinds [kind])); e.lineNumber = token->lineNumber; e.filePosition = token->filePosition; getScopeInfo(kind, parentKind, parentName); if (vStringLength(parentKind) > 0) { e.extensionFields.scopeKind = kindFromName (vStringValue(parentKind)); e.extensionFields.scopeName = vStringValue(parentName); } makeTagEntry (&e); vStringDelete (parentKind); vStringDelete (parentName); } }
static void readTokenFull (tokenInfo *const token, boolean include_newlines, vString *const repr) { int c; int i; token->type = TOKEN_UNDEFINED; token->keyword = KEYWORD_NONE; vStringClear (token->string); getNextChar: i = 0; do { c = fileGetc (); i++; } while (c == '\t' || c == ' ' || ((c == '\r' || c == '\n') && ! include_newlines)); token->lineNumber = getSourceLineNumber (); token->filePosition = getInputFilePosition (); if (repr) { if (i > 1) vStringPut (repr, ' '); vStringPut (repr, c); } switch (c) { case EOF: token->type = TOKEN_EOF; break; case '(': token->type = TOKEN_OPEN_PAREN; break; case ')': token->type = TOKEN_CLOSE_PAREN; break; case ';': token->type = TOKEN_SEMICOLON; break; case ',': token->type = TOKEN_COMMA; break; case '.': token->type = TOKEN_PERIOD; break; case ':': token->type = TOKEN_COLON; break; case '{': token->type = TOKEN_OPEN_CURLY; break; case '}': token->type = TOKEN_CLOSE_CURLY; break; case '=': token->type = TOKEN_EQUAL_SIGN; break; case '[': token->type = TOKEN_OPEN_SQUARE; break; case ']': token->type = TOKEN_CLOSE_SQUARE; break; case '+': case '-': { int d = fileGetc (); if (d == c) /* ++ or -- */ token->type = TOKEN_POSTFIX_OPERATOR; else { fileUngetc (d); token->type = TOKEN_BINARY_OPERATOR; } break; } case '*': case '%': case '?': case '>': case '<': case '^': case '|': case '&': token->type = TOKEN_BINARY_OPERATOR; break; case '\r': case '\n': /* This isn't strictly correct per the standard, but following the * real rules means understanding all statements, and that's not * what the parser currently does. What we do here is a guess, by * avoiding inserting semicolons that would make the statement on * the left invalid. Hopefully this should not have false negatives * (e.g. should not miss insertion of a semicolon) but might have * false positives (e.g. it will wrongfully emit a semicolon for the * newline in "foo\n+bar"). * This should however be mostly harmless as we only deal with * newlines in specific situations where we know a false positive * wouldn't hurt too bad. */ switch (LastTokenType) { /* these cannot be the end of a statement, so hold the newline */ case TOKEN_EQUAL_SIGN: case TOKEN_COLON: case TOKEN_PERIOD: case TOKEN_FORWARD_SLASH: case TOKEN_BINARY_OPERATOR: /* and these already end one, no need to duplicate it */ case TOKEN_SEMICOLON: case TOKEN_COMMA: case TOKEN_CLOSE_CURLY: case TOKEN_OPEN_CURLY: include_newlines = FALSE; /* no need to recheck */ goto getNextChar; break; default: token->type = TOKEN_SEMICOLON; } break; case '\'': case '"': token->type = TOKEN_STRING; parseString (token->string, c); token->lineNumber = getSourceLineNumber (); token->filePosition = getInputFilePosition (); if (repr) { vStringCat (repr, token->string); vStringPut (repr, c); } break; case '\\': c = fileGetc (); if (c != '\\' && c != '"' && !isspace (c)) fileUngetc (c); token->type = TOKEN_CHARACTER; token->lineNumber = getSourceLineNumber (); token->filePosition = getInputFilePosition (); break; case '/': { int d = fileGetc (); if ( (d != '*') && /* is this the start of a comment? */ (d != '/') ) /* is a one line comment? */ { fileUngetc (d); switch (LastTokenType) { case TOKEN_CHARACTER: case TOKEN_IDENTIFIER: case TOKEN_STRING: case TOKEN_CLOSE_CURLY: case TOKEN_CLOSE_PAREN: case TOKEN_CLOSE_SQUARE: token->type = TOKEN_FORWARD_SLASH; break; default: token->type = TOKEN_REGEXP; parseRegExp (); token->lineNumber = getSourceLineNumber (); token->filePosition = getInputFilePosition (); break; } } else { if (repr) /* remove the / we added */ repr->buffer[--repr->length] = 0; if (d == '*') { do { skipToCharacter ('*'); c = fileGetc (); if (c == '/') break; else fileUngetc (c); } while (c != EOF && c != '\0'); goto getNextChar; } else if (d == '/') /* is this the start of a comment? */ { skipToCharacter ('\n'); /* if we care about newlines, put it back so it is seen */ if (include_newlines) fileUngetc ('\n'); goto getNextChar; } } break; } case '#': /* skip shebang in case of e.g. Node.js scripts */ if (token->lineNumber > 1) token->type = TOKEN_UNDEFINED; else if ((c = fileGetc ()) != '!') { fileUngetc (c); token->type = TOKEN_UNDEFINED; } else { skipToCharacter ('\n'); goto getNextChar; } break; default: if (! isIdentChar (c)) token->type = TOKEN_UNDEFINED; else { parseIdentifier (token->string, c); token->lineNumber = getSourceLineNumber (); token->filePosition = getInputFilePosition (); token->keyword = analyzeToken (token->string); if (isKeyword (token, KEYWORD_NONE)) token->type = TOKEN_IDENTIFIER; else token->type = TOKEN_KEYWORD; if (repr && vStringLength (token->string) > 1) vStringCatS (repr, vStringValue (token->string) + 1); } break; } LastTokenType = token->type; }
static void parseString (vString *const string) { boolean verbatim = FALSE; boolean align = FALSE; boolean end = FALSE; vString *verbatimCloser = vStringNew (); vString *lastLine = vStringNew (); int prev = '\0'; int c; while (! end) { c = getcFromInputFile (); if (c == EOF) end = TRUE; else if (c == '"') { if (! verbatim) end = TRUE; else end = (boolean) (strcmp (vStringValue (lastLine), vStringValue (verbatimCloser)) == 0); } else if (c == '\n') { if (verbatim) vStringClear (lastLine); if (prev == '[' /* || prev == '{' */) { verbatim = TRUE; vStringClear (verbatimCloser); vStringClear (lastLine); if (prev == '{') vStringPut (verbatimCloser, '}'); else { vStringPut (verbatimCloser, ']'); align = TRUE; } vStringNCat (verbatimCloser, string, vStringLength (string) - 1); vStringClear (string); } if (verbatim && align) { do c = getcFromInputFile (); while (isspace (c)); } } else if (c == '%') c = parseEscapedCharacter (); if (! end) { vStringPut (string, c); if (verbatim) { vStringPut (lastLine, c); vStringTerminate (lastLine); } prev = c; } } vStringTerminate (string); vStringDelete (lastLine); vStringDelete (verbatimCloser); }
/* * Rust is very liberal with nesting, so this function is used pretty much for any block */ static void parseBlock (lexerState *lexer, boolean delim, int kind, vString *scope) { int level = 1; if (delim) { if (lexer->cur_token != '{') return; advanceToken(lexer, TRUE); } while (lexer->cur_token != TOKEN_EOF) { if (lexer->cur_token == TOKEN_IDENT) { size_t old_scope_len = vStringLength(scope); if (strcmp(lexer->token_str->buffer, "fn") == 0) { parseFn(lexer, scope, kind); } else if(strcmp(lexer->token_str->buffer, "mod") == 0) { parseMod(lexer, scope, kind); } else if(strcmp(lexer->token_str->buffer, "static") == 0) { parseStatic(lexer, scope, kind); } else if(strcmp(lexer->token_str->buffer, "trait") == 0) { parseTrait(lexer, scope, kind); } else if(strcmp(lexer->token_str->buffer, "type") == 0) { parseType(lexer, scope, kind); } else if(strcmp(lexer->token_str->buffer, "impl") == 0) { parseImpl(lexer, scope, kind); } else if(strcmp(lexer->token_str->buffer, "struct") == 0) { parseStructOrEnum(lexer, scope, kind, TRUE); } else if(strcmp(lexer->token_str->buffer, "enum") == 0) { parseStructOrEnum(lexer, scope, kind, FALSE); } else if(strcmp(lexer->token_str->buffer, "macro_rules") == 0) { parseMacroRules(lexer, scope, kind); } else { advanceToken(lexer, TRUE); if (lexer->cur_token == '!') { skipMacro(lexer); } } resetScope(scope, old_scope_len); } else if (lexer->cur_token == '{') { level++; advanceToken(lexer, TRUE); } else if (lexer->cur_token == '}') { level--; advanceToken(lexer, TRUE); } else if (lexer->cur_token == '\'') { /* Skip over the 'static lifetime, as it confuses the static parser above */ advanceToken(lexer, TRUE); if (lexer->cur_token == TOKEN_IDENT && strcmp(lexer->token_str->buffer, "static") == 0) advanceToken(lexer, TRUE); } else { advanceToken(lexer, TRUE); } if (delim && level <= 0) break; } }
static void findBetaTags (void) { vString *line = vStringNew (); boolean incomment = FALSE; boolean inquote = FALSE; boolean dovirtuals = BetaKinds [K_VIRTUAL].enabled; boolean dopatterns = BetaKinds [K_PATTERN].enabled; do { boolean foundfragmenthere = FALSE; /* find fragment definition (line that starts and ends with --) */ int last; int first; int c; vStringClear (line); while ((c = fileGetc ()) != EOF && c != '\n' && c != '\r') vStringPut (line, c); vStringTerminate (line); last = vStringLength (line) - 1; first = 0; /* skip white space at start and end of line */ while (last && isspace ((int) vStringChar (line, last))) last--; while (first < last && isspace ((int) vStringChar (line, first))) first++; /* if line still has a reasonable length and ... */ if (last - first > 4 && (vStringChar (line, first) == '-' && vStringChar (line, first + 1) == '-' && vStringChar (line, last) == '-' && vStringChar (line, last - 1) == '-')) { if (!incomment && !inquote) { foundfragmenthere = TRUE; /* skip past -- and whitespace. Also skip back past 'dopart' or 'attributes' to the :. We have to do this because there is no sensible way to include whitespace in a ctags token so the conventional space after the ':' would mess us up */ last -= 2; first += 2; while (last && vStringChar (line, last) != ':') last--; while (last && (isspace ((int) vStringChar (line, last-1)))) last--; while (first < last && (isspace ((int) vStringChar (line, first)) || vStringChar (line, first) == '-')) first++; /* If there's anything left it is a fragment title */ if (first < last - 1) { vStringChar (line, last) = 0; if (strcasecmp ("LIB", vStringValue (line) + first) && strcasecmp ("PROGRAM", vStringValue (line) + first)) { makeBetaTag (vStringValue (line) + first, K_FRAGMENT); } } } } else { int pos = 0; int len = vStringLength (line); if (inquote) goto stringtext; if (incomment) goto commenttext; programtext: for ( ; pos < len; pos++) { if (vStringChar (line, pos) == '\'') { pos++; inquote = TRUE; goto stringtext; } if (vStringChar (line, pos) == '{') { pos++; incomment = TRUE; goto commenttext; } if (vStringChar (line, pos) == '(' && pos < len - 1 && vStringChar (line, pos+1) == '*') { pos +=2; incomment = TRUE; goto commenttext; } /* * SLOT definition looks like this: * <<SLOT nameofslot: dopart>> * or * <<SLOT nameofslot: descriptor>> */ if (!foundfragmenthere && vStringChar (line, pos) == '<' && pos+1 < len && vStringChar (line, pos+1) == '<' && strstr (vStringValue (line) + pos, ">>")) { /* Found slot name, get start and end */ int eoname; char c2; pos += 2; /* skip past << */ /* skip past space before SLOT */ while (pos < len && isspace ((int) vStringChar (line, pos))) pos++; /* skip past SLOT */ if (pos+4 <= len && !strncasecmp (vStringValue(line) + pos, "SLOT", (size_t)4)) pos += 4; /* skip past space after SLOT */ while (pos < len && isspace ((int) vStringChar (line, pos))) pos++; eoname = pos; /* skip to end of name */ while (eoname < len && (c2 = vStringChar (line, eoname)) != '>' && c2 != ':' && !isspace ((int) c2)) eoname++; if (eoname < len) { vStringChar (line, eoname) = 0; if (strcasecmp ("LIB", vStringValue (line) + pos) && strcasecmp ("PROGRAM", vStringValue (line) + pos) && strcasecmp ("SLOT", vStringValue (line) + pos)) { makeBetaTag (vStringValue (line) + pos, K_SLOT); } } if (eoname+1 < len) { pos = eoname + 1; } else { pos = len; continue; } } /* Only patterns that are virtual, extensions of virtuals or * final bindings are normally included so as not to overload * totally. * That means one of the forms name:: name:< or name::< */ if (!foundfragmenthere && vStringChar (line, pos) == ':' && (dopatterns || (dovirtuals && (vStringChar (line, pos+1) == ':' || vStringChar (line, pos+1) == '<') ) ) ) { /* Found pattern name, get start and end */ int eoname = pos; int soname; while (eoname && isspace ((int) vStringChar (line, eoname-1))) eoname--; foundanothername: /* terminate right after name */ vStringChar (line, eoname) = 0; soname = eoname; while (soname && isbident (vStringChar (line, soname-1))) { soname--; } if (soname != eoname) { makeBetaTag (vStringValue (line) + soname, K_PATTERN); /* scan back past white space */ while (soname && isspace ((int) vStringChar (line, soname-1))) soname--; if (soname && vStringChar (line, soname-1) == ',') { /* we found a new pattern name before comma */ eoname = soname; goto foundanothername; } } } } goto endofline; commenttext: for ( ; pos < len; pos++) { if (vStringChar (line, pos) == '*' && pos < len - 1 && vStringChar (line, pos+1) == ')') { pos += 2; incomment = FALSE; goto programtext; } if (vStringChar (line, pos) == '}') { pos++; incomment = FALSE; goto programtext; } } goto endofline; stringtext: for ( ; pos < len; pos++) { if (vStringChar (line, pos) == '\\') { if (pos < len - 1) pos++; } else if (vStringChar (line, pos) == '\'') { pos++; /* support obsolete '' syntax */ if (pos < len && vStringChar (line, pos) == '\'') { continue; } inquote = FALSE; goto programtext; } } } endofline: inquote = FALSE; /* This shouldn't really make a difference */ } while (!feof (File.fp)); }
/* Store the current character in lexerState::token_str if there is space * (set by MAX_STRING_LENGTH), and then read the next character from the file */ static void advanceAndStoreChar (lexerState *lexer) { if (vStringLength(lexer->token_str) < MAX_STRING_LENGTH) vStringPut(lexer->token_str, (char) lexer->cur_c); advanceChar(lexer); }
static void addGenericName (tokenInfo *const token) { vStringUpper (token->string); if (vStringLength (token->string) > 0) stringListAdd (GenericNames, vStringNewCopy (token->string)); }
/* Algorithm adapted from from GNU etags. * Perl support by Bart Robinson <*****@*****.**> * Perl sub names: look for /^ [ \t\n]sub [ \t\n]+ [^ \t\n{ (]+/ */ static void findPerlTags (void) { vString *name = vStringNew (); vString *package = NULL; boolean skipPodDoc = FALSE; const unsigned char *line; /* Core modules AutoLoader and SelfLoader support delayed compilation * by allowing Perl code that follows __END__ and __DATA__ tokens, * respectively. When we detect that one of these modules is used * in the file, we continue processing even after we see the * corresponding token that would usually terminate parsing of the * file. */ enum { RESPECT_END = (1 << 0), RESPECT_DATA = (1 << 1), } respect_token = RESPECT_END | RESPECT_DATA; while ((line = fileReadLine ()) != NULL) { boolean spaceRequired = FALSE; boolean qualified = FALSE; const unsigned char *cp = line; perlKind kind = K_NONE; tagEntryInfo e; if (skipPodDoc) { if (strncmp ((const char*) line, "=cut", (size_t) 4) == 0) skipPodDoc = FALSE; continue; } else if (line [0] == '=') { skipPodDoc = isPodWord ((const char*)line + 1); continue; } else if (strcmp ((const char*) line, "__DATA__") == 0) { if (respect_token & RESPECT_DATA) break; else continue; } else if (strcmp ((const char*) line, "__END__") == 0) { if (respect_token & RESPECT_END) break; else continue; } else if (line [0] == '#') continue; while (isspace (*cp)) cp++; if (strncmp((const char*) cp, "sub", (size_t) 3) == 0) { TRACE("this looks like a sub\n"); cp += 3; kind = K_SUBROUTINE; spaceRequired = TRUE; qualified = TRUE; } else if (strncmp((const char*) cp, "use", (size_t) 3) == 0) { cp += 3; if (!isspace(*cp)) continue; while (*cp && isspace (*cp)) ++cp; if (strncmp((const char*) cp, "AutoLoader", (size_t) 10) == 0) { respect_token &= ~RESPECT_END; continue; } if (strncmp((const char*) cp, "SelfLoader", (size_t) 10) == 0) { respect_token &= ~RESPECT_DATA; continue; } if (strncmp((const char*) cp, "constant", (size_t) 8) != 0) continue; cp += 8; /* Skip up to the first non-space character, skipping empty * and comment lines. */ while (isspace(*cp)) cp++; while (!*cp || '#' == *cp) { cp = fileReadLine (); if (!cp) goto END_MAIN_WHILE; while (isspace (*cp)) cp++; } if ('{' == *cp) { ++cp; if (0 == parseConstantsFromHashRef(cp, name, package)) { vStringClear(name); continue; } else goto END_MAIN_WHILE; } kind = K_CONSTANT; spaceRequired = FALSE; qualified = TRUE; } else if (strncmp((const char*) cp, "package", (size_t) 7) == 0 && ('\0' == cp[7] || isspace(cp[7]))) { cp += 7; while (isspace (*cp)) cp++; while (!*cp || '#' == *cp) { cp = fileReadLine (); if (!cp) goto END_MAIN_WHILE; while (isspace (*cp)) cp++; } if (package == NULL) package = vStringNew (); else vStringClear (package); const unsigned char *const first = cp; while (*cp && (int) *cp != ';' && !isspace ((int) *cp)) { vStringPut (package, (int) *cp); cp++; } vStringCatS (package, "::"); cp = first; /* Rewind */ kind = K_PACKAGE; spaceRequired = FALSE; qualified = TRUE; } else if (strncmp((const char*) cp, "format", (size_t) 6) == 0) { cp += 6; kind = K_FORMAT; spaceRequired = TRUE; qualified = TRUE; } else { if (isIdentifier1 (*cp)) { const unsigned char *p = cp; while (isIdentifier (*p)) ++p; while (isspace (*p)) ++p; if ((int) *p == ':' && (int) *(p + 1) != ':') kind = K_LABEL; } } if (kind != K_NONE) { TRACE("cp0: %s\n", (const char *) cp); if (spaceRequired && *cp && !isspace (*cp)) continue; TRACE("cp1: %s\n", (const char *) cp); while (isspace (*cp)) cp++; while (!*cp || '#' == *cp) { /* Gobble up empty lines and comments */ cp = fileReadLine (); if (!cp) goto END_MAIN_WHILE; while (isspace (*cp)) cp++; } while (isIdentifier (*cp) || (K_PACKAGE == kind && ':' == *cp)) { vStringPut (name, (int) *cp); cp++; } if (K_FORMAT == kind && vStringLength (name) == 0 && /* cp did not advance */ '=' == *cp) { /* format's name is optional. If it's omitted, 'STDOUT' is assumed. */ vStringCatS (name, "STDOUT"); } vStringTerminate (name); TRACE("name: %s\n", name->buffer); if (0 == vStringLength(name)) { vStringClear(name); continue; } if (K_SUBROUTINE == kind) { /* * isSubroutineDeclaration() may consume several lines. So * we record line positions. */ initTagEntry(&e, vStringValue(name), NULL); if (TRUE == isSubroutineDeclaration(cp)) { if (TRUE == PerlKinds[K_SUBROUTINE_DECLARATION].enabled) { kind = K_SUBROUTINE_DECLARATION; } else { vStringClear (name); continue; } } else if (! PerlKinds[kind].enabled) { continue; } e.kind = &(PerlKinds[kind]); makeTagEntry(&e); if (Option.include.qualifiedTags && qualified && package != NULL && vStringLength (package) > 0) { vString *const qualifiedName = vStringNew (); vStringCopy (qualifiedName, package); vStringCat (qualifiedName, name); e.name = vStringValue(qualifiedName); makeTagEntry(&e); vStringDelete (qualifiedName); } } else if (vStringLength (name) > 0) { makeSimpleTag (name, PerlKinds, kind); if (Option.include.qualifiedTags && qualified && K_PACKAGE != kind && package != NULL && vStringLength (package) > 0) { vString *const qualifiedName = vStringNew (); vStringCopy (qualifiedName, package); vStringCat (qualifiedName, name); makeSimpleTag (qualifiedName, PerlKinds, kind); vStringDelete (qualifiedName); } } vStringClear (name); } } END_MAIN_WHILE: vStringDelete (name); if (package != NULL) vStringDelete (package); }
static void findConfTags (void) { vString *name = vStringNew (); vString *scope = vStringNew (); const unsigned char *line; while ((line = fileReadLine ()) != NULL) { const unsigned char* cp = line; boolean possible = TRUE; if (isspace ((int) *cp) || *cp == '#' || (*cp != '\0' && *cp == '/' && *(cp+1) == '/')) continue; /* look for a section */ if (*cp != '\0' && *cp == '[') { ++cp; while (*cp != '\0' && *cp != ']') { vStringPut (name, (int) *cp); ++cp; } vStringTerminate (name); makeSimpleTag (name, ConfKinds, K_SECTION); /* remember section name */ vStringCopy (scope, name); vStringTerminate (scope); vStringClear (name); continue; } while (*cp != '\0') { /* We look for any sequence of identifier characters following a white space */ if (possible && isIdentifier ((int) *cp)) { while (isIdentifier ((int) *cp)) { vStringPut (name, (int) *cp); ++cp; } vStringTerminate (name); vStringStripTrailing (name); while (isspace ((int) *cp)) ++cp; if (*cp == '=') { if (vStringLength (scope) > 0) makeSimpleScopedTag (name, ConfKinds, K_KEY, "section", vStringValue(scope), NULL); else makeSimpleTag (name, ConfKinds, K_KEY); } vStringClear (name); } else if (isspace ((int) *cp)) possible = TRUE; else possible = FALSE; if (*cp != '\0') ++cp; } } vStringDelete (name); vStringDelete (scope); }
static void findTag (tokenInfo *const token) { if (currentContext->kind != K_UNDEFINED) { /* Drop context, but only if an end token is found */ dropContext (token); } if (token->kind == K_CONSTANT && vStringItem (token->name, 0) == '`') { /* Bug #961001: Verilog compiler directives are line-based. */ int c = skipWhite (vGetc ()); readIdentifier (token, c); createTag (token); /* Skip the rest of the line. */ do { c = vGetc(); } while (c != EOF && c != '\n'); vUngetc (c); } else if (token->kind == K_BLOCK) { /* Process begin..end blocks */ processBlock (token); } else if (token->kind == K_FUNCTION || token->kind == K_TASK) { /* Functions are treated differently because they may also include the * type of the return value. * Tasks are treated in the same way, although not having a return * value.*/ processFunction (token); } else if (token->kind == K_ASSERTION) { if (vStringLength (currentContext->blockName) > 0) { vStringCopy (token->name, currentContext->blockName); createTag (token); skipToSemiColon (); } } else if (token->kind == K_TYPEDEF) { processTypedef (token); } else if (token->kind == K_CLASS) { processClass (token); } else if (token->kind == K_IGNORE && isSingleStatement (token)) { currentContext->singleStat = TRUE; } else if (isVariable (token)) { int c = skipWhite (vGetc ()); tagNameList (token, c); } else if (token->kind != K_UNDEFINED && token->kind != K_IGNORE) { int c = skipWhite (vGetc ()); if (isIdentifierCharacter (c)) { readIdentifier (token, c); while (getKind (token) == K_IGNORE) { c = skipWhite (vGetc ()); readIdentifier (token, c); } createTag (token); /* Get port list if required */ c = skipWhite (vGetc ()); if (c == '(' && hasSimplePortList (token)) { processPortList (c); } else { vUngetc (c); } } } }
extern void createRTags(void) { vString *vLine = vStringNew(); vString *name = vStringNew(); int ikind; const unsigned char *line; while ((line = fileReadLine()) != NULL) { const unsigned char *cp = (const unsigned char*)line; vStringClear(name); while ((*cp != '\0') && (*cp != '#')) { /* iterate to the end of line or to a comment */ ikind = -1; switch (*cp) { case 'l': case 's': if (strncasecmp((const char*)cp, "library", (size_t)7) == 0) { /* load a library: library(tools) */ cp += 7; SKIPSPACE(cp); if (*cp == '(') ikind = 1; else cp -= 7; } else if (strncasecmp((const char*)cp, "source", (size_t)6) == 0) { /* load a source file: source("myfile.r") */ cp += 6; SKIPSPACE(cp); if (*cp == '(') ikind = 2; else cp -= 6; } if (ikind != -1) { cp++; vStringClear(name); while ((!isspace((int)*cp)) && *cp != '\0' && *cp != ')') { vStringPut(name, (int)*cp); cp++; } vStringTerminate(name); /* if the string really exists, make a tag of it */ if (vStringLength(name) > 0) makeRTag(name, ikind); /* prepare for the next iteration */ vStringClear(name); } else { vStringPut(name, (int)*cp); cp++; } break; case '<': cp++; if (*cp == '-') { /* assignment: ident <- someval */ cp++; SKIPSPACE(cp); if (*cp == '\0') { /* not in this line, read next */ /* sometimes functions are declared this way: ident <- function(...) { ... } I don't know if there is a reason to write the function keyword in a new line */ if ((line = fileReadLine()) != NULL) { cp = (const unsigned char*)line; SKIPSPACE(cp); } } if (strncasecmp((const char*)cp, "function", (size_t)8) == 0) { /* it's a function: ident <- function(args) */ cp += 8; vStringTerminate(name); /* if the string really exists, make a tag of it */ if (vStringLength(name) > 0) makeRTag(name, 0); /* prepare for the next iteration */ vStringClear(name); break; } } case ' ': case '\x009': /* skip whitespace */ cp++; break; default: /* collect all characters that could be a part of an identifier */ vStringPut(name, (int)*cp); cp++; break; } } } vStringDelete(name); vStringDelete(vLine); }
/* Adds a name to the end of the scope string */ static void addToScope (vString *scope, vString *name) { if (vStringLength(scope) > 0) vStringCatS(scope, "::"); vStringCat(scope, name); }
static void readToken (tokenInfo *const token) { int c; static tokenType lastTokenType = TOKEN_NONE; boolean firstWhitespace = TRUE; boolean whitespace; token->type = TOKEN_NONE; token->keyword = KEYWORD_NONE; vStringClear (token->string); getNextChar: do { c = getcFromInputFile (); token->lineNumber = getInputLineNumber (); token->filePosition = getInputFilePosition (); if (c == '\n' && (lastTokenType == TOKEN_IDENTIFIER || lastTokenType == TOKEN_STRING || lastTokenType == TOKEN_OTHER || lastTokenType == TOKEN_CLOSE_PAREN || lastTokenType == TOKEN_CLOSE_CURLY || lastTokenType == TOKEN_CLOSE_SQUARE)) { c = ';'; // semicolon injection } whitespace = c == '\t' || c == ' ' || c == '\r' || c == '\n'; if (signature && whitespace && firstWhitespace && vStringLength (signature) < MAX_SIGNATURE_LENGTH) { firstWhitespace = FALSE; vStringPut(signature, ' '); } } while (whitespace); switch (c) { case EOF: token->type = TOKEN_EOF; break; case ';': token->type = TOKEN_SEMICOLON; break; case '/': { boolean hasNewline = FALSE; int d = getcFromInputFile (); switch (d) { case '/': skipToCharacterInInputFile ('\n'); /* Line comments start with the * character sequence // and * continue through the next * newline. A line comment acts * like a newline. */ ungetcToInputFile ('\n'); goto getNextChar; case '*': do { do { d = getcFromInputFile (); if (d == '\n') { hasNewline = TRUE; } } while (d != EOF && d != '*'); c = getcFromInputFile (); if (c == '/') break; else ungetcToInputFile (c); } while (c != EOF && c != '\0'); ungetcToInputFile (hasNewline ? '\n' : ' '); goto getNextChar; default: token->type = TOKEN_OTHER; ungetcToInputFile (d); break; } } break; case '"': case '\'': case '`': token->type = TOKEN_STRING; parseString (token->string, c); token->lineNumber = getInputLineNumber (); token->filePosition = getInputFilePosition (); break; case '<': { int d = getcFromInputFile (); if (d == '-') token->type = TOKEN_LEFT_ARROW; else { ungetcToInputFile (d); token->type = TOKEN_OTHER; } } break; case '(': token->type = TOKEN_OPEN_PAREN; break; case ')': token->type = TOKEN_CLOSE_PAREN; break; case '{': token->type = TOKEN_OPEN_CURLY; break; case '}': token->type = TOKEN_CLOSE_CURLY; break; case '[': token->type = TOKEN_OPEN_SQUARE; break; case ']': token->type = TOKEN_CLOSE_SQUARE; break; case '*': token->type = TOKEN_STAR; break; case '.': token->type = TOKEN_DOT; break; case ',': token->type = TOKEN_COMMA; break; default: if (isStartIdentChar (c)) { parseIdentifier (token->string, c); token->lineNumber = getInputLineNumber (); token->filePosition = getInputFilePosition (); token->keyword = lookupKeyword (vStringValue (token->string), Lang_go); if (isKeyword (token, KEYWORD_NONE)) token->type = TOKEN_IDENTIFIER; else token->type = TOKEN_KEYWORD; } else token->type = TOKEN_OTHER; break; } if (signature && vStringLength (signature) < MAX_SIGNATURE_LENGTH) { if (token->type == TOKEN_LEFT_ARROW) vStringCatS(signature, "<-"); else if (token->type == TOKEN_STRING) { // only struct member annotations can appear in function prototypes // so only `` type strings are possible vStringPut(signature, '`'); vStringCat(signature, token->string); vStringPut(signature, '`'); } else if (token->type == TOKEN_IDENTIFIER || token->type == TOKEN_KEYWORD) vStringCat(signature, token->string); else if (c != EOF) vStringPut(signature, c); } lastTokenType = token->type; }
/* Algorithm adapted from from GNU etags. * Perl support by Bart Robinson <*****@*****.**> * Perl sub names: look for /^ [ \t\n]sub [ \t\n]+ [^ \t\n{ (]+/ */ static void findPerlTags (void) { vString *name = vStringNew (); vString *package = NULL; bool skipPodDoc = false; const unsigned char *line; while ((line = readLineFromInputFile ()) != NULL) { bool spaceRequired = false; bool qualified = false; const unsigned char *cp = line; perlKind kind = K_NONE; tagEntryInfo e; if (skipPodDoc) { if (strncmp ((const char*) line, "=cut", (size_t) 4) == 0) skipPodDoc = false; continue; } else if (line [0] == '=') { skipPodDoc = isPodWord ((const char*)line + 1); continue; } else if (strcmp ((const char*) line, "__DATA__") == 0) break; else if (strcmp ((const char*) line, "__END__") == 0) break; else if (line [0] == '#') continue; while (isspace (*cp)) cp++; if (strncmp((const char*) cp, "sub", (size_t) 3) == 0) { TRACE("this looks like a sub\n"); cp += 3; kind = K_SUBROUTINE; spaceRequired = true; qualified = true; } else if (strncmp((const char*) cp, "use", (size_t) 3) == 0) { cp += 3; if (!isspace(*cp)) continue; while (*cp && isspace (*cp)) ++cp; if (strncmp((const char*) cp, "constant", (size_t) 8) != 0) continue; cp += 8; kind = K_CONSTANT; spaceRequired = true; qualified = true; } else if (strncmp((const char*) cp, "package", (size_t) 7) == 0) { /* This will point to space after 'package' so that a tag can be made */ const unsigned char *space = cp += 7; if (package == NULL) package = vStringNew (); else vStringClear (package); while (isspace (*cp)) cp++; while ((int) *cp != ';' && !isspace ((int) *cp)) { vStringPut (package, (int) *cp); cp++; } vStringCatS (package, "::"); cp = space; /* Rewind */ kind = K_PACKAGE; spaceRequired = true; qualified = true; } else if (strncmp((const char*) cp, "format", (size_t) 6) == 0) { cp += 6; kind = K_FORMAT; spaceRequired = true; qualified = true; } else { if (isIdentifier1 (*cp)) { const unsigned char *p = cp; while (isIdentifier (*p)) ++p; while (isspace (*p)) ++p; if ((int) *p == ':' && (int) *(p + 1) != ':') kind = K_LABEL; } } if (kind != K_NONE) { TRACE("cp0: %s\n", (const char *) cp); if (spaceRequired && *cp && !isspace (*cp)) continue; TRACE("cp1: %s\n", (const char *) cp); while (isspace (*cp)) cp++; while (!*cp || '#' == *cp) { /* Gobble up empty lines and comments */ cp = readLineFromInputFile (); if (!cp) goto END_MAIN_WHILE; while (isspace (*cp)) cp++; } while (isIdentifier (*cp) || (K_PACKAGE == kind && ':' == *cp)) { vStringPut (name, (int) *cp); cp++; } if (K_FORMAT == kind && vStringLength (name) == 0 && /* cp did not advance */ '=' == *cp) { /* format's name is optional. If it's omitted, 'STDOUT' is assumed. */ vStringCatS (name, "STDOUT"); } TRACE("name: %s\n", name->buffer); if (0 == vStringLength(name)) { vStringClear(name); continue; } if (K_SUBROUTINE == kind) { /* * isSubroutineDeclaration() may consume several lines. So * we record line positions. */ initTagEntry(&e, vStringValue(name), &(PerlKinds[kind])); if (true == isSubroutineDeclaration(cp)) { if (true == PerlKinds[K_SUBROUTINE_DECLARATION].enabled) { kind = K_SUBROUTINE_DECLARATION; } else { vStringClear (name); continue; } } makeTagEntry(&e); if (isXtagEnabled(XTAG_QUALIFIED_TAGS) && qualified && package != NULL && vStringLength (package) > 0) { vString *const qualifiedName = vStringNew (); vStringCopy (qualifiedName, package); vStringCat (qualifiedName, name); e.name = vStringValue(qualifiedName); makeTagEntry(&e); vStringDelete (qualifiedName); } } else if (vStringLength (name) > 0) { makeSimpleTag (name, PerlKinds, kind); if (isXtagEnabled(XTAG_QUALIFIED_TAGS) && qualified && K_PACKAGE != kind && package != NULL && vStringLength (package) > 0) { vString *const qualifiedName = vStringNew (); vStringCopy (qualifiedName, package); vStringCat (qualifiedName, name); makeSimpleTag (qualifiedName, PerlKinds, kind); vStringDelete (qualifiedName); } } vStringClear (name); } } END_MAIN_WHILE: vStringDelete (name); if (package != NULL) vStringDelete (package); }
static boolean loadPathKind (xcmdPath *const path, char* line, char *args[]) { const char* backup = line; char* off; vString *desc; kindOption *kind; if (line[0] == '\0') return FALSE; else if (!isblank(line[1])) { error (WARNING, "[%s] a space after letter is not found in kind description line: %s", args[0], backup); return FALSE; } path->kinds = xRealloc (path->kinds, path->n_kinds + 1, kindOption); kind = &path->kinds [path->n_kinds]; memset (kind, 0, sizeof (*kind)); kind->enabled = TRUE; kind->letter = line[0]; kind->name = NULL; kind->description = NULL; kind->referenceOnly = FALSE; kind->nRoles = 0; kind->roles = NULL; verbose (" kind letter: <%c>\n", kind->letter); for (line++; isblank(*line); line++) ; /* do nothing */ if (*line == '\0') { error (WARNING, "[%s] unexpectedly a kind description line is terminated: %s", args[0], backup); return FALSE; } Assert (!isblank (*line)); off = strrstr(line, "[off]"); if (off == line) { error (WARNING, "[%s] [off] is given but no kind description is found: %s", args[0], backup); return FALSE; } else if (off) { if (!isblank (*(off - 1))) { error (WARNING, "[%s] a whitespace must precede [off] flag: %s", args[0], backup); return FALSE; } kind->enabled = FALSE; *off = '\0'; } desc = vStringNewInit (line); vStringStripTrailing (desc); Assert (vStringLength (desc) > 0); kind->description = vStringDeleteUnwrap (desc); /* TODO: This conversion should be part of protocol. */ { char *tmp = eStrdup (kind->description); char *c; for (c = tmp; *c != '\0'; c++) { if (*c == ' ' || *c == '\t') *c = '_'; } kind->name = tmp; } path->n_kinds += 1; return TRUE; }
static vString* extractVimFileType(FILE* input) { /* http://vimdoc.sourceforge.net/htmldoc/options.html#modeline [text]{white}{vi:|vim:|ex:}[white]se[t] {options}:[text] options=> filetype=TYPE or ft=TYPE 'modelines' 'mls' number (default 5) global {not in Vi} If 'modeline' is on 'modelines' gives the number of lines that is checked for set commands. */ vString* filetype = NULL; #define RING_SIZE 5 vString* ring[RING_SIZE]; int i, j; unsigned int k; const char* const prefix[] = { "vim:", "vi:", "ex:" }; for (i = 0; i < RING_SIZE; i++) ring[i] = vStringNew (); i = 0; while ((readLine (ring[i++], input)) != NULL) if (i == RING_SIZE) i = 0; j = i; do { const char* p; j--; if (j < 0) j = RING_SIZE - 1; for (k = 0; k < ARRAY_SIZE(prefix); k++) if ((p = strstr (vStringValue (ring[j]), prefix[k])) != NULL) { p += strlen(prefix[k]); for ( ; isspace ((int) *p) ; ++p) ; /* no-op */ filetype = determineVimFileType(p); break; } } while (((i == RING_SIZE)? (j != RING_SIZE - 1): (j != i)) && (!filetype)); for (i = RING_SIZE - 1; i >= 0; i--) vStringDelete (ring[i]); #undef RING_SIZE if (filetype && (vStringLength (filetype) == 0)) { vStringDelete (filetype); filetype = NULL; } return filetype; /* TODO: [text]{white}{vi:|vim:|ex:}[white]{options} */ }