static CXCursor getCursorUsingTokens(CXTranslationUnit tu, CXCursor cursor, unsigned int desiredOffset) { // The following does not return a more definitive cursor. // For example, if a compound statement is returned, this does not find // any variables in the statement. CXSourceRange range = clang_getCursorExtent(cursor); CXToken *tokens; unsigned numTokens; clang_tokenize(tu, range, &tokens, &numTokens); unsigned int closestOffset = 0; for(unsigned int i=0; i<numTokens; i++) { CXSourceLocation loc = clang_getTokenLocation(tu, tokens[i]); unsigned int offset; CXFile file; clang_getSpellingLocation(loc, &file, nullptr, nullptr, &offset); if(offset < desiredOffset && offset > closestOffset) { closestOffset = offset; cursor = clang_getCursor(tu, loc); } } clang_disposeTokens(tu, tokens, numTokens); return cursor; }
highlight_group(clang::translation_unit const &trans_unit, clang::token_pack &tokens) { auto &tu(trans_unit.impl); std::vector<CXCursor> cursors(tokens.size()); clang_annotateTokens(tu, tokens.begin(), tokens.size(), cursors.data()); auto cursor(cursors.cbegin()); for(auto token(tokens.cbegin()); token != tokens.cend(); ++token, ++cursor) { CXTokenKind const kind{ clang_getTokenKind(*token) }; clang::string const spell{ clang_getTokenSpelling(tu, *token) }; auto const loc(clang_getTokenLocation(tu, *token)); CXFile file{}; unsigned line{}, column{}, offset{}; clang_getFileLocation(loc, &file, &line, &column, &offset); auto const cur(*cursor); auto const cursor_kind(cur.kind); auto const cursor_type(clang_getCursorType(cur).kind); auto const mapped(clang::token::map_token_kind(kind, cursor_kind, cursor_type)); if(mapped.size()) { //std::cout << spell.c_str() << " : " << mapped << std::endl; emplace_back(mapped, line, column, spell.c_str()); } else { /* std::cout << "unmapped: " << spell.c_str() << std::endl; */ } } }
int ClangParser::Private::getCurrentTokenLine() { uint l, c; if (numTokens==0) return 1; // guard against filters that reduce the number of lines if (curToken>=numTokens) curToken=numTokens-1; CXSourceLocation start = clang_getTokenLocation(tu,tokens[curToken]); clang_getSpellingLocation(start, 0, &l, &c, 0); return l; }
std::vector<CXCursor> my_annotateTokens( CXTranslationUnit tu, CXToken *tokens, unsigned token_count ) { std::vector<CXCursor> cursors( token_count ); for ( auto n = 0u; n < token_count; ++n ) { auto location = clang_getTokenLocation( tu, tokens[ n ] ); cursors[n] = ( clang_getCursor( tu, location ) ); } return cursors; }
void QAnnotatedTokenSet::dump(std::string &str){ Q_D(QAnnotatedTokenSet); CXString displayName = clang_getCursorDisplayName(d->cursor); str.append("Cursor :"); str.append(clang_getCString(displayName)); str.append("\n Tokens :"); clang_disposeString(displayName); for ( QAnnotatedTokenSet::Iterator it = begin(); it != end(); ++it ){ CXString tokenString = clang_getTokenSpelling(d->translationUnit, (*it)->token().token); CXSourceLocation tokenLocation = clang_getTokenLocation(d->translationUnit, (*it)->token().token); unsigned int column, line; clang_getSpellingLocation(tokenLocation, 0, &line, &column, 0); std::stringstream stream; stream << " (" << line << "," << column << ") \""; str.append(stream.str() + clang_getCString(tokenString) + "\""); clang_disposeString(tokenString); } str.append("\n"); }
// FIXME: change this to just printing comments, and call write_token() // directly from write_html() for non-comments. void Html_File::write_comment_split(FILE* f, CXFile file, CXToken tok) { unsigned line; unsigned column; unsigned offset; CXSourceLocation loc = clang_getTokenLocation(tu_file_->tu(), tok); clang_getExpansionLocation(loc, &file, &line, &column, &offset); CXTokenKind kind = clang_getTokenKind(tok); CXString s = clang_getTokenSpelling(tu_file_->tu(), tok); std::string sub = clang_getCString(s); clang_disposeString(s); // actually split up multi-line comments and send one at // a time -- that way each line gets line numbers. if (kind == CXToken_Comment || kind == CXToken_Literal) { std::string str = sub.c_str(); if (kind == CXToken_Comment) str = fix(sub.c_str()); size_t i; size_t begin = 0; for (i = 0; i < str.length(); ++i) { if (str[i] == '\n') { sub = str.substr(begin, i-begin); if (begin) line++; column = 1; begin = i + 1; write_token(f, file, tok, sub.c_str(), line, column); } } if (begin) { line++; column = 1; } sub = str.substr(begin, i-begin); } write_token(f, file, tok, sub.c_str(), line, column); }
std::string make_vimson_from_tokens(std::vector<Token> tokens) const { return "[" + std::accumulate(std::begin(tokens), std::end(tokens), std::string{}, [&](std::string const& acc, Token const& token){ auto const kind = clang_getTokenKind(token); auto const spell = owned(clang_getTokenSpelling(translation_unit, token)); auto const location = clang_getTokenLocation(translation_unit, token); CXFile file; unsigned int line, column, offset; clang_getFileLocation(location, &file, &line, &column, &offset); auto const source_name = owned(clang_getFileName(file)); return acc + "{'spell':'" + clang_getCString(*spell) + "','kind':'" + get_kind_spelling(kind) + "','file':'" + clang_getCString(*source_name) + "','line':" + std::to_string(line) + ",'column':" + std::to_string(column) + ",'offset':" + std::to_string(offset) + "},"; }) + "]"; }
void Html_File::write_token(FILE* f, CXFile file, CXToken tok, const char* str, unsigned line, unsigned column) { static bool preprocessor = false; static bool include = false; CXSourceLocation tloc = clang_getTokenLocation(tu_file_->tu(), tok); CXCursor c = clang_getCursor(tu_file_->tu(), tloc); if (cur_line_ <= line) cur_column_ = 1; for (; cur_line_ <= line; ++cur_line_) fprintf (f, "\n<a name=\"l%05i\"></a>%05i", cur_line_, cur_line_); for (; cur_column_ <= column; ++cur_column_) fprintf (f , " "); switch (clang_getTokenKind(tok)) { case (CXToken_Punctuation): if (str[0] == '#') preprocessor = true; fprintf(f, "%s", str); break; case (CXToken_Keyword): fprintf(f, "<span class=\"keyword\">%s</span>", str); break; case (CXToken_Comment): fprintf(f, "<span class=\"comment\">%s</span>", str); break; case (CXToken_Literal): { //include = false; // disable include links for now if (include) { include = false; // found an include file std::string t; const char* p = str; while (*p) { if (*p != '"') t += *p; ++p; } // first, use this file's path, then all the include paths bool found_include = false; char path[PATH_MAX]; std::string includefile = realpath(dirname(tu_file_->source_filename()), path); includefile += "/" + t; struct stat st; if (stat(includefile.c_str(), &st) == 0) { found_include = true; } else { for (std::vector<std::string>::const_iterator i = includes_.begin(), e = includes_.end(); i != e; ++i) { includefile = realpath((*i).c_str(), path); includefile += "/" + t; if (stat(includefile.c_str(), &st) == 0) { found_include = true; break; } } } if (found_include) { if (files_.find(includefile) != files_.end()) { t = make_filename(includefile, html_dir_, prefix_, ".html", false); fprintf(f, "<a class=\"code\" href=\"%s\" title="">%s</a>", t.c_str(), str); break; } std::map<std::string, Definition>::iterator i = defmap_.find(includefile); if (i != defmap_.end()) { t = i->second.file.c_str(); fprintf(f, "<a class=\"code\" href=\"%s\" title="">%s</a>", t.c_str(), str); break; } } } // not an include or include not found std::string s = fix(str); fprintf(f, "%s", s.c_str() ); break; } case (CXToken_Identifier): { if (preprocessor) { preprocessor = false; if (strcmp(str, "include") == 0) include = true; fprintf(f, "<span class=\"code\">%s</span>", str); break; } if (clang_isUnexposed(c.kind)) { fprintf(f, "<span class=\"code\">%s</span>", str); fprintf(f, "<!-- origin line: %i : %s : kind = %i -->", __LINE__, str, c.kind); break; } // Calling clang_getCursorDefinition() does not work properly // for template classes, i.e., it will find the method // declaration, not the definition, if they differ. However, // once you have the declaration's location, you can use it // get that cursor, and find the definition that way. CXSourceLocation decloc = clang_getCursorLocation(clang_getCursorDefinition(c)); CXCursor cref = clang_getCursorDefinition(clang_getCursor(tu_file_->tu(), decloc)); if (clang_isUnexposed(cref.kind)) { fprintf(f, "<span class=\"code\">%s</span>", str); fprintf(f, "<!-- origin line: %i : (ref) %s : kind = %i -->", __LINE__, str, cref.kind); break; } std::string rfile; std::string html_dir; unsigned refl = line; bool found = false; if (!clang_Cursor_isNull(cref) && cref.kind != CXCursor_Namespace) { CXSourceLocation refloc = clang_getCursorLocation(cref); if (!clang_equalLocations(tloc, refloc)) { CXFile cxfile; unsigned col; unsigned off; clang_getExpansionLocation(refloc, &cxfile, &refl, &col, &off); if (cxfile == file) { found = true; fprintf(f, "<!-- origin line: %i : (ref) %s : kind = %i -->", __LINE__, str, cref.kind); } else { CXString cxfn = clang_getFileName(cxfile); const char* fn = clang_getCString(cxfn); if (fn) { if (files_.find(fn) != files_.end()) { rfile = fn; found = true; fprintf(f, "<!-- origin line: %i : (ref) %s : kind = %i -->", __LINE__, str, cref.kind); } } clang_disposeString(cxfn); } } } else if (!clang_isDeclaration(c.kind) && c.kind != CXCursor_Namespace) { CXCursor ref = clang_getCursorReferenced(c); if (ref.kind != CXCursor_Namespace) { std::string fsn = munge_fullyscopedname(fullyScopedName(ref)); if (fsn.empty()) { fprintf(f, "<!-- origin line: %i : (fsn empty) %s : kind = %i -->", __LINE__, str, c.kind); } else { std::map<std::string, Definition>::iterator r = defmap_.find(fsn); if (r != defmap_.end()) { found = true; fprintf(f, "<!-- origin line: %i : %s : kind = %i -->", __LINE__, fsn.c_str(), c.kind); rfile = r->second.file.c_str(); html_dir = r->second.html_path.c_str(); refl = r->second.line; } } } } // since we are linking to lines, no need to link to same line if (found && (!rfile.empty() || refl != line)) { if (!rfile.empty()) rfile = make_filename(rfile, html_dir, prefix_, ".html", !html_dir.empty()); fprintf(f, "<a class=\"code\" href=\"%s#l%05i\" title="">%s</a>", rfile.c_str(), refl , str); break; } fprintf(f, "<span class=\"code\">%s</span>", str); break; } } cur_column_ += strlen(str); }
source_location token::location() { return { clang_getTokenLocation(tu, tok) }; }
void ClangParser::writeSources(CodeOutputInterface &ol,FileDef *fd) { TooltipManager::instance()->clearTooltips(); // (re)set global parser state g_currentDefinition=0; g_currentMemberDef=0; g_currentLine=0; g_searchForBody=FALSE; g_insideBody=FALSE; g_bracketCount=0; unsigned int line=1,column=1; QCString lineNumber,lineAnchor; ol.startCodeLine(TRUE); writeLineNumber(ol,fd,line); for (unsigned int i=0;i<p->numTokens;i++) { CXSourceLocation start = clang_getTokenLocation(p->tu, p->tokens[i]); unsigned int l, c; clang_getSpellingLocation(start, 0, &l, &c, 0); if (l > line) column = 1; while (line<l) { line++; ol.endCodeLine(); ol.startCodeLine(TRUE); writeLineNumber(ol,fd,line); } while (column<c) { ol.codify(" "); column++; } CXString tokenString = clang_getTokenSpelling(p->tu, p->tokens[i]); char const *s = clang_getCString(tokenString); CXCursorKind cursorKind = clang_getCursorKind(p->cursors[i]); CXTokenKind tokenKind = clang_getTokenKind(p->tokens[i]); //printf("%d:%d %s cursorKind=%d tokenKind=%d\n",line,column,s,cursorKind,tokenKind); switch (tokenKind) { case CXToken_Keyword: if (strcmp(s,"operator")==0) { linkIdentifier(ol,fd,line,column,s,i); } else { codifyLines(ol,fd,s,line,column, cursorKind==CXCursor_PreprocessingDirective ? "preprocessor" : keywordToType(s)); } break; case CXToken_Literal: if (cursorKind==CXCursor_InclusionDirective) { linkInclude(ol,fd,line,column,s); } else if (s[0]=='"' || s[0]=='\'') { codifyLines(ol,fd,s,line,column,"stringliteral"); } else { codifyLines(ol,fd,s,line,column); } break; case CXToken_Comment: codifyLines(ol,fd,s,line,column,"comment"); break; default: // CXToken_Punctuation or CXToken_Identifier if (tokenKind==CXToken_Punctuation) { detectFunctionBody(s); //printf("punct %s: %d\n",s,cursorKind); } switch (cursorKind) { case CXCursor_PreprocessingDirective: codifyLines(ol,fd,s,line,column,"preprocessor"); break; case CXCursor_MacroDefinition: codifyLines(ol,fd,s,line,column,"preprocessor"); break; case CXCursor_InclusionDirective: linkInclude(ol,fd,line,column,s); break; case CXCursor_MacroExpansion: linkMacro(ol,fd,line,column,s); break; default: if (tokenKind==CXToken_Identifier || (tokenKind==CXToken_Punctuation && // for operators (cursorKind==CXCursor_DeclRefExpr || cursorKind==CXCursor_MemberRefExpr || cursorKind==CXCursor_CallExpr || cursorKind==CXCursor_ObjCMessageExpr) ) ) { linkIdentifier(ol,fd,line,column,s,i); if (Doxygen::searchIndex) { ol.addWord(s,FALSE); } } else { codifyLines(ol,fd,s,line,column); } break; } } clang_disposeString(tokenString); } ol.endCodeLine(); TooltipManager::instance()->writeTooltips(ol); }
void ClangParser::writeSources(CodeOutputInterface &ol, QSharedPointer<FileDef> fd) { TooltipManager::instance()->clearTooltips(); // set global parser state g_currentDefinition = QSharedPointer<Definition>(); g_currentMemberDef = QSharedPointer<MemberDef>(); g_currentLine = 0; g_searchForBody = false; g_insideBody = false; g_bracketCount = 0; unsigned int line = 1; unsigned int column = 1; QString lineNumber; QString lineAnchor; ol.startCodeLine(true); writeLineNumber(ol, fd, line); for (unsigned int i = 0; i < p->numTokens; i++) { CXSourceLocation start = clang_getTokenLocation(p->tu, p->tokens[i]); unsigned int t_line; unsigned int t_col; clang_getSpellingLocation(start, 0, &t_line, &t_col, 0); if (t_line > line) { column = 1; } while (line < t_line) { line++; ol.endCodeLine(); ol.startCodeLine(true); writeLineNumber(ol, fd, line); } while (column < t_col) { ol.codify(" "); column++; } CXString tokenString = clang_getTokenSpelling(p->tu, p->tokens[i]); char const *s = clang_getCString(tokenString); CXCursorKind cursorKind = clang_getCursorKind(p->cursors[i]); CXTokenKind tokenKind = clang_getTokenKind(p->tokens[i]); switch (tokenKind) { case CXToken_Keyword: if (strcmp(s, "operator") == 0) { linkIdentifier(ol, fd, line, column, s, i); } else { QString temp; if (cursorKind == CXCursor_PreprocessingDirective) { temp = "preprocessor"; } else { temp = keywordToType(s); } codifyLines(ol, fd, s, line, column, temp); } break; case CXToken_Literal: if (cursorKind == CXCursor_InclusionDirective) { linkInclude(ol, fd, line, column, s); } else if (s[0] == '"' || s[0] == '\'') { codifyLines(ol, fd, s, line, column, "stringliteral"); } else { codifyLines(ol, fd, s, line, column, ""); } break; case CXToken_Comment: codifyLines(ol, fd, s, line, column, "comment"); break; default: // CXToken_Punctuation or CXToken_Identifier if (tokenKind == CXToken_Punctuation) { detectFunctionBody(s); } switch (cursorKind) { case CXCursor_PreprocessingDirective: codifyLines(ol, fd, s, line, column, "preprocessor"); break; case CXCursor_MacroDefinition: codifyLines(ol, fd, s, line, column, "preprocessor"); break; case CXCursor_InclusionDirective: linkInclude(ol, fd, line, column, s); break; case CXCursor_MacroExpansion: linkMacro(ol, fd, line, column, s); break; default: if (tokenKind == CXToken_Identifier || (tokenKind == CXToken_Punctuation && (cursorKind == CXCursor_DeclRefExpr || cursorKind == CXCursor_MemberRefExpr || cursorKind == CXCursor_CallExpr || cursorKind == CXCursor_ObjCMessageExpr)) ) { linkIdentifier(ol, fd, line, column, s, i); if (Doxy_Globals::searchIndex) { ol.addWord(s, false); } } else { codifyLines(ol, fd, s, line, column, ""); } break; } } clang_disposeString(tokenString); } ol.endCodeLine(); TooltipManager::instance()->writeTooltips(ol); }