highlight_group(clang::translation_unit const &trans_unit,
clang::token_pack &tokens)
{
auto &tu(trans_unit.impl);
std::vector<CXCursor> cursors(tokens.size());
clang_annotateTokens(tu, tokens.begin(), tokens.size(), cursors.data());
auto cursor(cursors.cbegin());
for(auto token(tokens.cbegin()); token != tokens.cend();
++token, ++cursor)
{
CXTokenKind const kind{ clang_getTokenKind(*token) };
clang::string const spell{ clang_getTokenSpelling(tu, *token) };
auto const loc(clang_getTokenLocation(tu, *token));
CXFile file{};
unsigned line{}, column{}, offset{};
clang_getFileLocation(loc, &file, &line, &column, &offset);
auto const cur(*cursor);
auto const cursor_kind(cur.kind);
auto const cursor_type(clang_getCursorType(cur).kind);
auto const mapped(clang::token::map_token_kind(kind, cursor_kind,
cursor_type));
if(mapped.size())
{
//std::cout << spell.c_str() << " : " << mapped << std::endl;
emplace_back(mapped, line, column, spell.c_str());
}
else
{ /* std::cout << "unmapped: " << spell.c_str() << std::endl; */ }
}
}
void
ast_json::record_token(
json_t& obj, CXToken token, CXTranslationUnit translation_unit)
{
// Record token kind
switch (clang_getTokenKind(token)) {
case CXToken_Punctuation:
obj[lex_token::KIND] = "punctuation";
break;
case CXToken_Keyword:
obj[lex_token::KIND] = "keyword";
break;
case CXToken_Identifier:
obj[lex_token::KIND] = "identifier";
break;
case CXToken_Literal:
obj[lex_token::KIND] = "literal";
break;
case CXToken_Comment:
obj[lex_token::KIND] = "comment";
break;
}
// Get attributes
CXString spelling = clang_getTokenSpelling(translation_unit, token);
CXSourceRange extent = clang_getTokenExtent(translation_unit, token);
obj[lex_token::DATA] = std::string(clang_getCString(spelling));
record_extent(obj, extent);
// Reclaim memory
clang_disposeString(spelling);
}
QASTField::QASTField(
QAnnotatedTokenSet *tokenSet,
QSourceLocation* cursorLocation,
QSourceLocation* rangeStartLocation,
QSourceLocation* rangeEndLocation,
QASTNode* parent)
: QASTNode("field", tokenSet, cursorLocation, rangeStartLocation, rangeEndLocation, parent){
// Get Identifier
// --------------
CXString id = clang_getCursorSpelling(tokenSet->cursor());
setIdentifier(clang_getCString(id));
// Get Field Type
// ---------------
CXType type = clang_getCursorType(tokenSet->cursor());
CXString typeSpelling = clang_getTypeSpelling(type);
m_fieldType = clang_getCString(typeSpelling);
clang_disposeString(typeSpelling);
// Determine field type
// --------------------
if ( type.kind == CXType_Unexposed || type.kind == CXType_Invalid || m_fieldType.contains("int") ){
m_fieldType = "";
bool doubleColonFlag = false;
for ( QAnnotatedTokenSet::Iterator it = tokenSet->begin(); it != tokenSet->end(); ++it ){
CXToken t = (*it)->token().token;
CXString tSpelling = clang_getTokenSpelling(tokenSet->translationUnit(), t);
const char* tCSpelling = clang_getCString(tSpelling);
CXTokenKind tKind = clang_getTokenKind(t);
if ( tKind == CXToken_Identifier && std::string(clang_getCString(id)) == tCSpelling ){
break;
} else if ( tKind == CXToken_Punctuation && std::string("::") == tCSpelling ){
doubleColonFlag = true;
m_fieldType += tCSpelling;
} else if ( ( tKind == CXToken_Identifier || tKind == CXToken_Keyword ) &&
!m_fieldType.isEmpty() && !doubleColonFlag ){
m_fieldType += QString(" ") + tCSpelling;
} else {
doubleColonFlag = false;
m_fieldType += tCSpelling;
}
}
}
clang_disposeString(id);
}
void HighlightingMark::collectKinds(CXToken *cxToken, const Cursor &cursor)
{
auto cxTokenKind = clang_getTokenKind(*cxToken);
types = HighlightingTypes();
switch (cxTokenKind) {
case CXToken_Keyword: types.mainHighlightingType = HighlightingType::Keyword; break;
case CXToken_Punctuation: types.mainHighlightingType = punctationKind(cursor); break;
case CXToken_Identifier: identifierKind(cursor, Recursion::FirstPass); break;
case CXToken_Comment: types.mainHighlightingType = HighlightingType::Comment; break;
case CXToken_Literal: types.mainHighlightingType = literalKind(cursor); break;
}
}
/// @todo - this is a duplicate of a function in ParseBase.cpp
void appendCursorTokenString(CXCursor cursor, std::string &str)
{
CXSourceRange range = clang_getCursorExtent(cursor);
CXToken *tokens = 0;
unsigned int nTokens = 0;
CXTranslationUnit tu = clang_Cursor_getTranslationUnit(cursor);
clang_tokenize(tu, range, &tokens, &nTokens);
for (size_t i = 0; i < nTokens; i++)
{
CXTokenKind kind = clang_getTokenKind(tokens[i]);
if(kind != CXToken_Comment)
{
if(str.length() != 0)
str += " ";
CXStringDisposer spelling = clang_getTokenSpelling(tu, tokens[i]);
str += spelling;
}
}
clang_disposeTokens(tu, tokens, nTokens);
}
std::string getFirstNonCommentToken(CXCursor cursor)
{
CXSourceRange range = clang_getCursorExtent(cursor);
CXToken *tokens = 0;
unsigned int nTokens = 0;
CXTranslationUnit tu = clang_Cursor_getTranslationUnit(cursor);
clang_tokenize(tu, range, &tokens, &nTokens);
std::string str;
for (size_t i = 0; i < nTokens; i++)
{
CXTokenKind kind = clang_getTokenKind(tokens[i]);
if(kind != CXToken_Comment)
{
CXStringDisposer spelling = clang_getTokenSpelling(tu, tokens[i]);
str += spelling;
break;
}
}
clang_disposeTokens(tu, tokens, nTokens);
return str;
}
// FIXME: change this to just printing comments, and call write_token()
// directly from write_html() for non-comments.
void
Html_File::write_comment_split(FILE* f, CXFile file, CXToken tok) {
unsigned line;
unsigned column;
unsigned offset;
CXSourceLocation loc = clang_getTokenLocation(tu_file_->tu(), tok);
clang_getExpansionLocation(loc, &file, &line, &column, &offset);
CXTokenKind kind = clang_getTokenKind(tok);
CXString s = clang_getTokenSpelling(tu_file_->tu(), tok);
std::string sub = clang_getCString(s);
clang_disposeString(s);
// actually split up multi-line comments and send one at
// a time -- that way each line gets line numbers.
if (kind == CXToken_Comment || kind == CXToken_Literal) {
std::string str = sub.c_str();
if (kind == CXToken_Comment)
str = fix(sub.c_str());
size_t i;
size_t begin = 0;
for (i = 0; i < str.length(); ++i) {
if (str[i] == '\n') {
sub = str.substr(begin, i-begin);
if (begin)
line++; column = 1;
begin = i + 1;
write_token(f, file, tok, sub.c_str(), line, column);
}
}
if (begin) {
line++;
column = 1;
}
sub = str.substr(begin, i-begin);
}
write_token(f, file, tok, sub.c_str(), line, column);
}
std::string make_vimson_from_tokens(std::vector<Token> tokens) const
{
return "[" + std::accumulate(std::begin(tokens), std::end(tokens), std::string{}, [&](std::string const& acc, Token const& token){
auto const kind = clang_getTokenKind(token);
auto const spell = owned(clang_getTokenSpelling(translation_unit, token));
auto const location = clang_getTokenLocation(translation_unit, token);
CXFile file;
unsigned int line, column, offset;
clang_getFileLocation(location, &file, &line, &column, &offset);
auto const source_name = owned(clang_getFileName(file));
return acc
+ "{'spell':'" + clang_getCString(*spell)
+ "','kind':'" + get_kind_spelling(kind)
+ "','file':'" + clang_getCString(*source_name)
+ "','line':" + std::to_string(line)
+ ",'column':" + std::to_string(column)
+ ",'offset':" + std::to_string(offset)
+ "},";
}) + "]";
}
void TokenRange::tokenize(CXTranslationUnit transUnit)
{
CXCursor cursor = clang_getTranslationUnitCursor(transUnit);
CXSourceRange range = clang_getCursorExtent(cursor);
CXToken *tokens = 0;
unsigned int numTokens = 0;
clang_tokenize(transUnit, range, &tokens, &numTokens);
resize(numTokens);
if(numTokens > 0)
{
for (size_t i = 0; i < numTokens-1; i++)
{
at(i).setKind(clang_getTokenKind(tokens[i]));
CXSourceRange tokRange = clang_getTokenExtent(transUnit, tokens[i]);
clang_getExpansionLocation(clang_getRangeStart(tokRange), NULL, NULL,
NULL, &at(i).mStartOffset);
clang_getExpansionLocation(clang_getRangeEnd(tokRange), NULL, NULL,
NULL, &at(i).mEndOffset);
}
}
clang_disposeTokens(transUnit, tokens, numTokens);
}
void
Html_File::write_token(FILE* f,
CXFile file,
CXToken tok,
const char* str,
unsigned line,
unsigned column)
{
static bool preprocessor = false;
static bool include = false;
CXSourceLocation tloc = clang_getTokenLocation(tu_file_->tu(), tok);
CXCursor c = clang_getCursor(tu_file_->tu(), tloc);
if (cur_line_ <= line) cur_column_ = 1;
for (; cur_line_ <= line; ++cur_line_)
fprintf (f, "\n<a name=\"l%05i\"></a>%05i", cur_line_, cur_line_);
for (; cur_column_ <= column; ++cur_column_)
fprintf (f , " ");
switch (clang_getTokenKind(tok)) {
case (CXToken_Punctuation):
if (str[0] == '#')
preprocessor = true;
fprintf(f, "%s", str);
break;
case (CXToken_Keyword):
fprintf(f, "<span class=\"keyword\">%s</span>", str);
break;
case (CXToken_Comment):
fprintf(f, "<span class=\"comment\">%s</span>", str);
break;
case (CXToken_Literal): {
//include = false; // disable include links for now
if (include) {
include = false;
// found an include file
std::string t;
const char* p = str;
while (*p) {
if (*p != '"')
t += *p;
++p;
}
// first, use this file's path, then all the include paths
bool found_include = false;
char path[PATH_MAX];
std::string includefile = realpath(dirname(tu_file_->source_filename()), path);
includefile += "/" + t;
struct stat st;
if (stat(includefile.c_str(), &st) == 0) {
found_include = true;
} else {
for (std::vector<std::string>::const_iterator i = includes_.begin(),
e = includes_.end(); i != e; ++i) {
includefile = realpath((*i).c_str(), path);
includefile += "/" + t;
if (stat(includefile.c_str(), &st) == 0) {
found_include = true;
break;
}
}
}
if (found_include) {
if (files_.find(includefile) != files_.end()) {
t = make_filename(includefile, html_dir_, prefix_, ".html", false);
fprintf(f, "<a class=\"code\" href=\"%s\" title="">%s</a>",
t.c_str(), str);
break;
}
std::map<std::string, Definition>::iterator i = defmap_.find(includefile);
if (i != defmap_.end()) {
t = i->second.file.c_str();
fprintf(f, "<a class=\"code\" href=\"%s\" title="">%s</a>",
t.c_str(), str);
break;
}
}
}
// not an include or include not found
std::string s = fix(str);
fprintf(f, "%s", s.c_str() );
break;
}
case (CXToken_Identifier): {
if (preprocessor) {
preprocessor = false;
if (strcmp(str, "include") == 0)
include = true;
fprintf(f, "<span class=\"code\">%s</span>", str);
break;
}
if (clang_isUnexposed(c.kind)) {
fprintf(f, "<span class=\"code\">%s</span>", str);
fprintf(f, "<!-- origin line: %i : %s : kind = %i -->",
__LINE__, str, c.kind);
break;
}
// Calling clang_getCursorDefinition() does not work properly
// for template classes, i.e., it will find the method
// declaration, not the definition, if they differ. However,
// once you have the declaration's location, you can use it
// get that cursor, and find the definition that way.
CXSourceLocation decloc =
clang_getCursorLocation(clang_getCursorDefinition(c));
CXCursor cref =
clang_getCursorDefinition(clang_getCursor(tu_file_->tu(),
decloc));
if (clang_isUnexposed(cref.kind)) {
fprintf(f, "<span class=\"code\">%s</span>", str);
fprintf(f, "<!-- origin line: %i : (ref) %s : kind = %i -->",
__LINE__, str, cref.kind);
break;
}
std::string rfile;
std::string html_dir;
unsigned refl = line;
bool found = false;
if (!clang_Cursor_isNull(cref) && cref.kind != CXCursor_Namespace) {
CXSourceLocation refloc = clang_getCursorLocation(cref);
if (!clang_equalLocations(tloc, refloc)) {
CXFile cxfile;
unsigned col;
unsigned off;
clang_getExpansionLocation(refloc, &cxfile, &refl, &col, &off);
if (cxfile == file) {
found = true;
fprintf(f, "<!-- origin line: %i : (ref) %s : kind = %i -->",
__LINE__, str, cref.kind);
}
else {
CXString cxfn = clang_getFileName(cxfile);
const char* fn = clang_getCString(cxfn);
if (fn) {
if (files_.find(fn) != files_.end()) {
rfile = fn;
found = true;
fprintf(f, "<!-- origin line: %i : (ref) %s : kind = %i -->",
__LINE__, str, cref.kind);
}
}
clang_disposeString(cxfn);
}
}
}
else if (!clang_isDeclaration(c.kind) && c.kind != CXCursor_Namespace) {
CXCursor ref = clang_getCursorReferenced(c);
if (ref.kind != CXCursor_Namespace) {
std::string fsn = munge_fullyscopedname(fullyScopedName(ref));
if (fsn.empty()) {
fprintf(f, "<!-- origin line: %i : (fsn empty) %s : kind = %i -->",
__LINE__, str, c.kind);
} else {
std::map<std::string, Definition>::iterator r = defmap_.find(fsn);
if (r != defmap_.end()) {
found = true;
fprintf(f, "<!-- origin line: %i : %s : kind = %i -->",
__LINE__, fsn.c_str(), c.kind);
rfile = r->second.file.c_str();
html_dir = r->second.html_path.c_str();
refl = r->second.line;
}
}
}
}
// since we are linking to lines, no need to link to same line
if (found && (!rfile.empty() || refl != line)) {
if (!rfile.empty())
rfile = make_filename(rfile, html_dir, prefix_, ".html", !html_dir.empty());
fprintf(f, "<a class=\"code\" href=\"%s#l%05i\" title="">%s</a>",
rfile.c_str(), refl , str);
break;
}
fprintf(f, "<span class=\"code\">%s</span>", str);
break;
}
}
cur_column_ += strlen(str);
}
CXTokenKind token::kind()
{
return clang_getTokenKind(tok);
}
void ClangParser::writeSources(CodeOutputInterface &ol,FileDef *fd)
{
TooltipManager::instance()->clearTooltips();
// (re)set global parser state
g_currentDefinition=0;
g_currentMemberDef=0;
g_currentLine=0;
g_searchForBody=FALSE;
g_insideBody=FALSE;
g_bracketCount=0;
unsigned int line=1,column=1;
QCString lineNumber,lineAnchor;
ol.startCodeLine(TRUE);
writeLineNumber(ol,fd,line);
for (unsigned int i=0;i<p->numTokens;i++)
{
CXSourceLocation start = clang_getTokenLocation(p->tu, p->tokens[i]);
unsigned int l, c;
clang_getSpellingLocation(start, 0, &l, &c, 0);
if (l > line) column = 1;
while (line<l)
{
line++;
ol.endCodeLine();
ol.startCodeLine(TRUE);
writeLineNumber(ol,fd,line);
}
while (column<c) { ol.codify(" "); column++; }
CXString tokenString = clang_getTokenSpelling(p->tu, p->tokens[i]);
char const *s = clang_getCString(tokenString);
CXCursorKind cursorKind = clang_getCursorKind(p->cursors[i]);
CXTokenKind tokenKind = clang_getTokenKind(p->tokens[i]);
//printf("%d:%d %s cursorKind=%d tokenKind=%d\n",line,column,s,cursorKind,tokenKind);
switch (tokenKind)
{
case CXToken_Keyword:
if (strcmp(s,"operator")==0)
{
linkIdentifier(ol,fd,line,column,s,i);
}
else
{
codifyLines(ol,fd,s,line,column,
cursorKind==CXCursor_PreprocessingDirective ? "preprocessor" :
keywordToType(s));
}
break;
case CXToken_Literal:
if (cursorKind==CXCursor_InclusionDirective)
{
linkInclude(ol,fd,line,column,s);
}
else if (s[0]=='"' || s[0]=='\'')
{
codifyLines(ol,fd,s,line,column,"stringliteral");
}
else
{
codifyLines(ol,fd,s,line,column);
}
break;
case CXToken_Comment:
codifyLines(ol,fd,s,line,column,"comment");
break;
default: // CXToken_Punctuation or CXToken_Identifier
if (tokenKind==CXToken_Punctuation)
{
detectFunctionBody(s);
//printf("punct %s: %d\n",s,cursorKind);
}
switch (cursorKind)
{
case CXCursor_PreprocessingDirective:
codifyLines(ol,fd,s,line,column,"preprocessor");
break;
case CXCursor_MacroDefinition:
codifyLines(ol,fd,s,line,column,"preprocessor");
break;
case CXCursor_InclusionDirective:
linkInclude(ol,fd,line,column,s);
break;
case CXCursor_MacroExpansion:
linkMacro(ol,fd,line,column,s);
break;
default:
if (tokenKind==CXToken_Identifier ||
(tokenKind==CXToken_Punctuation && // for operators
(cursorKind==CXCursor_DeclRefExpr ||
cursorKind==CXCursor_MemberRefExpr ||
cursorKind==CXCursor_CallExpr ||
cursorKind==CXCursor_ObjCMessageExpr)
)
)
{
linkIdentifier(ol,fd,line,column,s,i);
if (Doxygen::searchIndex)
{
ol.addWord(s,FALSE);
}
}
else
{
codifyLines(ol,fd,s,line,column);
}
break;
}
}
clang_disposeString(tokenString);
}
ol.endCodeLine();
TooltipManager::instance()->writeTooltips(ol);
}
void TokenizeSource(CXTranslationUnit tu)
{
CXSourceRange range =
clang_getCursorExtent( clang_getTranslationUnitCursor(tu) );
CXToken *tokens;
unsigned int token_count;
clang_tokenize( tu, range, &tokens, &token_count );
//CXCursor cursors[ token_count ];
//clang_annotateTokens( tu, tokens, token_count, cursors );
auto cursors = my_annotateTokens( tu, tokens, token_count );
for ( auto t = 0u; t < token_count; ++t ) {
auto tkind = clang_getTokenKind(tokens[t] );
auto tspelling = tkind == CXToken_Identifier ?
CursorKindSpelling( cursors[ t ] ) :
TokenKindSpelling( tkind );
auto textent = clang_getTokenExtent( tu, tokens[t] );
auto tstartloc = clang_getRangeStart( textent );
auto tendloc = clang_getRangeEnd( textent );
auto tokspell = clang_getTokenSpelling( tu, tokens[ t ] );
std::cout << "TokenSpelling: " << tokspell << "\n";
std::cout << "Cursor: " << cursors[ t ] << "\n";
// if ( !( cursors[t].kind >= CXCursor_FirstInvalid &&
// cursors[t].kind <= CXCursor_LastInvalid ) ) {
// auto rr = clang_getCursorExtent( cursors[ t ] );
// std::cout << "Range: " << rr << "\n";
// }
// std::cout << clang_getCursorDisplayName( cursors[ t ] ) << "\n";
// std::cout << "USR: "******"\n";
unsigned int startoffset, endoffset;
clang_getSpellingLocation( tstartloc, nullptr, nullptr, nullptr, &startoffset );
clang_getSpellingLocation( tendloc, nullptr, nullptr, nullptr, &endoffset );
// TODO: testing this hack for int -> identifier instead of keyword
// but this loses const to an identifier also! fvck!
if ( tspelling == "Keyword" ) {
auto type = clang_getCursorType( cursors[ t ] );
auto typekind = type.kind;
CXString typespelling;
if ( cursors[t].kind == CXCursor_FunctionDecl ||
cursors[t].kind == CXCursor_CXXMethod ) {
type = clang_getResultType( type );
typekind = type.kind;
typespelling = clang_getTypeSpelling( type );
}
else
typespelling = clang_getTypeSpelling( type );
// std::cout << "Type = " << type << " kind: " << typekind << "\n";
// std::cout << clang_getCString(typespelling) << " <-> " << clang_getCString(tokspell) << "\n";
// std::cout << " Const? " << clang_isConstQualifiedType( type ) << "\n";
if ( (( typekind >= CXType_FirstBuiltin && typekind <= CXType_LastBuiltin ) &&
( std::string(clang_getCString(typespelling)) ==
std::string(clang_getCString(tokspell) ) )) ||
// ( cursors[t].kind == CXCursor_VarDecl ) ||
( cursors[t].kind == CXCursor_ParmDecl ) )
tspelling = "Identifier";
}
//if ( tspelling != "Punctuation" )
std::cout
<< startoffset << ":" << endoffset << " @ "
<< tspelling << "\n";
clang_disposeString( tokspell );
}
std::cout << "\n" << end_pattern << "\n";
clang_disposeTokens( tu, tokens, token_count );
}
enum CXChildVisitResult visit_program(CXCursor cursor, CXCursor parent, CXClientData data){
json_t* program = (json_t*)data;
json_t* js = NULL;
enum CXChildVisitResult ret = CXChildVisit_Continue;
switch (clang_getCursorKind(cursor)) {
case CXCursor_FunctionDecl:
case CXCursor_VarDecl:
js = json_object();
json_object_set_new(js, "name", render_string(clang_getCursorSpelling(cursor)));
json_object_set_new(js, "display_name", render_string(clang_getCursorDisplayName(cursor)));
json_object_set_new(js, "type", render_type(clang_getCursorType(cursor)));
if (clang_getCursorKind(cursor) == CXCursor_FunctionDecl) {
json_object_set_new(js, "argument_names", json_array());
json_object_set_new(js, "kind", json_string("function"));
} else {
json_object_set_new(js, "kind", json_string("variable"));
}
switch (clang_getCursorLinkage(cursor)) {
case CXLinkage_Internal:
json_object_set_new(js, "linkage", json_string("static"));
break;
case CXLinkage_UniqueExternal:
json_object_set_new(js, "linkage", json_string("anonymous"));
break;
case CXLinkage_External:
break;
default:
json_object_set_new(js, "linkage", json_string("unknown"));
}
clang_visitChildren(cursor, visit_object, (CXClientData)js);
break;
case CXCursor_StructDecl:
case CXCursor_UnionDecl:
js = json_object();
json_object_set_new(js, "kind", json_string(clang_getCursorKind(cursor) == CXCursor_UnionDecl ?
"union" : "struct"));
json_object_set_new(js, "name", render_string(clang_getCursorSpelling(cursor)));
json_object_set_new(js, "fields", json_array());
clang_visitChildren(cursor, visit_structure, (CXClientData)js);
ret = CXChildVisit_Recurse;
break;
case CXCursor_EnumDecl:
js = json_object();
json_object_set_new(js, "kind", json_string("enum"));
json_object_set_new(js, "name", render_string(clang_getCursorSpelling(cursor)));
json_object_set_new(js, "values", json_array());
clang_visitChildren(cursor, visit_enum, (CXClientData)js);
break;
case CXCursor_TypedefDecl:
js = json_object();
json_object_set_new(js, "kind", json_string("typedef"));
json_object_set_new(js, "name", render_string(clang_getCursorSpelling(cursor)));
json_object_set_new(js, "type", render_type(clang_getTypedefDeclUnderlyingType(cursor)));
ret = CXChildVisit_Recurse;
break;
case CXCursor_FieldDecl:
break;
case CXCursor_MacroDefinition:
js = json_object();
json_object_set_new(js, "kind", json_string("macro"));
{
CXToken* tokens;
unsigned ntokens, i;
CXString name = clang_getCursorSpelling(cursor);
char value_buf[1024] = "";
json_object_set_new(js, "name", render_string(clang_getCursorSpelling(cursor)));
clang_tokenize(TU, clang_getCursorExtent(cursor), &tokens, &ntokens);
for (i=0; i<ntokens; i++) {
CXString str = clang_getTokenSpelling(TU, tokens[i]);
CXTokenKind tkind = clang_getTokenKind(tokens[i]);
if (i == 0 && !strcmp(clang_getCString(name), clang_getCString(str))) {
// macro name
} else if (i == ntokens - 1 &&
tkind == CXToken_Punctuation && !strcmp("#", clang_getCString(str))) {
// weird clang terminator thingy
} else if (tkind == CXToken_Comment) {
// comment
} else {
if (strlen(value_buf) > 0) {
strncat(value_buf, " ", sizeof(value_buf) - strlen(value_buf) - 1);
}
strncat(value_buf, clang_getCString(str), sizeof(value_buf) - strlen(value_buf) - 1);
}
clang_disposeString(str);
}
clang_disposeTokens(TU, tokens, ntokens);
if (strlen(value_buf) > 0) {
long int intval;
double dblval;
char* endptr_int;
char* endptr_dbl;
intval = strtol(value_buf, &endptr_int, 0);
dblval = strtod(value_buf, &endptr_dbl);
if (endptr_int[0] == 0 ||
(endptr_int[1] == 0 && strchr("UuLl", endptr_int[0]) != NULL)) {
json_object_set_new(js, "value", json_integer(intval));
} else if (endptr_dbl[0] == 0 ||
(endptr_dbl[1] == 0 && strchr("fF", endptr_dbl[0]) != NULL)) {
json_object_set_new(js, "value", json_real(dblval));
} else {
json_object_set_new(js, "value", json_string(value_buf));
}
}
}
break;
/*
case CXCursor_PreprocessingDirective:
case CXCursor_MacroExpansion:
js = json_object();
json_object_set_new(js, "kind", json_string("macro"));
json_object_set_new(js, "name", render_string(clang_getCursorSpelling(cursor)));
putstring(clang_getCursorSpelling(cursor));
putstring(clang_getCursorDisplayName(cursor));
printf("%d\n", clang_getEnumConstantDeclValue(cursor));
ret = CXChildVisit_Recurse;
*/
default:
js = json_object();
json_object_set_new(js, "name", render_string(clang_getCursorSpelling(cursor)));
json_object_set_new(js, "type", render_type(clang_getCursorType(cursor)));
json_object_set_new(js, "kind", json_string("wtf"));
json_object_set_new(js, "wtf", render_string(clang_getCursorKindSpelling(clang_getCursorKind(cursor))));
}
if (js) {
json_t* str;
if (!json_object_get(program, "")) json_object_set_new(program, "", json_array());
str = render_string(clang_getCursorUSR(cursor));
if (strlen(json_string_value(str)) == 0) {
json_decref(str);
json_array_append_new(json_object_get(program, ""), js);
} else {
json_object_set_with_key_new(program, render_string(clang_getCursorUSR(cursor)), js);
}
}
return ret;
}
int perform_token_annotation(int argc, const char **argv) {
const char *input = argv[1];
char *filename = 0;
unsigned line, second_line;
unsigned column, second_column;
CXIndex CIdx;
CXTranslationUnit TU = 0;
int errorCode;
struct CXUnsavedFile *unsaved_files = 0;
int num_unsaved_files = 0;
CXToken *tokens;
unsigned num_tokens;
CXSourceRange range;
CXSourceLocation startLoc, endLoc;
CXFile file = 0;
CXCursor *cursors = 0;
unsigned i;
input += strlen("-test-annotate-tokens=");
if ((errorCode = parse_file_line_column(input, &filename, &line, &column,
&second_line, &second_column)))
return errorCode;
if (parse_remapped_files(argc, argv, 2, &unsaved_files, &num_unsaved_files))
return -1;
CIdx = clang_createIndex(0, 1);
TU = clang_createTranslationUnitFromSourceFile(CIdx, argv[argc - 1],
argc - num_unsaved_files - 3,
argv + num_unsaved_files + 2,
num_unsaved_files,
unsaved_files);
if (!TU) {
fprintf(stderr, "unable to parse input\n");
clang_disposeIndex(CIdx);
free(filename);
free_remapped_files(unsaved_files, num_unsaved_files);
return -1;
}
errorCode = 0;
file = clang_getFile(TU, filename);
if (!file) {
fprintf(stderr, "file %s is not in this translation unit\n", filename);
errorCode = -1;
goto teardown;
}
startLoc = clang_getLocation(TU, file, line, column);
if (clang_equalLocations(clang_getNullLocation(), startLoc)) {
fprintf(stderr, "invalid source location %s:%d:%d\n", filename, line,
column);
errorCode = -1;
goto teardown;
}
endLoc = clang_getLocation(TU, file, second_line, second_column);
if (clang_equalLocations(clang_getNullLocation(), endLoc)) {
fprintf(stderr, "invalid source location %s:%d:%d\n", filename,
second_line, second_column);
errorCode = -1;
goto teardown;
}
range = clang_getRange(startLoc, endLoc);
clang_tokenize(TU, range, &tokens, &num_tokens);
cursors = (CXCursor *)malloc(num_tokens * sizeof(CXCursor));
clang_annotateTokens(TU, tokens, num_tokens, cursors);
for (i = 0; i != num_tokens; ++i) {
const char *kind = "<unknown>";
CXString spelling = clang_getTokenSpelling(TU, tokens[i]);
CXSourceRange extent = clang_getTokenExtent(TU, tokens[i]);
unsigned start_line, start_column, end_line, end_column;
switch (clang_getTokenKind(tokens[i])) {
case CXToken_Punctuation: kind = "Punctuation"; break;
case CXToken_Keyword: kind = "Keyword"; break;
case CXToken_Identifier: kind = "Identifier"; break;
case CXToken_Literal: kind = "Literal"; break;
case CXToken_Comment: kind = "Comment"; break;
}
clang_getInstantiationLocation(clang_getRangeStart(extent),
0, &start_line, &start_column, 0);
clang_getInstantiationLocation(clang_getRangeEnd(extent),
0, &end_line, &end_column, 0);
printf("%s: \"%s\" ", kind, clang_getCString(spelling));
PrintExtent(stdout, start_line, start_column, end_line, end_column);
if (!clang_isInvalid(cursors[i].kind)) {
printf(" ");
PrintCursor(cursors[i]);
}
printf("\n");
}
free(cursors);
teardown:
PrintDiagnostics(TU);
clang_disposeTranslationUnit(TU);
clang_disposeIndex(CIdx);
free(filename);
free_remapped_files(unsaved_files, num_unsaved_files);
return errorCode;
}
void ClangParser::writeSources(CodeOutputInterface &ol, QSharedPointer<FileDef> fd)
{
TooltipManager::instance()->clearTooltips();
// set global parser state
g_currentDefinition = QSharedPointer<Definition>();
g_currentMemberDef = QSharedPointer<MemberDef>();
g_currentLine = 0;
g_searchForBody = false;
g_insideBody = false;
g_bracketCount = 0;
unsigned int line = 1;
unsigned int column = 1;
QString lineNumber;
QString lineAnchor;
ol.startCodeLine(true);
writeLineNumber(ol, fd, line);
for (unsigned int i = 0; i < p->numTokens; i++) {
CXSourceLocation start = clang_getTokenLocation(p->tu, p->tokens[i]);
unsigned int t_line;
unsigned int t_col;
clang_getSpellingLocation(start, 0, &t_line, &t_col, 0);
if (t_line > line) {
column = 1;
}
while (line < t_line) {
line++;
ol.endCodeLine();
ol.startCodeLine(true);
writeLineNumber(ol, fd, line);
}
while (column < t_col) {
ol.codify(" ");
column++;
}
CXString tokenString = clang_getTokenSpelling(p->tu, p->tokens[i]);
char const *s = clang_getCString(tokenString);
CXCursorKind cursorKind = clang_getCursorKind(p->cursors[i]);
CXTokenKind tokenKind = clang_getTokenKind(p->tokens[i]);
switch (tokenKind) {
case CXToken_Keyword:
if (strcmp(s, "operator") == 0) {
linkIdentifier(ol, fd, line, column, s, i);
} else {
QString temp;
if (cursorKind == CXCursor_PreprocessingDirective) {
temp = "preprocessor";
} else {
temp = keywordToType(s);
}
codifyLines(ol, fd, s, line, column, temp);
}
break;
case CXToken_Literal:
if (cursorKind == CXCursor_InclusionDirective) {
linkInclude(ol, fd, line, column, s);
} else if (s[0] == '"' || s[0] == '\'') {
codifyLines(ol, fd, s, line, column, "stringliteral");
} else {
codifyLines(ol, fd, s, line, column, "");
}
break;
case CXToken_Comment:
codifyLines(ol, fd, s, line, column, "comment");
break;
default:
// CXToken_Punctuation or CXToken_Identifier
if (tokenKind == CXToken_Punctuation) {
detectFunctionBody(s);
}
switch (cursorKind) {
case CXCursor_PreprocessingDirective:
codifyLines(ol, fd, s, line, column, "preprocessor");
break;
case CXCursor_MacroDefinition:
codifyLines(ol, fd, s, line, column, "preprocessor");
break;
case CXCursor_InclusionDirective:
linkInclude(ol, fd, line, column, s);
break;
case CXCursor_MacroExpansion:
linkMacro(ol, fd, line, column, s);
break;
default:
if (tokenKind == CXToken_Identifier || (tokenKind == CXToken_Punctuation &&
(cursorKind == CXCursor_DeclRefExpr || cursorKind == CXCursor_MemberRefExpr ||
cursorKind == CXCursor_CallExpr || cursorKind == CXCursor_ObjCMessageExpr)) ) {
linkIdentifier(ol, fd, line, column, s, i);
if (Doxy_Globals::searchIndex) {
ol.addWord(s, false);
}
} else {
codifyLines(ol, fd, s, line, column, "");
}
break;
}
}
clang_disposeString(tokenString);
}
ol.endCodeLine();
TooltipManager::instance()->writeTooltips(ol);
}