void
ast_json::record_token(
json_t& obj, CXToken token, CXTranslationUnit translation_unit)
{
// Record token kind
switch (clang_getTokenKind(token)) {
case CXToken_Punctuation:
obj[lex_token::KIND] = "punctuation";
break;
case CXToken_Keyword:
obj[lex_token::KIND] = "keyword";
break;
case CXToken_Identifier:
obj[lex_token::KIND] = "identifier";
break;
case CXToken_Literal:
obj[lex_token::KIND] = "literal";
break;
case CXToken_Comment:
obj[lex_token::KIND] = "comment";
break;
}
// Get attributes
CXString spelling = clang_getTokenSpelling(translation_unit, token);
CXSourceRange extent = clang_getTokenExtent(translation_unit, token);
obj[lex_token::DATA] = std::string(clang_getCString(spelling));
record_extent(obj, extent);
// Reclaim memory
clang_disposeString(spelling);
}
highlight_group(clang::translation_unit const &trans_unit,
clang::token_pack &tokens)
{
auto &tu(trans_unit.impl);
std::vector<CXCursor> cursors(tokens.size());
clang_annotateTokens(tu, tokens.begin(), tokens.size(), cursors.data());
auto cursor(cursors.cbegin());
for(auto token(tokens.cbegin()); token != tokens.cend();
++token, ++cursor)
{
CXTokenKind const kind{ clang_getTokenKind(*token) };
clang::string const spell{ clang_getTokenSpelling(tu, *token) };
auto const loc(clang_getTokenLocation(tu, *token));
CXFile file{};
unsigned line{}, column{}, offset{};
clang_getFileLocation(loc, &file, &line, &column, &offset);
auto const cur(*cursor);
auto const cursor_kind(cur.kind);
auto const cursor_type(clang_getCursorType(cur).kind);
auto const mapped(clang::token::map_token_kind(kind, cursor_kind,
cursor_type));
if(mapped.size())
{
//std::cout << spell.c_str() << " : " << mapped << std::endl;
emplace_back(mapped, line, column, spell.c_str());
}
else
{ /* std::cout << "unmapped: " << spell.c_str() << std::endl; */ }
}
}
SourceRangeContainer FollowSymbol::followSymbol(CXTranslationUnit tu,
const Cursor &fullCursor,
uint line,
uint column)
{
std::unique_ptr<Tokens> tokens(new Tokens(fullCursor));
if (!tokens->tokenCount)
tokens.reset(new Tokens(tu));
if (!tokens->tokenCount)
return SourceRangeContainer();
QVector<CXCursor> cursors(static_cast<int>(tokens->tokenCount));
clang_annotateTokens(tu, tokens->data, tokens->tokenCount, cursors.data());
int tokenIndex = getTokenIndex(tu, *tokens, line, column);
QTC_ASSERT(tokenIndex >= 0, return SourceRangeContainer());
const Utf8String tokenSpelling = ClangString(
clang_getTokenSpelling(tu, tokens->data[tokenIndex]));
if (tokenSpelling.isEmpty())
return SourceRangeContainer();
Cursor cursor{cursors[tokenIndex]};
if (cursor.kind() == CXCursor_InclusionDirective) {
CXFile file = clang_getIncludedFile(cursors[tokenIndex]);
const ClangString filename(clang_getFileName(file));
const SourceLocation loc(tu, filename, 1, 1);
return SourceRange(loc, loc);
}
// For definitions we can always find a declaration in current TU
if (cursor.isDefinition())
return extractMatchingTokenRange(cursor.canonical(), tokenSpelling);
if (!cursor.isDeclaration()) {
// This is the symbol usage
// We want to return definition
cursor = cursor.referenced();
if (cursor.isNull() || !cursor.isDefinition()) {
// We can't find definition in this TU
return SourceRangeContainer();
}
return extractMatchingTokenRange(cursor, tokenSpelling);
}
cursor = cursor.definition();
// If we are able to find a definition in current TU
if (!cursor.isNull())
return extractMatchingTokenRange(cursor, tokenSpelling);
return SourceRangeContainer();
}
QASTField::QASTField(
QAnnotatedTokenSet *tokenSet,
QSourceLocation* cursorLocation,
QSourceLocation* rangeStartLocation,
QSourceLocation* rangeEndLocation,
QASTNode* parent)
: QASTNode("field", tokenSet, cursorLocation, rangeStartLocation, rangeEndLocation, parent){
// Get Identifier
// --------------
CXString id = clang_getCursorSpelling(tokenSet->cursor());
setIdentifier(clang_getCString(id));
// Get Field Type
// ---------------
CXType type = clang_getCursorType(tokenSet->cursor());
CXString typeSpelling = clang_getTypeSpelling(type);
m_fieldType = clang_getCString(typeSpelling);
clang_disposeString(typeSpelling);
// Determine field type
// --------------------
if ( type.kind == CXType_Unexposed || type.kind == CXType_Invalid || m_fieldType.contains("int") ){
m_fieldType = "";
bool doubleColonFlag = false;
for ( QAnnotatedTokenSet::Iterator it = tokenSet->begin(); it != tokenSet->end(); ++it ){
CXToken t = (*it)->token().token;
CXString tSpelling = clang_getTokenSpelling(tokenSet->translationUnit(), t);
const char* tCSpelling = clang_getCString(tSpelling);
CXTokenKind tKind = clang_getTokenKind(t);
if ( tKind == CXToken_Identifier && std::string(clang_getCString(id)) == tCSpelling ){
break;
} else if ( tKind == CXToken_Punctuation && std::string("::") == tCSpelling ){
doubleColonFlag = true;
m_fieldType += tCSpelling;
} else if ( ( tKind == CXToken_Identifier || tKind == CXToken_Keyword ) &&
!m_fieldType.isEmpty() && !doubleColonFlag ){
m_fieldType += QString(" ") + tCSpelling;
} else {
doubleColonFlag = false;
m_fieldType += tCSpelling;
}
}
}
clang_disposeString(id);
}
/// @todo - this is a duplicate of a function in ParseBase.cpp
void appendCursorTokenString(CXCursor cursor, std::string &str)
{
CXSourceRange range = clang_getCursorExtent(cursor);
CXToken *tokens = 0;
unsigned int nTokens = 0;
CXTranslationUnit tu = clang_Cursor_getTranslationUnit(cursor);
clang_tokenize(tu, range, &tokens, &nTokens);
for (size_t i = 0; i < nTokens; i++)
{
CXTokenKind kind = clang_getTokenKind(tokens[i]);
if(kind != CXToken_Comment)
{
if(str.length() != 0)
str += " ";
CXStringDisposer spelling = clang_getTokenSpelling(tu, tokens[i]);
str += spelling;
}
}
clang_disposeTokens(tu, tokens, nTokens);
}
void QAnnotatedTokenSet::dump(std::string &str){
Q_D(QAnnotatedTokenSet);
CXString displayName = clang_getCursorDisplayName(d->cursor);
str.append("Cursor :");
str.append(clang_getCString(displayName));
str.append("\n Tokens :");
clang_disposeString(displayName);
for ( QAnnotatedTokenSet::Iterator it = begin(); it != end(); ++it ){
CXString tokenString = clang_getTokenSpelling(d->translationUnit, (*it)->token().token);
CXSourceLocation tokenLocation = clang_getTokenLocation(d->translationUnit, (*it)->token().token);
unsigned int column, line;
clang_getSpellingLocation(tokenLocation, 0, &line, &column, 0);
std::stringstream stream;
stream << " (" << line << "," << column << ") \"";
str.append(stream.str() + clang_getCString(tokenString) + "\"");
clang_disposeString(tokenString);
}
str.append("\n");
}
std::string getFirstNonCommentToken(CXCursor cursor)
{
CXSourceRange range = clang_getCursorExtent(cursor);
CXToken *tokens = 0;
unsigned int nTokens = 0;
CXTranslationUnit tu = clang_Cursor_getTranslationUnit(cursor);
clang_tokenize(tu, range, &tokens, &nTokens);
std::string str;
for (size_t i = 0; i < nTokens; i++)
{
CXTokenKind kind = clang_getTokenKind(tokens[i]);
if(kind != CXToken_Comment)
{
CXStringDisposer spelling = clang_getTokenSpelling(tu, tokens[i]);
str += spelling;
break;
}
}
clang_disposeTokens(tu, tokens, nTokens);
return str;
}
// FIXME: change this to just printing comments, and call write_token()
// directly from write_html() for non-comments.
void
Html_File::write_comment_split(FILE* f, CXFile file, CXToken tok) {
unsigned line;
unsigned column;
unsigned offset;
CXSourceLocation loc = clang_getTokenLocation(tu_file_->tu(), tok);
clang_getExpansionLocation(loc, &file, &line, &column, &offset);
CXTokenKind kind = clang_getTokenKind(tok);
CXString s = clang_getTokenSpelling(tu_file_->tu(), tok);
std::string sub = clang_getCString(s);
clang_disposeString(s);
// actually split up multi-line comments and send one at
// a time -- that way each line gets line numbers.
if (kind == CXToken_Comment || kind == CXToken_Literal) {
std::string str = sub.c_str();
if (kind == CXToken_Comment)
str = fix(sub.c_str());
size_t i;
size_t begin = 0;
for (i = 0; i < str.length(); ++i) {
if (str[i] == '\n') {
sub = str.substr(begin, i-begin);
if (begin)
line++; column = 1;
begin = i + 1;
write_token(f, file, tok, sub.c_str(), line, column);
}
}
if (begin) {
line++;
column = 1;
}
sub = str.substr(begin, i-begin);
}
write_token(f, file, tok, sub.c_str(), line, column);
}
std::string make_vimson_from_tokens(std::vector<Token> tokens) const
{
return "[" + std::accumulate(std::begin(tokens), std::end(tokens), std::string{}, [&](std::string const& acc, Token const& token){
auto const kind = clang_getTokenKind(token);
auto const spell = owned(clang_getTokenSpelling(translation_unit, token));
auto const location = clang_getTokenLocation(translation_unit, token);
CXFile file;
unsigned int line, column, offset;
clang_getFileLocation(location, &file, &line, &column, &offset);
auto const source_name = owned(clang_getFileName(file));
return acc
+ "{'spell':'" + clang_getCString(*spell)
+ "','kind':'" + get_kind_spelling(kind)
+ "','file':'" + clang_getCString(*source_name)
+ "','line':" + std::to_string(line)
+ ",'column':" + std::to_string(column)
+ ",'offset':" + std::to_string(offset)
+ "},";
}) + "]";
}
void printTreeRecursive(struct treeNode* node, CXTranslationUnit cxtup) {
depth++;
if(node->modified == 0) {
CXSourceRange range = clang_getCursorExtent(node->cursor);
CXSourceLocation rstart = clang_getRangeStart(range);
if(clang_Location_isFromMainFile(rstart) != 0) {
enum CXCursorKind cursorkind = clang_getCursorKind(node->cursor);
if(depth == 1 && cursorkind != CXCursor_MacroExpansion) {
nodenum++;
if(cursorkind == CXCursor_StructDecl) {
CXToken* currTokens;
unsigned int numCurrTokens;
clang_tokenize(cxtup, range, &currTokens, &numCurrTokens);
CXToken token = currTokens[numCurrTokens-1];
CXString tokenstring = clang_getTokenSpelling(cxtup, token);
char* tokenstr = clang_getCString(tokenstring);
clang_disposeTokens(cxtup, currTokens, numCurrTokens);
//printf("\ntoken: %s\n", tokenstr);
if(strcmp(tokenstr, ";")) {
clang_disposeString(tokenstring);
// Print nothing, do nothing.
} else {
clang_disposeString(tokenstring);
goto PRINT;
}
} else {
PRINT:
1+1;
CXToken* tokens;
unsigned int numTokens;
clang_tokenize(cxtup, range, &tokens, &numTokens);
CXString tokenstring;
CXSourceRange tokenrange;
unsigned* startcol = malloc(sizeof(unsigned));
unsigned* startline = malloc(sizeof(unsigned));
unsigned* endcol = malloc(sizeof(unsigned));
unsigned* endline = malloc(sizeof(unsigned));
for(int i = 0; i<numTokens; i++) {
tokenrange = clang_getTokenExtent(cxtup, tokens[i]);
tokenstring = clang_getTokenSpelling(cxtup, tokens[i]);
CXSourceLocation currend = clang_getRangeEnd(tokenrange);
clang_getFileLocation(currend, NULL, endline, endcol, NULL);
if(prevcol != 0) {
CXSourceLocation currstart = clang_getRangeStart(tokenrange);
clang_getFileLocation(currstart, NULL, startline, startcol, NULL);
//printf("L%u-%u, C%u-%u", *startline, prevline, *startcol, prevcol);
int startl = *startline;
int startc = *startcol;
for(int i = 0; i < startl-prevline; i++) {
printf("\n");
//printf("*startline-prevline = %u, i = %u, prevline = %u\n", start-prevline, i, prevline);
}
if(startc-prevcol >= 0) {
for(int i = 0; i < startc-prevcol; i++) {
printf(" ");
}
} else {
for(int i = 1; i < startc; i++) {
printf(" ");
}
}
}
char* tstr = clang_getCString(tokenstring);
/*if(i == 0 && !strcmp(tstr, lastPrintedToken)) {
// Do nothing, print nothing.
} else {*/
printf("%s", tstr);
//}
prevline = *endline;
prevcol = *endcol;
char* str = clang_getCString(tokenstring);
strcpy(lastPrintedToken, str);
clang_disposeString(tokenstring);
}
//printf("%i\n", lastPrintedToken);
//printf("\n");
clang_disposeTokens(cxtup, tokens, numTokens);
free(startcol);
free(startline);
free(endcol);
free(endline);
}
}
}
} else if(depth == 1) {
//printf("Modified node detected.");
if(node->validcursor == false) {
//Print modified
//printf("%s\n", node->newContent);
} else {
struct treeNode** nodes = malloc((node->modified) * sizeof(struct treeNode));
//printf("%i\n", sizeof(nodes))
nodes[node->modified-1] = NULL;
//printf("nodes[node->modified-1]: %i\n", nodes[node->modified-1]);
struct treeNode* next = NULL;
int nextnode = 0;
int smallestcol = INT_MAX;
int smallestline = INT_MAX;
int cscol = INT_MIN;
int csline = INT_MIN;
struct treeListNode* currnode = node->modifiedNodes;
CXSourceRange srange;
CXSourceLocation sloc;
bool first = true;
while(nodes[(node->modified)-1] == NULL) {
while(currnode != NULL) {
if(currnode->node->startline == -1) {
srange = clang_getCursorExtent(currnode->node->cursor);
sloc = clang_getRangeStart(srange);
clang_getFileLocation(sloc, NULL, &(currnode->node->startline), &(currnode->node->startcol), NULL);
//printf("Calculated node %i: startline: %i, startcol: %i\n", currnode->node, currnode->node->startline, currnode->node->startcol);
}
//printf("\ncurrnode->node: %i, nextnode: %i\nnstartline: %i, smallestline: %i ,csline: %i\nnstartcol: %i, smallestcol: %i, cscol: %i\n", currnode->node, nextnode, currnode->node->startline, smallestline, csline, currnode->node->startcol, smallestcol, cscol);
if(currnode->node->startline < smallestline) {
//printf("\n1\, %i < %i", currnode->node->startline, smallestline);
if((currnode->node->startline > csline) || (first == true)) {
//printf("\n2, %i > %i", currnode->node->startline);
smallestline = currnode->node->startline;
smallestcol = currnode->node->startcol;
csline = currnode->node->startline;
cscol = currnode->node->startcol;
next = currnode->node;
}
} else if(currnode->node->startline == smallestline) {
//printf("3\n");
if(currnode->node->startcol < smallestcol) {
//printf("4\n");
if((currnode->node->startcol > cscol) || (first == true)) {
//printf("5\n");
smallestline = currnode->node->startline;
smallestcol = currnode->node->startcol;
csline = currnode->node->startline;
cscol = currnode->node->startcol;
next = currnode->node;
}
}
}
//printf("\n6\n");
//printf("currnode->node: %i, nextnode: %i\nnstartline: %i, smallestline: %i ,csline: %i\nnstartcol: %i, smallestcol: %i, cscol: %i\n", currnode->node, nextnode, currnode->node->startline, smallestline, csline, currnode->node->startcol, smallestcol, cscol);
currnode = currnode->next;
}
first = false;
//printf("%i ", nodes[nextnode]);
//printf("%i\n", next);
nodes[nextnode] = next;
nextnode++;
smallestcol = INT_MAX;
smallestline = INT_MAX;
currnode = node->modifiedNodes;
}
for(int i = 0; i <= 1; i++) {
//printf("nodenum: %i, nstartline: %i, nstartcol: %i\n", i, nodes[i]->startline, nodes[i]->startcol);
}
CXSourceRange range = clang_getCursorExtent(node->cursor);
CXToken* tokens;
unsigned int numTokens;
int numNodes = nextnode;
nextnode = 0;
clang_tokenize(cxtup, range, &tokens, &numTokens);
for(int i = 0; i<numTokens; i++) {
CXSourceRange tokenrange = clang_getTokenExtent(cxtup, tokens[i]);
CXString tokenstring = clang_getTokenSpelling(cxtup, tokens[i]);
unsigned startline;
unsigned startcol;
unsigned endline;
unsigned endcol;
CXSourceLocation currend = clang_getRangeEnd(tokenrange);
clang_getFileLocation(currend, NULL, &endline, &endcol, NULL);
if(prevcol != 0) {
CXSourceLocation currstart = clang_getRangeStart(tokenrange);
clang_getFileLocation(currstart, NULL, &startline, &startcol, NULL);
//printf("L%u-%u, C%u-%u", *startline, prevline, *startcol, prevcol);
int startl = startline;
int startc = startcol;
for(int i = 0; i < startl-prevline; i++) {
printf("\n");
//printf("*startline-prevline = %u, i = %u, prevline = %u\n", start-prevline, i, prevline);
}
if(startc-prevcol >= 0) {
for(int i = 0; i < startc-prevcol; i++) {
printf(" ");
}
} else {
for(int i = 0; i < startc; i++) {
printf(" ");
}
}
}
char* tstr = clang_getCString(tokenstring);
if(i == 0 && !strcmp(tstr, lastPrintedToken)) {
// Do nothing, print nothing.
} else {
//printf("nextstartline: %i, startline: %i\n", nodes[nextnode]->startline, startline);
if((nextnode < numNodes) && (nodes[nextnode]->startline == startline) && (nodes[nextnode]->startcol == startcol)) {
int* nodenext = &(nodes[nextnode]->cursor);
if(nodenext == NULL) {
printf("%s\n", nodes[nextnode]->newContent);
nextnode++;
} else {
printf("%s", nodes[nextnode]->newContent);
prevline = endline;
prevcol = endcol;
nextnode++;
}
} else {
printf("%s", tstr);
prevline = endline;
prevcol = endcol;
}
}
//printf("%i\n", sizeof(nodes));
char* str = clang_getCString(tokenstring);
strcpy(lastPrintedToken, str);
clang_disposeString(tokenstring);
}
clang_disposeTokens(cxtup, tokens, numTokens);
free(nodes);
}
}
if(node->children != NULL) {
struct treeListNode* childlist = node->children;
while(childlist != NULL) {
printTreeRecursive(childlist->node, cxtup);
childlist = childlist->next;
}
}
depth--;
}
void ClangParser::writeSources(CodeOutputInterface &ol, QSharedPointer<FileDef> fd)
{
TooltipManager::instance()->clearTooltips();
// set global parser state
g_currentDefinition = QSharedPointer<Definition>();
g_currentMemberDef = QSharedPointer<MemberDef>();
g_currentLine = 0;
g_searchForBody = false;
g_insideBody = false;
g_bracketCount = 0;
unsigned int line = 1;
unsigned int column = 1;
QString lineNumber;
QString lineAnchor;
ol.startCodeLine(true);
writeLineNumber(ol, fd, line);
for (unsigned int i = 0; i < p->numTokens; i++) {
CXSourceLocation start = clang_getTokenLocation(p->tu, p->tokens[i]);
unsigned int t_line;
unsigned int t_col;
clang_getSpellingLocation(start, 0, &t_line, &t_col, 0);
if (t_line > line) {
column = 1;
}
while (line < t_line) {
line++;
ol.endCodeLine();
ol.startCodeLine(true);
writeLineNumber(ol, fd, line);
}
while (column < t_col) {
ol.codify(" ");
column++;
}
CXString tokenString = clang_getTokenSpelling(p->tu, p->tokens[i]);
char const *s = clang_getCString(tokenString);
CXCursorKind cursorKind = clang_getCursorKind(p->cursors[i]);
CXTokenKind tokenKind = clang_getTokenKind(p->tokens[i]);
switch (tokenKind) {
case CXToken_Keyword:
if (strcmp(s, "operator") == 0) {
linkIdentifier(ol, fd, line, column, s, i);
} else {
QString temp;
if (cursorKind == CXCursor_PreprocessingDirective) {
temp = "preprocessor";
} else {
temp = keywordToType(s);
}
codifyLines(ol, fd, s, line, column, temp);
}
break;
case CXToken_Literal:
if (cursorKind == CXCursor_InclusionDirective) {
linkInclude(ol, fd, line, column, s);
} else if (s[0] == '"' || s[0] == '\'') {
codifyLines(ol, fd, s, line, column, "stringliteral");
} else {
codifyLines(ol, fd, s, line, column, "");
}
break;
case CXToken_Comment:
codifyLines(ol, fd, s, line, column, "comment");
break;
default:
// CXToken_Punctuation or CXToken_Identifier
if (tokenKind == CXToken_Punctuation) {
detectFunctionBody(s);
}
switch (cursorKind) {
case CXCursor_PreprocessingDirective:
codifyLines(ol, fd, s, line, column, "preprocessor");
break;
case CXCursor_MacroDefinition:
codifyLines(ol, fd, s, line, column, "preprocessor");
break;
case CXCursor_InclusionDirective:
linkInclude(ol, fd, line, column, s);
break;
case CXCursor_MacroExpansion:
linkMacro(ol, fd, line, column, s);
break;
default:
if (tokenKind == CXToken_Identifier || (tokenKind == CXToken_Punctuation &&
(cursorKind == CXCursor_DeclRefExpr || cursorKind == CXCursor_MemberRefExpr ||
cursorKind == CXCursor_CallExpr || cursorKind == CXCursor_ObjCMessageExpr)) ) {
linkIdentifier(ol, fd, line, column, s, i);
if (Doxy_Globals::searchIndex) {
ol.addWord(s, false);
}
} else {
codifyLines(ol, fd, s, line, column, "");
}
break;
}
}
clang_disposeString(tokenString);
}
ol.endCodeLine();
TooltipManager::instance()->writeTooltips(ol);
}
void ClangParser::writeSources(CodeOutputInterface &ol,FileDef *fd)
{
TooltipManager::instance()->clearTooltips();
// (re)set global parser state
g_currentDefinition=0;
g_currentMemberDef=0;
g_currentLine=0;
g_searchForBody=FALSE;
g_insideBody=FALSE;
g_bracketCount=0;
unsigned int line=1,column=1;
QCString lineNumber,lineAnchor;
ol.startCodeLine(TRUE);
writeLineNumber(ol,fd,line);
for (unsigned int i=0;i<p->numTokens;i++)
{
CXSourceLocation start = clang_getTokenLocation(p->tu, p->tokens[i]);
unsigned int l, c;
clang_getSpellingLocation(start, 0, &l, &c, 0);
if (l > line) column = 1;
while (line<l)
{
line++;
ol.endCodeLine();
ol.startCodeLine(TRUE);
writeLineNumber(ol,fd,line);
}
while (column<c) { ol.codify(" "); column++; }
CXString tokenString = clang_getTokenSpelling(p->tu, p->tokens[i]);
char const *s = clang_getCString(tokenString);
CXCursorKind cursorKind = clang_getCursorKind(p->cursors[i]);
CXTokenKind tokenKind = clang_getTokenKind(p->tokens[i]);
//printf("%d:%d %s cursorKind=%d tokenKind=%d\n",line,column,s,cursorKind,tokenKind);
switch (tokenKind)
{
case CXToken_Keyword:
if (strcmp(s,"operator")==0)
{
linkIdentifier(ol,fd,line,column,s,i);
}
else
{
codifyLines(ol,fd,s,line,column,
cursorKind==CXCursor_PreprocessingDirective ? "preprocessor" :
keywordToType(s));
}
break;
case CXToken_Literal:
if (cursorKind==CXCursor_InclusionDirective)
{
linkInclude(ol,fd,line,column,s);
}
else if (s[0]=='"' || s[0]=='\'')
{
codifyLines(ol,fd,s,line,column,"stringliteral");
}
else
{
codifyLines(ol,fd,s,line,column);
}
break;
case CXToken_Comment:
codifyLines(ol,fd,s,line,column,"comment");
break;
default: // CXToken_Punctuation or CXToken_Identifier
if (tokenKind==CXToken_Punctuation)
{
detectFunctionBody(s);
//printf("punct %s: %d\n",s,cursorKind);
}
switch (cursorKind)
{
case CXCursor_PreprocessingDirective:
codifyLines(ol,fd,s,line,column,"preprocessor");
break;
case CXCursor_MacroDefinition:
codifyLines(ol,fd,s,line,column,"preprocessor");
break;
case CXCursor_InclusionDirective:
linkInclude(ol,fd,line,column,s);
break;
case CXCursor_MacroExpansion:
linkMacro(ol,fd,line,column,s);
break;
default:
if (tokenKind==CXToken_Identifier ||
(tokenKind==CXToken_Punctuation && // for operators
(cursorKind==CXCursor_DeclRefExpr ||
cursorKind==CXCursor_MemberRefExpr ||
cursorKind==CXCursor_CallExpr ||
cursorKind==CXCursor_ObjCMessageExpr)
)
)
{
linkIdentifier(ol,fd,line,column,s,i);
if (Doxygen::searchIndex)
{
ol.addWord(s,FALSE);
}
}
else
{
codifyLines(ol,fd,s,line,column);
}
break;
}
}
clang_disposeString(tokenString);
}
ol.endCodeLine();
TooltipManager::instance()->writeTooltips(ol);
}
std::string token::spelling()
{
return string(clang_getTokenSpelling(tu, tok)).str();
}
QCString ClangParser::lookup(uint line,const char *symbol)
{
//printf("ClangParser::lookup(%d,%s)\n",line,symbol);
QCString result;
if (symbol==0) return result;
static bool clangAssistedParsing = Config_getBool(CLANG_ASSISTED_PARSING);
if (!clangAssistedParsing) return result;
int sl = strlen(symbol);
uint l = p->getCurrentTokenLine();
while (l>=line && p->curToken>0)
{
if (l==line) // already at the right line
{
p->curToken--; // linear search to start of the line
l = p->getCurrentTokenLine();
}
else
{
p->curToken/=2; // binary search backward
l = p->getCurrentTokenLine();
}
}
bool found=FALSE;
while (l<=line && p->curToken<p->numTokens && !found)
{
CXString tokenString = clang_getTokenSpelling(p->tu, p->tokens[p->curToken]);
//if (l==line)
//{
// printf("try to match symbol %s with token %s\n",symbol,clang_getCString(tokenString));
//}
const char *ts = clang_getCString(tokenString);
int tl = strlen(ts);
int startIndex = p->curToken;
if (l==line && strncmp(ts,symbol,tl)==0) // found partial match at the correct line
{
int offset = tl;
while (offset<sl) // symbol spans multiple tokens
{
//printf("found partial match\n");
p->curToken++;
if (p->curToken>=p->numTokens)
{
break; // end of token stream
}
l = p->getCurrentTokenLine();
clang_disposeString(tokenString);
tokenString = clang_getTokenSpelling(p->tu, p->tokens[p->curToken]);
ts = clang_getCString(tokenString);
tl = ts ? strlen(ts) : 0;
// skip over any spaces in the symbol
char c;
while (offset<sl && ((c=symbol[offset])==' ' || c=='\t' || c=='\r' || c=='\n'))
{
offset++;
}
if (strncmp(ts,symbol+offset,tl)!=0) // next token matches?
{
//printf("no match '%s'<->'%s'\n",ts,symbol+offset);
break; // no match
}
//printf("partial match '%s'<->'%s'\n",ts,symbol+offset);
offset+=tl;
}
if (offset==sl) // symbol matches the token(s)
{
CXCursor c = p->cursors[p->curToken];
CXString usr = clang_getCursorUSR(c);
//printf("found full match %s usr='******'\n",symbol,clang_getCString(usr));
result = clang_getCString(usr);
clang_disposeString(usr);
found=TRUE;
}
else // reset token cursor to start of the search
{
p->curToken = startIndex;
}
}
clang_disposeString(tokenString);
p->curToken++;
if (p->curToken<p->numTokens)
{
l = p->getCurrentTokenLine();
}
}
//if (!found)
//{
// printf("Did not find symbol %s at line %d :-(\n",symbol,line);
//}
//else
//{
// printf("Found symbol %s usr=%s\n",symbol,result.data());
//}
return result;
}
int perform_token_annotation(int argc, const char **argv) {
const char *input = argv[1];
char *filename = 0;
unsigned line, second_line;
unsigned column, second_column;
CXIndex CIdx;
CXTranslationUnit TU = 0;
int errorCode;
struct CXUnsavedFile *unsaved_files = 0;
int num_unsaved_files = 0;
CXToken *tokens;
unsigned num_tokens;
CXSourceRange range;
CXSourceLocation startLoc, endLoc;
CXFile file = 0;
CXCursor *cursors = 0;
unsigned i;
input += strlen("-test-annotate-tokens=");
if ((errorCode = parse_file_line_column(input, &filename, &line, &column,
&second_line, &second_column)))
return errorCode;
if (parse_remapped_files(argc, argv, 2, &unsaved_files, &num_unsaved_files))
return -1;
CIdx = clang_createIndex(0, 1);
TU = clang_createTranslationUnitFromSourceFile(CIdx, argv[argc - 1],
argc - num_unsaved_files - 3,
argv + num_unsaved_files + 2,
num_unsaved_files,
unsaved_files);
if (!TU) {
fprintf(stderr, "unable to parse input\n");
clang_disposeIndex(CIdx);
free(filename);
free_remapped_files(unsaved_files, num_unsaved_files);
return -1;
}
errorCode = 0;
file = clang_getFile(TU, filename);
if (!file) {
fprintf(stderr, "file %s is not in this translation unit\n", filename);
errorCode = -1;
goto teardown;
}
startLoc = clang_getLocation(TU, file, line, column);
if (clang_equalLocations(clang_getNullLocation(), startLoc)) {
fprintf(stderr, "invalid source location %s:%d:%d\n", filename, line,
column);
errorCode = -1;
goto teardown;
}
endLoc = clang_getLocation(TU, file, second_line, second_column);
if (clang_equalLocations(clang_getNullLocation(), endLoc)) {
fprintf(stderr, "invalid source location %s:%d:%d\n", filename,
second_line, second_column);
errorCode = -1;
goto teardown;
}
range = clang_getRange(startLoc, endLoc);
clang_tokenize(TU, range, &tokens, &num_tokens);
cursors = (CXCursor *)malloc(num_tokens * sizeof(CXCursor));
clang_annotateTokens(TU, tokens, num_tokens, cursors);
for (i = 0; i != num_tokens; ++i) {
const char *kind = "<unknown>";
CXString spelling = clang_getTokenSpelling(TU, tokens[i]);
CXSourceRange extent = clang_getTokenExtent(TU, tokens[i]);
unsigned start_line, start_column, end_line, end_column;
switch (clang_getTokenKind(tokens[i])) {
case CXToken_Punctuation: kind = "Punctuation"; break;
case CXToken_Keyword: kind = "Keyword"; break;
case CXToken_Identifier: kind = "Identifier"; break;
case CXToken_Literal: kind = "Literal"; break;
case CXToken_Comment: kind = "Comment"; break;
}
clang_getInstantiationLocation(clang_getRangeStart(extent),
0, &start_line, &start_column, 0);
clang_getInstantiationLocation(clang_getRangeEnd(extent),
0, &end_line, &end_column, 0);
printf("%s: \"%s\" ", kind, clang_getCString(spelling));
PrintExtent(stdout, start_line, start_column, end_line, end_column);
if (!clang_isInvalid(cursors[i].kind)) {
printf(" ");
PrintCursor(cursors[i]);
}
printf("\n");
}
free(cursors);
teardown:
PrintDiagnostics(TU);
clang_disposeTranslationUnit(TU);
clang_disposeIndex(CIdx);
free(filename);
free_remapped_files(unsaved_files, num_unsaved_files);
return errorCode;
}
QString ClangParser::lookup(uint line, const QString &symbol)
{
QString result;
if (symbol.isEmpty()) {
return result;
}
int sl = symbol.length();
uint l = p->getCurrentTokenLine();
while (l >= line && p->curToken > 0) {
if (l == line) {
// already at the right line
p->curToken--; // linear search to start of the line
l = p->getCurrentTokenLine();
} else {
p->curToken /= 2; // binary search backward
l = p->getCurrentTokenLine();
}
}
bool found = false;
while (l <= line && p->curToken < p->numTokens && !found) {
CXString tokenString = clang_getTokenSpelling(p->tu, p->tokens[p->curToken]);
const char *ts = clang_getCString(tokenString);
int tl = strlen(ts);
int startIndex = p->curToken;
if (l == line && strncmp(ts, symbol.toUtf8(), tl) == 0) {
// found partial match at the correct line
int offset = tl;
while (offset < sl) {
// symbol spans multiple tokens
p->curToken++;
if (p->curToken >= p->numTokens) {
break; // end of token stream
}
l = p->getCurrentTokenLine();
clang_disposeString(tokenString);
tokenString = clang_getTokenSpelling(p->tu, p->tokens[p->curToken]);
ts = clang_getCString(tokenString);
tl = ts ? strlen(ts) : 0;
// skip over any spaces in the symbol
QChar c;
while (offset < sl && ((c = symbol[offset]) == ' ' || c == '\t' || c == '\r' || c == '\n')) {
offset++;
}
if (strncmp(ts, symbol.mid(offset).toUtf8(), tl) != 0) {
// next token matches?
break; // no match
}
offset += tl;
}
if (offset == sl) {
// symbol matches the token(s)
CXCursor c = p->cursors[p->curToken];
CXString usr = clang_getCursorUSR(c);
result = clang_getCString(usr);
clang_disposeString(usr);
found = true;
} else {
// reset token cursor to start of the search
p->curToken = startIndex;
}
}
clang_disposeString(tokenString);
p->curToken++;
if (p->curToken < p->numTokens) {
l = p->getCurrentTokenLine();
}
}
return result;
}
void TokenizeSource(CXTranslationUnit tu)
{
CXSourceRange range =
clang_getCursorExtent( clang_getTranslationUnitCursor(tu) );
CXToken *tokens;
unsigned int token_count;
clang_tokenize( tu, range, &tokens, &token_count );
//CXCursor cursors[ token_count ];
//clang_annotateTokens( tu, tokens, token_count, cursors );
auto cursors = my_annotateTokens( tu, tokens, token_count );
for ( auto t = 0u; t < token_count; ++t ) {
auto tkind = clang_getTokenKind(tokens[t] );
auto tspelling = tkind == CXToken_Identifier ?
CursorKindSpelling( cursors[ t ] ) :
TokenKindSpelling( tkind );
auto textent = clang_getTokenExtent( tu, tokens[t] );
auto tstartloc = clang_getRangeStart( textent );
auto tendloc = clang_getRangeEnd( textent );
auto tokspell = clang_getTokenSpelling( tu, tokens[ t ] );
std::cout << "TokenSpelling: " << tokspell << "\n";
std::cout << "Cursor: " << cursors[ t ] << "\n";
// if ( !( cursors[t].kind >= CXCursor_FirstInvalid &&
// cursors[t].kind <= CXCursor_LastInvalid ) ) {
// auto rr = clang_getCursorExtent( cursors[ t ] );
// std::cout << "Range: " << rr << "\n";
// }
// std::cout << clang_getCursorDisplayName( cursors[ t ] ) << "\n";
// std::cout << "USR: "******"\n";
unsigned int startoffset, endoffset;
clang_getSpellingLocation( tstartloc, nullptr, nullptr, nullptr, &startoffset );
clang_getSpellingLocation( tendloc, nullptr, nullptr, nullptr, &endoffset );
// TODO: testing this hack for int -> identifier instead of keyword
// but this loses const to an identifier also! fvck!
if ( tspelling == "Keyword" ) {
auto type = clang_getCursorType( cursors[ t ] );
auto typekind = type.kind;
CXString typespelling;
if ( cursors[t].kind == CXCursor_FunctionDecl ||
cursors[t].kind == CXCursor_CXXMethod ) {
type = clang_getResultType( type );
typekind = type.kind;
typespelling = clang_getTypeSpelling( type );
}
else
typespelling = clang_getTypeSpelling( type );
// std::cout << "Type = " << type << " kind: " << typekind << "\n";
// std::cout << clang_getCString(typespelling) << " <-> " << clang_getCString(tokspell) << "\n";
// std::cout << " Const? " << clang_isConstQualifiedType( type ) << "\n";
if ( (( typekind >= CXType_FirstBuiltin && typekind <= CXType_LastBuiltin ) &&
( std::string(clang_getCString(typespelling)) ==
std::string(clang_getCString(tokspell) ) )) ||
// ( cursors[t].kind == CXCursor_VarDecl ) ||
( cursors[t].kind == CXCursor_ParmDecl ) )
tspelling = "Identifier";
}
//if ( tspelling != "Punctuation" )
std::cout
<< startoffset << ":" << endoffset << " @ "
<< tspelling << "\n";
clang_disposeString( tokspell );
}
std::cout << "\n" << end_pattern << "\n";
clang_disposeTokens( tu, tokens, token_count );
}
enum CXChildVisitResult visit_program(CXCursor cursor, CXCursor parent, CXClientData data){
json_t* program = (json_t*)data;
json_t* js = NULL;
enum CXChildVisitResult ret = CXChildVisit_Continue;
switch (clang_getCursorKind(cursor)) {
case CXCursor_FunctionDecl:
case CXCursor_VarDecl:
js = json_object();
json_object_set_new(js, "name", render_string(clang_getCursorSpelling(cursor)));
json_object_set_new(js, "display_name", render_string(clang_getCursorDisplayName(cursor)));
json_object_set_new(js, "type", render_type(clang_getCursorType(cursor)));
if (clang_getCursorKind(cursor) == CXCursor_FunctionDecl) {
json_object_set_new(js, "argument_names", json_array());
json_object_set_new(js, "kind", json_string("function"));
} else {
json_object_set_new(js, "kind", json_string("variable"));
}
switch (clang_getCursorLinkage(cursor)) {
case CXLinkage_Internal:
json_object_set_new(js, "linkage", json_string("static"));
break;
case CXLinkage_UniqueExternal:
json_object_set_new(js, "linkage", json_string("anonymous"));
break;
case CXLinkage_External:
break;
default:
json_object_set_new(js, "linkage", json_string("unknown"));
}
clang_visitChildren(cursor, visit_object, (CXClientData)js);
break;
case CXCursor_StructDecl:
case CXCursor_UnionDecl:
js = json_object();
json_object_set_new(js, "kind", json_string(clang_getCursorKind(cursor) == CXCursor_UnionDecl ?
"union" : "struct"));
json_object_set_new(js, "name", render_string(clang_getCursorSpelling(cursor)));
json_object_set_new(js, "fields", json_array());
clang_visitChildren(cursor, visit_structure, (CXClientData)js);
ret = CXChildVisit_Recurse;
break;
case CXCursor_EnumDecl:
js = json_object();
json_object_set_new(js, "kind", json_string("enum"));
json_object_set_new(js, "name", render_string(clang_getCursorSpelling(cursor)));
json_object_set_new(js, "values", json_array());
clang_visitChildren(cursor, visit_enum, (CXClientData)js);
break;
case CXCursor_TypedefDecl:
js = json_object();
json_object_set_new(js, "kind", json_string("typedef"));
json_object_set_new(js, "name", render_string(clang_getCursorSpelling(cursor)));
json_object_set_new(js, "type", render_type(clang_getTypedefDeclUnderlyingType(cursor)));
ret = CXChildVisit_Recurse;
break;
case CXCursor_FieldDecl:
break;
case CXCursor_MacroDefinition:
js = json_object();
json_object_set_new(js, "kind", json_string("macro"));
{
CXToken* tokens;
unsigned ntokens, i;
CXString name = clang_getCursorSpelling(cursor);
char value_buf[1024] = "";
json_object_set_new(js, "name", render_string(clang_getCursorSpelling(cursor)));
clang_tokenize(TU, clang_getCursorExtent(cursor), &tokens, &ntokens);
for (i=0; i<ntokens; i++) {
CXString str = clang_getTokenSpelling(TU, tokens[i]);
CXTokenKind tkind = clang_getTokenKind(tokens[i]);
if (i == 0 && !strcmp(clang_getCString(name), clang_getCString(str))) {
// macro name
} else if (i == ntokens - 1 &&
tkind == CXToken_Punctuation && !strcmp("#", clang_getCString(str))) {
// weird clang terminator thingy
} else if (tkind == CXToken_Comment) {
// comment
} else {
if (strlen(value_buf) > 0) {
strncat(value_buf, " ", sizeof(value_buf) - strlen(value_buf) - 1);
}
strncat(value_buf, clang_getCString(str), sizeof(value_buf) - strlen(value_buf) - 1);
}
clang_disposeString(str);
}
clang_disposeTokens(TU, tokens, ntokens);
if (strlen(value_buf) > 0) {
long int intval;
double dblval;
char* endptr_int;
char* endptr_dbl;
intval = strtol(value_buf, &endptr_int, 0);
dblval = strtod(value_buf, &endptr_dbl);
if (endptr_int[0] == 0 ||
(endptr_int[1] == 0 && strchr("UuLl", endptr_int[0]) != NULL)) {
json_object_set_new(js, "value", json_integer(intval));
} else if (endptr_dbl[0] == 0 ||
(endptr_dbl[1] == 0 && strchr("fF", endptr_dbl[0]) != NULL)) {
json_object_set_new(js, "value", json_real(dblval));
} else {
json_object_set_new(js, "value", json_string(value_buf));
}
}
}
break;
/*
case CXCursor_PreprocessingDirective:
case CXCursor_MacroExpansion:
js = json_object();
json_object_set_new(js, "kind", json_string("macro"));
json_object_set_new(js, "name", render_string(clang_getCursorSpelling(cursor)));
putstring(clang_getCursorSpelling(cursor));
putstring(clang_getCursorDisplayName(cursor));
printf("%d\n", clang_getEnumConstantDeclValue(cursor));
ret = CXChildVisit_Recurse;
*/
default:
js = json_object();
json_object_set_new(js, "name", render_string(clang_getCursorSpelling(cursor)));
json_object_set_new(js, "type", render_type(clang_getCursorType(cursor)));
json_object_set_new(js, "kind", json_string("wtf"));
json_object_set_new(js, "wtf", render_string(clang_getCursorKindSpelling(clang_getCursorKind(cursor))));
}
if (js) {
json_t* str;
if (!json_object_get(program, "")) json_object_set_new(program, "", json_array());
str = render_string(clang_getCursorUSR(cursor));
if (strlen(json_string_value(str)) == 0) {
json_decref(str);
json_array_append_new(json_object_get(program, ""), js);
} else {
json_object_set_with_key_new(program, render_string(clang_getCursorUSR(cursor)), js);
}
}
return ret;
}
