void
ast_json::record_token(
json_t& obj, CXToken token, CXTranslationUnit translation_unit)
{
// Record token kind
switch (clang_getTokenKind(token)) {
case CXToken_Punctuation:
obj[lex_token::KIND] = "punctuation";
break;
case CXToken_Keyword:
obj[lex_token::KIND] = "keyword";
break;
case CXToken_Identifier:
obj[lex_token::KIND] = "identifier";
break;
case CXToken_Literal:
obj[lex_token::KIND] = "literal";
break;
case CXToken_Comment:
obj[lex_token::KIND] = "comment";
break;
}
// Get attributes
CXString spelling = clang_getTokenSpelling(translation_unit, token);
CXSourceRange extent = clang_getTokenExtent(translation_unit, token);
obj[lex_token::DATA] = std::string(clang_getCString(spelling));
record_extent(obj, extent);
// Reclaim memory
clang_disposeString(spelling);
}
void Highlight::highlightBlock(const QString &text)
{
CXToken *tokens;
unsigned tokenCount;
clang_tokenize(cx_tu,range, &tokens, &tokenCount);
if (tokenCount > 0)
{
CXCursor *cursors = NULL;
cursors = (CXCursor *)calloc(sizeof(CXCursor), tokenCount);
clang_annotateTokens(cx_tu, tokens, tokenCount, cursors);
for (unsigned i=0 ; i<tokenCount ; i++)
{
CXSourceRange sr = clang_getTokenExtent(cx_tu, tokens[i]);
unsigned start, end;
CXSourceLocation s = clang_getRangeStart(sr);
CXSourceLocation e = clang_getRangeEnd(sr);
clang_getInstantiationLocation(s, 0, 0, 0, &start);
clang_getInstantiationLocation(e, 0, 0, 0, &end);
qDebug()<<"start:"<<start<<"end:"<<end;
/* if(start >end) {
int tmp = start;
start = end;
end = tmp;
}
*/
switch (cursors[i].kind)
{
case CXCursor_FirstRef... CXCursor_LastRef:
break;
case CXCursor_MacroDefinition:
break;
case CXCursor_MacroInstantiation:
break;
case CXCursor_FirstDecl...CXCursor_LastDecl:
break;
case CXCursor_ObjCMessageExpr:
break;
case CXCursor_DeclRefExpr:
break;
case CXCursor_PreprocessingDirective: {
}
break;
default:
break;
}
/* if(cursors[i].kind == CXCursor_FunctionDecl)
setFormat(start, end-start, m_formatSingleLineComment);*/
}
}
static int getTokenIndex(CXTranslationUnit tu, const Tokens &tokens, uint line, uint column)
{
int tokenIndex = -1;
for (int i = static_cast<int>(tokens.tokenCount - 1); i >= 0; --i) {
const SourceRange range(tu, clang_getTokenExtent(tu, tokens.data[i]));
if (range.contains(line, column)) {
tokenIndex = i;
break;
}
}
return tokenIndex;
}
HighlightingMark::HighlightingMark(const CXCursor &cxCursor,
CXToken *cxToken,
CXTranslationUnit cxTranslationUnit)
{
const SourceRange sourceRange = clang_getTokenExtent(cxTranslationUnit, *cxToken);
const auto start = sourceRange.start();
const auto end = sourceRange.end();
originalCursor = cxCursor;
line = start.line();
column = start.column();
length = end.offset() - start.offset();
collectKinds(cxToken, originalCursor);
}
void TokenRange::tokenize(CXTranslationUnit transUnit)
{
CXCursor cursor = clang_getTranslationUnitCursor(transUnit);
CXSourceRange range = clang_getCursorExtent(cursor);
CXToken *tokens = 0;
unsigned int numTokens = 0;
clang_tokenize(transUnit, range, &tokens, &numTokens);
resize(numTokens);
if(numTokens > 0)
{
for (size_t i = 0; i < numTokens-1; i++)
{
at(i).setKind(clang_getTokenKind(tokens[i]));
CXSourceRange tokRange = clang_getTokenExtent(transUnit, tokens[i]);
clang_getExpansionLocation(clang_getRangeStart(tokRange), NULL, NULL,
NULL, &at(i).mStartOffset);
clang_getExpansionLocation(clang_getRangeEnd(tokRange), NULL, NULL,
NULL, &at(i).mEndOffset);
}
}
clang_disposeTokens(transUnit, tokens, numTokens);
}
void printTreeRecursive(struct treeNode* node, CXTranslationUnit cxtup) {
depth++;
if(node->modified == 0) {
CXSourceRange range = clang_getCursorExtent(node->cursor);
CXSourceLocation rstart = clang_getRangeStart(range);
if(clang_Location_isFromMainFile(rstart) != 0) {
enum CXCursorKind cursorkind = clang_getCursorKind(node->cursor);
if(depth == 1 && cursorkind != CXCursor_MacroExpansion) {
nodenum++;
if(cursorkind == CXCursor_StructDecl) {
CXToken* currTokens;
unsigned int numCurrTokens;
clang_tokenize(cxtup, range, &currTokens, &numCurrTokens);
CXToken token = currTokens[numCurrTokens-1];
CXString tokenstring = clang_getTokenSpelling(cxtup, token);
char* tokenstr = clang_getCString(tokenstring);
clang_disposeTokens(cxtup, currTokens, numCurrTokens);
//printf("\ntoken: %s\n", tokenstr);
if(strcmp(tokenstr, ";")) {
clang_disposeString(tokenstring);
// Print nothing, do nothing.
} else {
clang_disposeString(tokenstring);
goto PRINT;
}
} else {
PRINT:
1+1;
CXToken* tokens;
unsigned int numTokens;
clang_tokenize(cxtup, range, &tokens, &numTokens);
CXString tokenstring;
CXSourceRange tokenrange;
unsigned* startcol = malloc(sizeof(unsigned));
unsigned* startline = malloc(sizeof(unsigned));
unsigned* endcol = malloc(sizeof(unsigned));
unsigned* endline = malloc(sizeof(unsigned));
for(int i = 0; i<numTokens; i++) {
tokenrange = clang_getTokenExtent(cxtup, tokens[i]);
tokenstring = clang_getTokenSpelling(cxtup, tokens[i]);
CXSourceLocation currend = clang_getRangeEnd(tokenrange);
clang_getFileLocation(currend, NULL, endline, endcol, NULL);
if(prevcol != 0) {
CXSourceLocation currstart = clang_getRangeStart(tokenrange);
clang_getFileLocation(currstart, NULL, startline, startcol, NULL);
//printf("L%u-%u, C%u-%u", *startline, prevline, *startcol, prevcol);
int startl = *startline;
int startc = *startcol;
for(int i = 0; i < startl-prevline; i++) {
printf("\n");
//printf("*startline-prevline = %u, i = %u, prevline = %u\n", start-prevline, i, prevline);
}
if(startc-prevcol >= 0) {
for(int i = 0; i < startc-prevcol; i++) {
printf(" ");
}
} else {
for(int i = 1; i < startc; i++) {
printf(" ");
}
}
}
char* tstr = clang_getCString(tokenstring);
/*if(i == 0 && !strcmp(tstr, lastPrintedToken)) {
// Do nothing, print nothing.
} else {*/
printf("%s", tstr);
//}
prevline = *endline;
prevcol = *endcol;
char* str = clang_getCString(tokenstring);
strcpy(lastPrintedToken, str);
clang_disposeString(tokenstring);
}
//printf("%i\n", lastPrintedToken);
//printf("\n");
clang_disposeTokens(cxtup, tokens, numTokens);
free(startcol);
free(startline);
free(endcol);
free(endline);
}
}
}
} else if(depth == 1) {
//printf("Modified node detected.");
if(node->validcursor == false) {
//Print modified
//printf("%s\n", node->newContent);
} else {
struct treeNode** nodes = malloc((node->modified) * sizeof(struct treeNode));
//printf("%i\n", sizeof(nodes))
nodes[node->modified-1] = NULL;
//printf("nodes[node->modified-1]: %i\n", nodes[node->modified-1]);
struct treeNode* next = NULL;
int nextnode = 0;
int smallestcol = INT_MAX;
int smallestline = INT_MAX;
int cscol = INT_MIN;
int csline = INT_MIN;
struct treeListNode* currnode = node->modifiedNodes;
CXSourceRange srange;
CXSourceLocation sloc;
bool first = true;
while(nodes[(node->modified)-1] == NULL) {
while(currnode != NULL) {
if(currnode->node->startline == -1) {
srange = clang_getCursorExtent(currnode->node->cursor);
sloc = clang_getRangeStart(srange);
clang_getFileLocation(sloc, NULL, &(currnode->node->startline), &(currnode->node->startcol), NULL);
//printf("Calculated node %i: startline: %i, startcol: %i\n", currnode->node, currnode->node->startline, currnode->node->startcol);
}
//printf("\ncurrnode->node: %i, nextnode: %i\nnstartline: %i, smallestline: %i ,csline: %i\nnstartcol: %i, smallestcol: %i, cscol: %i\n", currnode->node, nextnode, currnode->node->startline, smallestline, csline, currnode->node->startcol, smallestcol, cscol);
if(currnode->node->startline < smallestline) {
//printf("\n1\, %i < %i", currnode->node->startline, smallestline);
if((currnode->node->startline > csline) || (first == true)) {
//printf("\n2, %i > %i", currnode->node->startline);
smallestline = currnode->node->startline;
smallestcol = currnode->node->startcol;
csline = currnode->node->startline;
cscol = currnode->node->startcol;
next = currnode->node;
}
} else if(currnode->node->startline == smallestline) {
//printf("3\n");
if(currnode->node->startcol < smallestcol) {
//printf("4\n");
if((currnode->node->startcol > cscol) || (first == true)) {
//printf("5\n");
smallestline = currnode->node->startline;
smallestcol = currnode->node->startcol;
csline = currnode->node->startline;
cscol = currnode->node->startcol;
next = currnode->node;
}
}
}
//printf("\n6\n");
//printf("currnode->node: %i, nextnode: %i\nnstartline: %i, smallestline: %i ,csline: %i\nnstartcol: %i, smallestcol: %i, cscol: %i\n", currnode->node, nextnode, currnode->node->startline, smallestline, csline, currnode->node->startcol, smallestcol, cscol);
currnode = currnode->next;
}
first = false;
//printf("%i ", nodes[nextnode]);
//printf("%i\n", next);
nodes[nextnode] = next;
nextnode++;
smallestcol = INT_MAX;
smallestline = INT_MAX;
currnode = node->modifiedNodes;
}
for(int i = 0; i <= 1; i++) {
//printf("nodenum: %i, nstartline: %i, nstartcol: %i\n", i, nodes[i]->startline, nodes[i]->startcol);
}
CXSourceRange range = clang_getCursorExtent(node->cursor);
CXToken* tokens;
unsigned int numTokens;
int numNodes = nextnode;
nextnode = 0;
clang_tokenize(cxtup, range, &tokens, &numTokens);
for(int i = 0; i<numTokens; i++) {
CXSourceRange tokenrange = clang_getTokenExtent(cxtup, tokens[i]);
CXString tokenstring = clang_getTokenSpelling(cxtup, tokens[i]);
unsigned startline;
unsigned startcol;
unsigned endline;
unsigned endcol;
CXSourceLocation currend = clang_getRangeEnd(tokenrange);
clang_getFileLocation(currend, NULL, &endline, &endcol, NULL);
if(prevcol != 0) {
CXSourceLocation currstart = clang_getRangeStart(tokenrange);
clang_getFileLocation(currstart, NULL, &startline, &startcol, NULL);
//printf("L%u-%u, C%u-%u", *startline, prevline, *startcol, prevcol);
int startl = startline;
int startc = startcol;
for(int i = 0; i < startl-prevline; i++) {
printf("\n");
//printf("*startline-prevline = %u, i = %u, prevline = %u\n", start-prevline, i, prevline);
}
if(startc-prevcol >= 0) {
for(int i = 0; i < startc-prevcol; i++) {
printf(" ");
}
} else {
for(int i = 0; i < startc; i++) {
printf(" ");
}
}
}
char* tstr = clang_getCString(tokenstring);
if(i == 0 && !strcmp(tstr, lastPrintedToken)) {
// Do nothing, print nothing.
} else {
//printf("nextstartline: %i, startline: %i\n", nodes[nextnode]->startline, startline);
if((nextnode < numNodes) && (nodes[nextnode]->startline == startline) && (nodes[nextnode]->startcol == startcol)) {
int* nodenext = &(nodes[nextnode]->cursor);
if(nodenext == NULL) {
printf("%s\n", nodes[nextnode]->newContent);
nextnode++;
} else {
printf("%s", nodes[nextnode]->newContent);
prevline = endline;
prevcol = endcol;
nextnode++;
}
} else {
printf("%s", tstr);
prevline = endline;
prevcol = endcol;
}
}
//printf("%i\n", sizeof(nodes));
char* str = clang_getCString(tokenstring);
strcpy(lastPrintedToken, str);
clang_disposeString(tokenstring);
}
clang_disposeTokens(cxtup, tokens, numTokens);
free(nodes);
}
}
if(node->children != NULL) {
struct treeListNode* childlist = node->children;
while(childlist != NULL) {
printTreeRecursive(childlist->node, cxtup);
childlist = childlist->next;
}
}
depth--;
}
source_range token::extent()
{
return { clang_getTokenExtent(tu, tok) };
}
void TokenizeSource(CXTranslationUnit tu)
{
CXSourceRange range =
clang_getCursorExtent( clang_getTranslationUnitCursor(tu) );
CXToken *tokens;
unsigned int token_count;
clang_tokenize( tu, range, &tokens, &token_count );
//CXCursor cursors[ token_count ];
//clang_annotateTokens( tu, tokens, token_count, cursors );
auto cursors = my_annotateTokens( tu, tokens, token_count );
for ( auto t = 0u; t < token_count; ++t ) {
auto tkind = clang_getTokenKind(tokens[t] );
auto tspelling = tkind == CXToken_Identifier ?
CursorKindSpelling( cursors[ t ] ) :
TokenKindSpelling( tkind );
auto textent = clang_getTokenExtent( tu, tokens[t] );
auto tstartloc = clang_getRangeStart( textent );
auto tendloc = clang_getRangeEnd( textent );
auto tokspell = clang_getTokenSpelling( tu, tokens[ t ] );
std::cout << "TokenSpelling: " << tokspell << "\n";
std::cout << "Cursor: " << cursors[ t ] << "\n";
// if ( !( cursors[t].kind >= CXCursor_FirstInvalid &&
// cursors[t].kind <= CXCursor_LastInvalid ) ) {
// auto rr = clang_getCursorExtent( cursors[ t ] );
// std::cout << "Range: " << rr << "\n";
// }
// std::cout << clang_getCursorDisplayName( cursors[ t ] ) << "\n";
// std::cout << "USR: "******"\n";
unsigned int startoffset, endoffset;
clang_getSpellingLocation( tstartloc, nullptr, nullptr, nullptr, &startoffset );
clang_getSpellingLocation( tendloc, nullptr, nullptr, nullptr, &endoffset );
// TODO: testing this hack for int -> identifier instead of keyword
// but this loses const to an identifier also! fvck!
if ( tspelling == "Keyword" ) {
auto type = clang_getCursorType( cursors[ t ] );
auto typekind = type.kind;
CXString typespelling;
if ( cursors[t].kind == CXCursor_FunctionDecl ||
cursors[t].kind == CXCursor_CXXMethod ) {
type = clang_getResultType( type );
typekind = type.kind;
typespelling = clang_getTypeSpelling( type );
}
else
typespelling = clang_getTypeSpelling( type );
// std::cout << "Type = " << type << " kind: " << typekind << "\n";
// std::cout << clang_getCString(typespelling) << " <-> " << clang_getCString(tokspell) << "\n";
// std::cout << " Const? " << clang_isConstQualifiedType( type ) << "\n";
if ( (( typekind >= CXType_FirstBuiltin && typekind <= CXType_LastBuiltin ) &&
( std::string(clang_getCString(typespelling)) ==
std::string(clang_getCString(tokspell) ) )) ||
// ( cursors[t].kind == CXCursor_VarDecl ) ||
( cursors[t].kind == CXCursor_ParmDecl ) )
tspelling = "Identifier";
}
//if ( tspelling != "Punctuation" )
std::cout
<< startoffset << ":" << endoffset << " @ "
<< tspelling << "\n";
clang_disposeString( tokspell );
}
std::cout << "\n" << end_pattern << "\n";
clang_disposeTokens( tu, tokens, token_count );
}
int perform_token_annotation(int argc, const char **argv) {
const char *input = argv[1];
char *filename = 0;
unsigned line, second_line;
unsigned column, second_column;
CXIndex CIdx;
CXTranslationUnit TU = 0;
int errorCode;
struct CXUnsavedFile *unsaved_files = 0;
int num_unsaved_files = 0;
CXToken *tokens;
unsigned num_tokens;
CXSourceRange range;
CXSourceLocation startLoc, endLoc;
CXFile file = 0;
CXCursor *cursors = 0;
unsigned i;
input += strlen("-test-annotate-tokens=");
if ((errorCode = parse_file_line_column(input, &filename, &line, &column,
&second_line, &second_column)))
return errorCode;
if (parse_remapped_files(argc, argv, 2, &unsaved_files, &num_unsaved_files))
return -1;
CIdx = clang_createIndex(0, 1);
TU = clang_createTranslationUnitFromSourceFile(CIdx, argv[argc - 1],
argc - num_unsaved_files - 3,
argv + num_unsaved_files + 2,
num_unsaved_files,
unsaved_files);
if (!TU) {
fprintf(stderr, "unable to parse input\n");
clang_disposeIndex(CIdx);
free(filename);
free_remapped_files(unsaved_files, num_unsaved_files);
return -1;
}
errorCode = 0;
file = clang_getFile(TU, filename);
if (!file) {
fprintf(stderr, "file %s is not in this translation unit\n", filename);
errorCode = -1;
goto teardown;
}
startLoc = clang_getLocation(TU, file, line, column);
if (clang_equalLocations(clang_getNullLocation(), startLoc)) {
fprintf(stderr, "invalid source location %s:%d:%d\n", filename, line,
column);
errorCode = -1;
goto teardown;
}
endLoc = clang_getLocation(TU, file, second_line, second_column);
if (clang_equalLocations(clang_getNullLocation(), endLoc)) {
fprintf(stderr, "invalid source location %s:%d:%d\n", filename,
second_line, second_column);
errorCode = -1;
goto teardown;
}
range = clang_getRange(startLoc, endLoc);
clang_tokenize(TU, range, &tokens, &num_tokens);
cursors = (CXCursor *)malloc(num_tokens * sizeof(CXCursor));
clang_annotateTokens(TU, tokens, num_tokens, cursors);
for (i = 0; i != num_tokens; ++i) {
const char *kind = "<unknown>";
CXString spelling = clang_getTokenSpelling(TU, tokens[i]);
CXSourceRange extent = clang_getTokenExtent(TU, tokens[i]);
unsigned start_line, start_column, end_line, end_column;
switch (clang_getTokenKind(tokens[i])) {
case CXToken_Punctuation: kind = "Punctuation"; break;
case CXToken_Keyword: kind = "Keyword"; break;
case CXToken_Identifier: kind = "Identifier"; break;
case CXToken_Literal: kind = "Literal"; break;
case CXToken_Comment: kind = "Comment"; break;
}
clang_getInstantiationLocation(clang_getRangeStart(extent),
0, &start_line, &start_column, 0);
clang_getInstantiationLocation(clang_getRangeEnd(extent),
0, &end_line, &end_column, 0);
printf("%s: \"%s\" ", kind, clang_getCString(spelling));
PrintExtent(stdout, start_line, start_column, end_line, end_column);
if (!clang_isInvalid(cursors[i].kind)) {
printf(" ");
PrintCursor(cursors[i]);
}
printf("\n");
}
free(cursors);
teardown:
PrintDiagnostics(TU);
clang_disposeTranslationUnit(TU);
clang_disposeIndex(CIdx);
free(filename);
free_remapped_files(unsaved_files, num_unsaved_files);
return errorCode;
}
void TextEdit::slotReparse()
{
//connect(this,SIGNAL(textChanged()), this,SLOT(slotReparse()));
QTextCursor tc = textCursor();
tc.select(QTextCursor::WordUnderCursor);
QString prefix = tc.selectedText();
completionList->clear();
qDebug()<<prefix;
//clang_disposeTranslationUnit(cx_tu);
struct CXUnsavedFile unsaved_file;
unsaved_file.Filename = "/tmp/a.m";
unsaved_file.Contents = strdup(this->toPlainText().toStdString().c_str());
unsaved_file.Length = this->toPlainText().length();
clang_reparseTranslationUnit(
cx_tu, 1, &unsaved_file, CXTranslationUnit_PrecompiledPreamble);
CXFile file = clang_getFile(cx_tu, unsaved_file.Filename);
CXSourceLocation start = clang_getLocationForOffset(cx_tu, file, 0);
CXSourceLocation end = clang_getLocationForOffset(cx_tu, file, this->toPlainText().length());
CXSourceRange range= clang_getRange(start,end);
CXToken *tokens;
unsigned tokenCount;
clang_tokenize(cx_tu,range, &tokens, &tokenCount);
if (tokenCount > 0)
{
CXCursor *cursors = NULL;
cursors = (CXCursor *)calloc(sizeof(CXCursor), tokenCount);
clang_annotateTokens(cx_tu, tokens, tokenCount, cursors);
for (unsigned i=0 ; i<tokenCount ; i++)
{
CXSourceRange sr = clang_getTokenExtent(cx_tu, tokens[i]);
unsigned start, end;
CXSourceLocation s = clang_getRangeStart(sr);
CXSourceLocation e = clang_getRangeEnd(sr);
clang_getInstantiationLocation(s, 0, 0, 0, &start);
clang_getInstantiationLocation(e, 0, 0, 0, &end);
qDebug()<<"start:"<<start<<"end:"<<end;
/* if(start >end) {
int tmp = start;
start = end;
end = tmp;
}
*/
switch (cursors[i].kind)
{
case CXCursor_FirstRef... CXCursor_LastRef:
break;
case CXCursor_MacroDefinition:
break;
case CXCursor_MacroInstantiation:
break;
case CXCursor_FirstDecl...CXCursor_LastDecl:
break;
case CXCursor_ObjCMessageExpr:
break;
case CXCursor_DeclRefExpr:
break;
case CXCursor_PreprocessingDirective: {
}
break;
default:
break;
}
/* if(cursors[i].kind == CXCursor_FunctionDecl)
setFormat(start, end-start, m_formatSingleLineComment);*/
}
}
