PreprocessedContents tokenizeFromByteArray(const QByteArray& array) { PreprocessedContents to; const char* data = array.constData(); const char* dataEnd = data + array.size(); //unsigned int* target = to.data(); KDevVarLengthArray<char, 100> identifier; KDevelop::IndexedString::RunningHash hash; bool tokenizing = false; while(data < dataEnd) { if(!tokenizing) { if(isLetter(*data) || *data == '_') tokenizing = true; } if(tokenizing) { if(isLetterOrNumber(*data) || *data == '_') { hash.append(*data); identifier.append(*data); }else{ //End of token to.append( KDevelop::IndexedString(identifier.constData(), identifier.size(), hash.hash).index() ); //kDebug() << "word" << "\"" + KDevelop::IndexedString(to.back()).str() + "\""; hash.clear(); identifier.clear(); tokenizing = false; } } if(!tokenizing) to.append( indexFromCharacter(*data) ); ++data; } if(tokenizing) to.append( KDevelop::IndexedString(identifier.constData(), identifier.size(), hash.hash).index() ); /* kDebug() << QString::fromUtf8(stringFromContents(to)); kDebug() << QString::fromUtf8(array); Q_ASSERT(stringFromContents(to) == array);*/ return to; }