PreprocessedContents tokenizeFromByteArray(const QByteArray& array) {
  PreprocessedContents to;
  
  const char* data = array.constData();
  const char* dataEnd = data + array.size();
  //unsigned int* target = to.data();
  
  KDevVarLengthArray<char, 100> identifier;
  
  KDevelop::IndexedString::RunningHash hash;

  bool tokenizing = false;
  
  while(data < dataEnd) {
    
    if(!tokenizing) {
      if(isLetter(*data) || *data == '_')
        tokenizing = true;
    }
    
    if(tokenizing) {
      if(isLetterOrNumber(*data) || *data == '_') {
        hash.append(*data);
        identifier.append(*data);
      }else{
        //End of token
        to.append( KDevelop::IndexedString(identifier.constData(), identifier.size(), hash.hash).index() );
        //kDebug() << "word" << "\"" + KDevelop::IndexedString(to.back()).str() + "\"";
        hash.clear();
        identifier.clear();
        tokenizing = false;
      }
    }
    
    if(!tokenizing)
      to.append( indexFromCharacter(*data) );
    ++data;
  }
  
  if(tokenizing)
    to.append( KDevelop::IndexedString(identifier.constData(), identifier.size(), hash.hash).index() );
  
  
/*  kDebug() << QString::fromUtf8(stringFromContents(to));
  kDebug() << QString::fromUtf8(array);
  Q_ASSERT(stringFromContents(to) == array);*/
  
  return to;
}