void Shell::saveShellHistory()
{
#ifdef CONFIG_READLINE
// Only save history for interactive sessions
if (_listenOnSocket)
return;
// Construct the path name of the history file
QStringList pathList = QString(mt_history_file).split("/", QString::SkipEmptyParts);
pathList.pop_back();
QString path = pathList.join("/");
// Create history path, if it does not exist
if (!QDir::home().exists(path) && !QDir::home().mkpath(path))
debugerr("Error creating path for saving the history");
// Save the history for the next launch
QString histFile = QDir::home().absoluteFilePath(mt_history_file);
QByteArray ba = histFile.toLocal8Bit();
int ret = write_history(ba.constData());
if (ret)
debugerr("Error #" << ret << " occured when trying to save the "
"history data to \"" << histFile << "\"");
#endif
}
QString MemSpecParser::findGeneratedHeaders() const
{
QLinkedList<QString> fileNames, dirs, nextDirs;
for (int i = 0; generated_includes[i] != 0; ++i)
fileNames += QString(generated_includes[i]);
QDir kdir(_kernelSrcDir);
if (!kdir.cd("include"))
memSpecParserError(QString("Include directory \"%1\" cannot be found.")
.arg(kdir.absolutePath()));
QString ret;
dirs.append(kdir.absolutePath());
// Implemented a BFS of files
QLinkedList<QString>::iterator f_it;
while (!fileNames.isEmpty() && !dirs.isEmpty()) {
// Change to the next directory
QDir currDir(dirs.first());
dirs.pop_front();
// Search for all missing files in current directory
f_it = fileNames.begin();
while (f_it != fileNames.end()) {
QFileInfo info(currDir, *f_it);
if (info.exists() && info.isFile()) {
ret += "#include <" + kdir.relativeFilePath(info.absoluteFilePath()) + ">\n";
f_it = fileNames.erase(f_it);
}
else
++f_it;
}
// Append all directories to the list
foreach (QString d, currDir.entryList(QDir::Dirs|QDir::NoDotAndDotDot))
nextDirs.push_back(currDir.absoluteFilePath(d));
// Switch to the next directory level
if (dirs.isEmpty()) {
dirs = nextDirs;
nextDirs.clear();
}
}
// Did we find all headers? If not, then try to compile it anyway...
if (!fileNames.isEmpty()) {
QString s;
for (f_it = fileNames.begin(); f_it != fileNames.end(); ++f_it)
s += "\n" + *f_it;
debugerr("Failed to find the following headers in \""
<< kdir.absolutePath() << "\":" << s);
}
return ret;
}
void Shell::prepareReadline()
{
#ifdef CONFIG_READLINE
// Enable readline history
using_history();
// Load the readline history
QString histFile = QDir::home().absoluteFilePath(mt_history_file);
if (QFile::exists(histFile)) {
int ret = read_history(histFile.toLocal8Bit().constData());
if (ret)
debugerr("Error #" << ret << " occured when trying to read the "
"history data from \"" << histFile << "\"");
}
#endif
}
bool MemoryDump::verifyPhysMemSize(quint64 *expectedSize) const
{
// We can only verify the size if we have a size given
if (!_file || _file->size() <= 0)
return true;
Instance size = queryInstance("num_physpages");
if (!size.isValid()) {
debugerr("Failed to query variable \"num_physpages\".");
return true;
}
quint64 nr_pages = size.toULong();
if (expectedSize)
*expectedSize = nr_pages << 12;
// Make sure the no. of pages matches approximately +-10%
quint64 avail_pages = _file->size() >> 12;
return (avail_pages * 0.9 <= nr_pages) && (avail_pages * 1.1 >= nr_pages);
}
float MemoryMapBuilderCS::calculateNodeProb(const Instance &inst,
float parentProbability,
MemoryMap *map)
{
// if (inst.type()->name() == "sysfs_dirent")
// debugmsg("Calculating prob. of " << inst.typeName());
// Degradation of 0.1% per parent-child relation.
static const float degPerGeneration = 0.00001;
// static const float degPerGeneration = 0.0;
// Degradation of 20% for objects not matching the magic numbers
static const float degForInvalidMagicNumbers = 0.3;
// Degradation of 5% for address begin in userland
// static const float degForUserlandAddr = 0.05;
// Degradation of 90% for an invalid address of this node
static const float degForInvalidAddr = 0.9;
// // Max. degradation of 30% for non-aligned pointer childen the type of this
// // node has
// static const float degForNonAlignedChildAddr = 0.3;
// Max. degradation of 80% for invalid pointer childen the type of this node
// has
static const float degForInvalidChildAddr = 0.8;
// Invalid Instance degradation - 99%
static const float degInvalidInstance = 0.99;
// // Invalid Pointer degradation - 90%
// static const float degInvalidPointer = 0.1;
float prob = parentProbability < 0 ?
1.0 :
parentProbability * (1.0 - degPerGeneration);
if (map && parentProbability >= 0)
map->_shared->degPerGenerationCnt++;
// Is the instance valid?
if (!MemoryMapHeuristics::isValidInstance(inst)) {
// Instance is invalid, so do not check futher
return (prob * (1.0 - degInvalidInstance));
}
// Find the BaseType of this instance, dereference any lexical type(s)
const BaseType* instType =
inst.type()->dereferencedBaseType(BaseType::trLexical);
// Function Pointer ?
if (MemoryMapHeuristics::isFunctionPointer(inst)) {
if (!MemoryMapHeuristics::isValidFunctionPointer(inst))
// Invalid function pointer that has no default value
return (prob * (1.0 - degForInvalidAddr));
return prob;
}
// Pointer ?
else if (instType && instType->type() & rtPointer) {
// Verify. Default values are fine.
if (!MemoryMapHeuristics::isValidPointer(inst))
// Invalid pointer that has no default value
return (prob * (1.0 - degForInvalidAddr));
return prob;
}
// If this a union or struct, we have to consider the pointer members
// For unions, the largest prob. of all children is taken
else if ( instType && (instType->type() & StructOrUnion) ) {
if (!inst.isValidConcerningMagicNumbers())
prob *= (1.0 - degForInvalidMagicNumbers);
int testedChildren = 0;
int invalidChildren = countInvalidChildren(
inst.dereference(BaseType::trLexical), &testedChildren);
// A count < 0 results from an invalid list_head
if (invalidChildren < 0) {
return (prob * (1.0 - degInvalidInstance));
}
else if (invalidChildren > 0) {
float invalidPct = invalidChildren / (float) testedChildren;
prob *= invalidPct * (1.0 - degForInvalidChildAddr) + (1.0 - invalidPct);
if (map)
map->_shared->degForInvalidChildAddrCnt++;
}
}
if (prob < 0 || prob > 1) {
debugerr("Probability for of type '" << inst.typeName() << "' at 0x"
<< QString::number(inst.address(), 16) << " is " << prob
<< ", should be between 0 and 1.");
prob = 0; // Make it safe
}
return prob;
}
void MemoryMap::build(MemoryMapBuilderType type, float minProbability,
const QString& slubObjFile)
{
// Set _isBuilding to true now and to false later
VarSetter<bool> building(&_isBuilding, true, false);
_buildType = type;
// Clean up everything
clearRevmap();
_shared->reset();
_shared->minProbability = minProbability;
operationStarted();
if (type == btSibi) {
_verifier.resetWatchNodes();
_probPropagation = true;
}
else {
_probPropagation = false;
}
if (!_symbols || !_vmem) {
debugerr("Factory or VirtualMemory is NULL! Aborting!");
return;
}
_useRuleEngine = true;
_knowSrc = ksNoAltTypes;
debugmsg("Building map with TYPE RULES");
// Initialization of non-rule engine based map
if (!_useRuleEngine)
_perCpuOffset = perCpuOffsets();
// NON-PARALLEL PART OF BUILDING PROCESS
// Read slubs object file, if given
if (!slubObjFile.isEmpty()) {
_verifier.parseSlubData(slubObjFile);
}
else {
debugmsg("No slub object file given.");
}
// How many threads to create?
_shared->threadCount =
qMin(qMax(MultiThreading::maxThreads(), 1), MAX_BUILDER_THREADS);
// _shared->threadCount = 1;
if (_shared->threadCount > 1)
debugmsg("Building reverse map with " << _shared->threadCount
<< " threads.");
// Enable thread safety for VirtualMemory object
bool wasThreadSafe = _vmem->setThreadSafety(_shared->threadCount > 1);
// Create the builder threads
_threads = new MemoryMapBuilder*[_shared->threadCount];
for (int i = 0; i < _shared->threadCount; ++i) {
switch (type) {
case btSibi:
_threads[i] = new MemoryMapBuilderSV(this, i);
break;
case btChrschn:
case btSlubCache:
_threads[i] = new MemoryMapBuilderCS(this, i);
break;
}
}
// Find the list of loaded kernel modules
QStringList loadedModules = loadedKernelModules();
// Go through the global vars and add their instances to the queue
for (VariableList::const_iterator it = factory()->vars().begin(),
e = factory()->vars().end(); !interrupted() && it != e; ++it)
{
const Variable* v = *it;
// // For testing now only start with this one variable
// if (v->name() != "dentry_hashtable")
// continue;
// Skip all variables from kernel modules that are not loaded
if (v->symbolSource() == ssModule) {
// We must use the rule engine to process module variables
if (!_useRuleEngine)
continue;
// Ignore symbols in the ".modinfo" and "__versions" sections, they
// can't be resolved anyway
if (v->section() == ".modinfo" || v->section() == "__versions")
continue;
// Get file name of kernel module without path
QString name = v->origFileName();
int index = name.lastIndexOf('/');
if (index >= 0)
name = name.right(name.size() - index - 1);
if (!loadedModules.contains(name))
continue;
}
// Process all variables
try {
if (_useRuleEngine)
addVariableWithRules(v);
else
addVariableWithCandidates(v);
}
catch (ExpressionEvalException& e) {
// Do nothing
}
catch (GenericException& e) {
debugerr("Caught exception for variable " << v->name()
<< " at " << e.file << ":" << e.line << ": " << e.message);
}
checkOperationProgress();
}
// Add all functions to the map, but not to the queue
for (BaseTypeList::const_iterator it = factory()->types().begin(),
e = factory()->types().end(); it != e; ++it)
{
const BaseType* t = *it;
// Skip non-kernel and non-function types
if (!t->symbolSource() == ssKernel || t->type() != rtFunction || !t->size())
continue;
const Function* f = dynamic_cast<const Function*>(t);
MemoryMapNode* node = 0;
switch(_buildType) {
case btSlubCache:
case btChrschn:
node = new MemoryMapNode(this, f->name(), f->pcLow(), t, t->id());
break;
case btSibi:
node = new MemoryMapNodeSV(this, f->name(), f->pcLow(), t, t->id(), 0, 0, false);
break;
}
_roots.append(node);
addNodeToHashes(node);
}
// The btSlubCache type ONLY adds all slub objects to the map and does not
// follow any further pointers.
if (type == btSlubCache) {
// Add all slub objects to the map
for (AddressMap::const_iterator it = _verifier.slub().objects().begin(),
e = _verifier.slub().objects().end(); it != e; ++it)
{
const SlubCache& cache = _verifier.slub().caches().at(it.value());
MemoryMapNode* node;
if (cache.baseType)
node = new MemoryMapNode(this, cache.name, it.key(),
cache.baseType, 0);
else
node = new MemoryMapNode(this, cache.name, it.key(),
cache.objSize, 0);
_roots.append(node);
addNodeToHashes(node);
}
}
// Regular mode, process all instances in the queue.
else {
for (int i = 0; i < _shared->threadCount; ++i)
_threads[i]->start();
}
// PARALLEL PART OF BUILDING PROCESS
// Let the builders do the work, but regularly output some statistics
while (!interrupted() &&
//!_shared->queue.isEmpty() &&
(!_shared->lastNode ||
_shared->lastNode->probability() >= _shared->minProbability) &&
builderRunning())
{
checkOperationProgress();
// Sleep for 100ms
usleep(100*1000);
bool isRunning = false;
for (int i = _shared->threadCount - 1; i >= 0; i--) {
if (_threads[i]->isRunning())
isRunning = true;
}
if (!isRunning)
break;
}
Console::out() << endl;
// Interrupt all threads, doesn't harm if they are not running anymore
for (int i = 0; i < _shared->threadCount; ++i)
_threads[i]->interrupt();
// Now wait for all threads and free them again
// Threads need calculateNodeProbability of thread[0] so delete that at last
for (int i = _shared->threadCount - 1; i >= 0; i--) {
if (_threads[i]->isRunning())
{
_threads[i]->wait();
}
delete _threads[i];
}
delete _threads;
_threads = 0;
// Restore previous value
_vmem->setThreadSafety(wasThreadSafe);
debugmsg("Processed " << std::dec << _shared->processed << " instances in "
<< elapsedTime() << " minutes, statistic is being generated...");
operationStopped();
// Show statistics
_verifier.statistics();
debugmsg("Processed " << std::dec << _shared->processed << " instances in "
<< elapsedTime() << " minutes.");
}
u8int read_constant_pool(FILE* f, u16int *i, u16int *cpcount, cp_info** cpinfo)
{
u16int *j= (u16int*) malloc(sizeof(u16int));
s32int* integer, *i2;
u8int *raw = malloc(4), *raw2 = malloc(8);
cp_info* info;
debugmsg("Constant pool: \n");
for(*i=0;*i < ((*cpcount)-1); (*i)++)
{
info = (cp_info*) malloc(sizeof(cp_info));
memset(info, 0, sizeof(cp_info));
info->tag = fgetc(f);
switch(info->tag)
{
case CONSTANT_Class:
{
read_class(f, j, info);
debugmsg("#%d Class #%d\n", (*i)+1, MAKE_U16( ((CONSTANT_Class_info*) info->data)->name_index ));
break;
}
case CONSTANT_Fieldref:
{
read_fieldref(f, j, info);
debugmsg("#%d Fieldref #%d.#%d\n", (*i)+1,
MAKE_U16( ((CONSTANT_Fieldref_info*)info->data)->class_index ),
MAKE_U16( ((CONSTANT_Fieldref_info*)info->data)->name_and_type_index ));
break;
}
case CONSTANT_Methodref:
{
read_methodref(f, j, info);
debugmsg("#%d Methodref #%d.#%d\n", (*i)+1,
MAKE_U16( ((CONSTANT_Methodref_info*)info->data)->class_index ),
MAKE_U16( ((CONSTANT_Methodref_info*)info->data)->name_and_type_index ));
break;
}
case CONSTANT_InterfaceMethodref:
{
read_interfacemethodref(f, j, info);
debugmsg("#%d InterfaceMethodref #%d.#%d\n", (*i)+1,
MAKE_U16( ((CONSTANT_InterfaceMethodref_info*)info->data)->class_index ),
MAKE_U16( ((CONSTANT_InterfaceMethodref_info*)info->data)->name_and_type_index ));
break;
}
case CONSTANT_String:
{
read_string(f, j, info);
debugmsg("#%d String #%d\n", (*i)+1, MAKE_U16( ((CONSTANT_String_info*)info->data)->string_index));
break;
}
case CONSTANT_Integer:
{
read_integer(f, j, info);
MAKE_S32(raw, ((CONSTANT_Integer_info*)info->data)->bytes);
integer = (s32int*) raw;
debugmsg("#%d Integer %d\n", (*i) + 1, *integer);
break;
}
case CONSTANT_Float:
{
read_float(f, j, info);
COPY_FLOAT(raw, ((CONSTANT_Float_info*)info->data)->bytes);
debugmsg("#%d Float %f\n", (*i)+1, *((float32*) raw));
break;
}
case CONSTANT_Long:
{
read_long(f, j, info);
MAKE_S64(raw2, ((CONSTANT_Long_info*)info->data)->high_bytes );
debugmsg("#%d Long %ld\n", (*i)+1, *( (s64int*) raw2));
cpinfo[(*i)++] = NULL;
break;
}
case CONSTANT_Double:
{
read_double(f, j, info);
COPY_DOUBLE(raw2, ((CONSTANT_Double_info*) info->data)->high_bytes);
debugmsg("#%d Double %f\n", (*i)+1, *( (float64*) raw2));
cpinfo[(*i)++] = NULL;
break;
}
case CONSTANT_NameAndType:
{
read_nameandtype(f, j, info);
debugmsg("#%d NameAndType #%d.#%d\n", (*i)+1, MAKE_U16( ((CONSTANT_NameAndType_info*) info->data)->name_index) , MAKE_U16(((CONSTANT_NameAndType_info*) info->data)->descriptor_index ) );
break;
}
case CONSTANT_Utf8:
{
read_utf8(f, j, info);
debugmsg("#%d Utf8 %s\n", (*i)+1, utf8_to_cstring(((CONSTANT_Utf8_info*) info->data)->bytes));
break;
}
case CONSTANT_MethodHandle:
{
read_methodhandle(f, j, info);
debugmsg("#%d MethodHandle Kind:%d #%d\n", *(i)+1, ((CONSTANT_MethodHandle_info*) info->data)->reference_kind, MAKE_U16(((CONSTANT_MethodHandle_info*) info->data)->reference_index));
break;
}
case CONSTANT_MethodType:
{
read_methodtype(f, j, info);
debugmsg("#%d MethodType #%d\n", *(i)+1, MAKE_U16(((CONSTANT_MethodType_info*) info->data)->descriptor_index));
break;
}
case CONSTANT_InvokeDynamic:
{
read_invokedynamic(f,j,info);
debugmsg("#%d InvokeDynamic #%d.#%d\n", *(i)+1, MAKE_U16(((CONSTANT_InvokeDynamic_info*) info->data)->bootstrap_method_attr_index),
MAKE_U16(((CONSTANT_InvokeDynamic_info*) info->data)->name_and_type_index));
break;
}
default: {
debugerr("Unrecognized tag: %02X!\n", info->tag);
free(info);
return 0;
}
}
cpinfo[(*i)] = info;
}
debugmsg("Done reading Constant Pool\n");
free(j);
free(raw);
free(raw2);
return 1;
}
void KernelSourceParser::parse()
{
// Make sure the source directoy exists
if (!_srcDir.exists()) {
Console::err() << "Source directory \"" << _srcDir.absolutePath()
<< "\" does not exist."
<< endl;
return;
}
if (BugReport::log())
BugReport::log()->newFile("insightd");
else
BugReport::setLog(new BugReport("insightd"));
cleanUpThreads();
_factory->seenMagicNumbers.clear();
operationStarted();
_durationLastFileFinished = _duration;
// Collect files to process
_fileNames.clear();
_bytesTotal = _bytesRead = 0;
QString fileName;
CompileUnitIntHash::const_iterator it = _factory->sources().begin();
while (it != _factory->sources().end() && !Console::interrupted()) {
const CompileUnit* unit = it.value();
// Skip assembly files
if (!unit->name().endsWith(".S")) {
// Try regular file first
fileName = unit->name() + ".i";
if (_srcDir.exists(fileName)) {
_fileNames.append(fileName);
_bytesTotal += QFileInfo(_srcDir, fileName).size();
}
else {
// Try gzip'ed file next
fileName += ".gz";
if (_srcDir.exists(fileName)) {
_fileNames.append(fileName);
_bytesTotal += QFileInfo(_srcDir, fileName).size();
}
// We expect to find all files, so this is an error!
else
shellErr(QString("File not found: %1").arg(fileName));
}
}
++it;
}
// Create worker threads, limited to single-threading for debugging
#if defined(DEBUG_APPLY_USED_AS) || defined(DEBUG_USED_AS) || defined(DEBUG_POINTS_TO)
const int THREAD_COUNT = 1;
#else
const int THREAD_COUNT = MultiThreading::maxThreads();
#endif
for (int i = 0; i < THREAD_COUNT; ++i)
{
WorkerThread* thread = new WorkerThread(this);
_threads.append(thread);
thread->start();
}
// Show progress while parsing is not finished
while (!_threads[0]->wait(250))
checkOperationProgress();
// Wait for all threads to finish
for (int i = 0; i < THREAD_COUNT; ++i)
_threads[i]->wait();
cleanUpThreads();
operationStopped();
QString s = QString("\rParsed %1/%2 files in %3.")
.arg(_filesIndex)
.arg(_fileNames.size())
.arg(elapsedTimeVerbose());
shellOut(s, true);
_factory->sourceParcingFinished();
qint32 counter = 0;
for (QMultiHash<int, int>::iterator i = _factory->seenMagicNumbers.begin();
i != _factory->seenMagicNumbers.end(); ++i)
{
const BaseType* bt = _factory->findBaseTypeById(i.key());
if (!bt) {
debugerr(QString("It seems type 0x%1 does not exist (anymore)")
.arg((uint)i.key(), 0, 16));
continue;
}
// Just to be save...
assert(bt->type() & StructOrUnion);
const Structured* str = dynamic_cast<const Structured*>(bt);
assert(i.value() >= 0 && i.value() < str->members().size());
const StructuredMember* m = str->members().at(i.value());
if (m->hasConstantIntValues())
{
counter++;
s.clear();
QList<qint64> constInt = m->constantIntValues();
for (int j = 0; j < constInt.size(); ++j) {
if (j > 0)
s += ", ";
s += QString::number(constInt[j]);
}
}
else if (m->hasConstantStringValues())
{
counter++;
s.clear();
QList<QString> constString = (m->constantStringValues());
for (int j = 0; j < constString.size(); ++j) {
if (j > 0)
s += ", ";
s += constString[j];
}
// debugmsg(QString("Found Constant: %0 %1.%2 = {%3}")
// .arg(m->refType() ? m->refType()->prettyName() : QString("(unknown)"))
// .arg(m->belongsTo()->name())
// .arg(m->name())
// .arg(s));
}
}
// debugmsg(QString("Found %1 constants").arg(counter));
// In case there were errors, show the user some information
if (BugReport::log()) {
if (BugReport::log()->entries()) {
BugReport::log()->close();
Console::out() << endl
<< BugReport::log()->bugSubmissionHint(BugReport::log()->entries());
}
delete BugReport::log();
BugReport::setLog(0);
}
}
