void Fuzzer::Merge(const std::vector<std::string> &Corpora) {
if (Corpora.size() <= 1) {
Printf("Merge requires two or more corpus dirs\n");
return;
}
auto InitialCorpusDir = Corpora[0];
ReadDir(InitialCorpusDir, nullptr);
Printf("Merge: running the initial corpus '%s' of %d units\n",
InitialCorpusDir.c_str(), Corpus.size());
for (auto &U : Corpus)
RunOne(U);
std::vector<std::string> ExtraCorpora(Corpora.begin() + 1, Corpora.end());
size_t NumTried = 0;
size_t NumMerged = 0;
for (auto &C : ExtraCorpora) {
Corpus.clear();
ReadDir(C, nullptr);
Printf("Merge: merging the extra corpus '%s' of %zd units\n", C.c_str(),
Corpus.size());
for (auto &U : Corpus) {
NumTried++;
if (RunOne(U)) {
WriteToOutputCorpus(U);
NumMerged++;
}
}
}
Printf("Merge: written %zd out of %zd units\n", NumMerged, NumTried);
}
// Finds minimal number of units in 'Extra' that add coverage to 'Initial'.
// We do it by actually executing the units, sometimes more than once,
// because we may be using different coverage-like signals and the only
// common thing between them is that we can say "this unit found new stuff".
UnitVector Fuzzer::FindExtraUnits(const UnitVector &Initial,
const UnitVector &Extra) {
UnitVector Res = Extra;
size_t OldSize = Res.size();
for (int Iter = 0; Iter < 10; Iter++) {
ShuffleCorpus(&Res);
ResetCoverage();
for (auto &U : Initial)
RunOne(U);
Corpus.clear();
for (auto &U : Res)
if (RunOne(U))
Corpus.push_back(U);
char Stat[7] = "MIN ";
Stat[3] = '0' + Iter;
PrintStats(Stat);
size_t NewSize = Corpus.size();
assert(NewSize <= OldSize);
Res.swap(Corpus);
if (NewSize + 5 >= OldSize)
break;
OldSize = NewSize;
}
return Res;
}
void Fuzzer::ShuffleAndMinimize() {
bool PreferSmall = (Options.PreferSmallDuringInitialShuffle == 1 ||
(Options.PreferSmallDuringInitialShuffle == -1 &&
USF.GetRand().RandBool()));
if (Options.Verbosity)
Printf("PreferSmall: %d\n", PreferSmall);
PrintStats("READ ");
std::vector<Unit> NewCorpus;
if (Options.ShuffleAtStartUp) {
std::random_shuffle(Corpus.begin(), Corpus.end(), USF.GetRand());
if (PreferSmall)
std::stable_sort(
Corpus.begin(), Corpus.end(),
[](const Unit &A, const Unit &B) { return A.size() < B.size(); });
}
Unit &U = CurrentUnit;
for (const auto &C : Corpus) {
for (size_t First = 0; First < 1; First++) {
U.clear();
size_t Last = std::min(First + Options.MaxLen, C.size());
U.insert(U.begin(), C.begin() + First, C.begin() + Last);
if (Options.OnlyASCII)
ToASCII(U);
if (RunOne(U)) {
NewCorpus.push_back(U);
if (Options.Verbosity >= 2)
Printf("NEW0: %zd L %zd\n", LastRecordedBlockCoverage, U.size());
}
}
}
Corpus = NewCorpus;
for (auto &X : Corpus)
UnitHashesAddedToCorpus.insert(Hash(X));
PrintStats("INITED");
}
void Fuzzer::RereadOutputCorpus() {
if (Options.OutputCorpus.empty()) return;
std::vector<Unit> AdditionalCorpus;
ReadDirToVectorOfUnits(Options.OutputCorpus.c_str(), &AdditionalCorpus,
&EpochOfLastReadOfOutputCorpus);
if (Corpus.empty()) {
Corpus = AdditionalCorpus;
return;
}
if (!Options.Reload) return;
if (Options.Verbosity >= 2)
Printf("Reload: read %zd new units.\n", AdditionalCorpus.size());
for (auto &X : AdditionalCorpus) {
if (X.size() > (size_t)Options.MaxLen)
X.resize(Options.MaxLen);
if (UnitHashesAddedToCorpus.insert(Hash(X)).second) {
CurrentUnit.clear();
CurrentUnit.insert(CurrentUnit.begin(), X.begin(), X.end());
if (RunOne(CurrentUnit)) {
Corpus.push_back(X);
if (Options.Verbosity >= 1)
PrintStats("RELOAD");
}
}
}
}
void Fuzzer::ShuffleAndMinimize() {
size_t MaxCov = 0;
bool PreferSmall =
(Options.PreferSmallDuringInitialShuffle == 1 ||
(Options.PreferSmallDuringInitialShuffle == -1 && rand() % 2));
if (Options.Verbosity)
std::cerr << "PreferSmall: " << PreferSmall << "\n";
PrintStats("READ ", 0);
std::vector<Unit> NewCorpus;
std::random_shuffle(Corpus.begin(), Corpus.end());
if (PreferSmall)
std::stable_sort(
Corpus.begin(), Corpus.end(),
[](const Unit &A, const Unit &B) { return A.size() < B.size(); });
Unit &U = CurrentUnit;
for (const auto &C : Corpus) {
for (size_t First = 0; First < 1; First++) {
U.clear();
size_t Last = std::min(First + Options.MaxLen, C.size());
U.insert(U.begin(), C.begin() + First, C.begin() + Last);
size_t NewCoverage = RunOne(U);
if (NewCoverage) {
MaxCov = NewCoverage;
NewCorpus.push_back(U);
if (Options.Verbosity >= 2)
std::cerr << "NEW0: " << NewCoverage
<< " L " << U.size()
<< "\n";
}
}
}
Corpus = NewCorpus;
PrintStats("INITED", MaxCov);
}
void Fuzzer::ShuffleAndMinimize() {
if (Options.Verbosity)
std::cerr << "Shuffle: " << Corpus.size() << "\n";
std::vector<Unit> NewCorpus;
random_shuffle(Corpus.begin(), Corpus.end());
size_t MaxCov = 0;
Unit &U = CurrentUnit;
for (const auto &C : Corpus) {
for (size_t First = 0; First < 1; First++) {
U.clear();
size_t Last = std::min(First + Options.MaxLen, C.size());
U.insert(U.begin(), C.begin() + First, C.begin() + Last);
size_t NewCoverage = RunOne(U);
if (NewCoverage) {
MaxCov = NewCoverage;
NewCorpus.push_back(U);
if (Options.Verbosity >= 2)
std::cerr << "NEW0: " << NewCoverage << "\n";
}
}
}
Corpus = NewCorpus;
if (Options.Verbosity)
std::cerr << "Shuffle done: " << Corpus.size() << " IC: " << MaxCov << "\n";
}
size_t Fuzzer::MutateAndTestOne(Unit *U) {
size_t NewUnits = 0;
for (size_t i = 0; i < Options.MutateDepth; i++) {
Mutate(U, Options.MaxLen);
if (U->empty()) continue;
size_t NewCoverage = RunOne(*U);
if (NewCoverage) {
Corpus.push_back(*U);
NewUnits++;
if (Options.Verbosity) {
std::cerr << "#" << TotalNumberOfRuns
<< "\tNEW: " << NewCoverage
<< " L: " << U->size()
<< "\t";
if (U->size() < 30) {
PrintASCII(*U);
std::cerr << "\t";
Print(*U);
}
std::cerr << "\n";
}
WriteToOutputCorpus(*U);
if (Options.ExitOnFirst)
exit(0);
}
}
return NewUnits;
}
void ProxySession::Run(std::error_code& ec)
{
FileDesc client_fd(Connect(ec));
if(ec)
{
LOG(WARNING) << "Error connecting: " << ec.message();
return;
}
// now we have open fd's for both client and server we
// enter an event loop waiting for data to read from either source
FileDesc epoll_fd(epoll_create(2));
if(!epoll_fd.IsValid())
{
ec = std::error_code(errno, std::system_category());
return;
}
if(!RegisterForDataAvailable(epoll_fd, m_server_fd, ec) || !RegisterForDataAvailable(epoll_fd, client_fd, ec))
{
ec = std::error_code(errno, std::system_category());
return;
}
while(!ec)
{
RunOne(epoll_fd, client_fd, ec);
}
}
int Maker::RunCommands(bool keepGoing)
{
int rv =0 ; //FIXME rv start value!!!
bool stop = false;
Environment env;
GetEnvironment(env);
int count;
for (std::deque<Depends *>::iterator it = depends.begin(); (rv == 0 || keepGoing) && it != depends.end(); ++it)
{
CancelOne(*it);
}
for (std::deque<Depends *>::iterator it = depends.begin(); (rv == 0 || keepGoing) && it != depends.end(); ++it)
{
rv = RunOne(*it, env, keepGoing);
if (rv)
stop = true;
}
for (std::deque<Depends *>::iterator it = depends.begin(); it != depends.end(); ++it)
{
DeleteOne(*it);
}
if (stop)
return 2;
else
return rv;
}
void Fuzzer::RereadOutputCorpus(size_t MaxSize) {
if (Options.OutputCorpus.empty())
return;
std::vector<Unit> AdditionalCorpus;
ReadDirToVectorOfUnits(Options.OutputCorpus.c_str(), &AdditionalCorpus,
&EpochOfLastReadOfOutputCorpus, MaxSize);
if (Corpus.empty()) {
Corpus = AdditionalCorpus;
return;
}
if (!Options.Reload)
return;
if (Options.Verbosity >= 2)
Printf("Reload: read %zd new units.\n", AdditionalCorpus.size());
for (auto &X : AdditionalCorpus) {
if (X.size() > MaxSize)
X.resize(MaxSize);
if (UnitHashesAddedToCorpus.insert(Hash(X)).second) {
if (RunOne(X)) {
Corpus.push_back(X);
UpdateCorpusDistribution();
PrintStats("RELOAD");
}
}
}
}
void Fuzzer::RunOneAndUpdateCorpus(uint8_t *Data, size_t Size) {
if (TotalNumberOfRuns >= Options.MaxNumberOfRuns)
return;
if (Options.OnlyASCII)
ToASCII(Data, Size);
if (RunOne(Data, Size))
ReportNewCoverage({Data, Data + Size});
}
void Fuzzer::RunOneAndUpdateCorpus(Unit &U) {
if (TotalNumberOfRuns >= Options.MaxNumberOfRuns)
return;
if (Options.OnlyASCII)
ToASCII(U);
if (RunOne(U))
ReportNewCoverage(U);
}
// Experimental search heuristic: drilling.
// - Read, shuffle, execute and minimize the corpus.
// - Choose one random unit.
// - Reset the coverage.
// - Start fuzzing as if the chosen unit was the only element of the corpus.
// - When done, reset the coverage again.
// - Merge the newly created corpus into the original one.
void Fuzzer::Drill() {
// The corpus is already read, shuffled, and minimized.
assert(!Corpus.empty());
Options.PrintNEW = false; // Don't print NEW status lines when drilling.
Unit U = ChooseUnitToMutate();
CHECK_WEAK_API_FUNCTION(__sanitizer_reset_coverage);
__sanitizer_reset_coverage();
std::vector<Unit> SavedCorpus;
SavedCorpus.swap(Corpus);
Corpus.push_back(U);
assert(Corpus.size() == 1);
RunOne(U);
PrintStats("DRILL ");
std::string SavedOutputCorpusPath; // Don't write new units while drilling.
SavedOutputCorpusPath.swap(Options.OutputCorpus);
Loop();
__sanitizer_reset_coverage();
PrintStats("REINIT");
SavedOutputCorpusPath.swap(Options.OutputCorpus);
for (auto &U : SavedCorpus) {
CurrentUnit = U;
RunOne(U);
}
PrintStats("MERGE ");
Options.PrintNEW = true;
size_t NumMerged = 0;
for (auto &U : Corpus) {
CurrentUnit = U;
if (RunOne(U)) {
PrintStatusForNewUnit(U);
NumMerged++;
WriteToOutputCorpus(U);
}
}
PrintStats("MERGED");
if (NumMerged && Options.Verbosity)
Printf("Drilling discovered %zd new units\n", NumMerged);
}
static void RunAll() {
int i;
for (i = 0; i < benchmark_count; i++) {
run_data *rd = &bench_run_data[i];
bench_info *bi = &bench_info_list[i];
double usec_per_run;
printf("Running %s\n", bi->name);
++started;
//ReportStatus("Running %s (%d/%d)\n", bi->name, started, benchmark_count);
RunOne(bi, rd);
usec_per_run = (double)rd->elapsed / (double)rd->runs;
rd->score = 100.0 * bi->time_ref / usec_per_run;
printf("usec_per_run %f\n", usec_per_run);
//ReportStatus("%s: %d", bi->name, (int)rd->score);
}
started = 0;
}
void ActiveExecutor::Run()
{
while (!m_end)
{
try
{
RunOne();
}
catch (std::exception& e)
{
cdbg << "Executor: exception ignored: " << e.what() << "\n";
}
catch (...)
{
cdbg << "Executor: exception ignored\n";
}
}
}
void Fuzzer::ShuffleAndMinimize() {
PrintStats("READ ");
std::vector<Unit> NewCorpus;
if (Options.ShuffleAtStartUp)
ShuffleCorpus(&Corpus);
for (const auto &U : Corpus) {
if (RunOne(U)) {
NewCorpus.push_back(U);
if (Options.Verbosity >= 2)
Printf("NEW0: %zd L %zd\n", MaxCoverage.BlockCoverage, U.size());
}
}
Corpus = NewCorpus;
UpdateCorpusDistribution();
for (auto &X : Corpus)
UnitHashesAddedToCorpus.insert(Hash(X));
PrintStats("INITED");
CheckForMemoryLeaks();
}
void Fuzzer::RereadOutputCorpus() {
if (Options.OutputCorpus.empty()) return;
std::vector<Unit> AdditionalCorpus;
ReadDirToVectorOfUnits(Options.OutputCorpus.c_str(), &AdditionalCorpus,
&EpochOfLastReadOfOutputCorpus);
if (Corpus.empty()) {
Corpus = AdditionalCorpus;
return;
}
if (!Options.Reload) return;
for (auto &X : AdditionalCorpus) {
if (X.size() > (size_t)Options.MaxLen)
X.resize(Options.MaxLen);
if (UnitsAddedAfterInitialLoad.insert(X).second) {
Corpus.push_back(X);
CurrentUnit.clear();
CurrentUnit.insert(CurrentUnit.begin(), X.begin(), X.end());
size_t NewCoverage = RunOne(CurrentUnit);
if (NewCoverage && Options.Verbosity >= 1)
PrintStats("RELOAD", NewCoverage);
}
}
}
void CG3DApp::Run()
{
Init();
while (RunOne());
Shutdown();
}
int Maker::RunOne(Depends *depend, Environment &env, bool keepGoing)
{
int rv = 0;
if (depend->GetRuleList()->IsBuilt())
return 0;
RuleList *rl = depend->GetRuleList();
Eval::PushruleStack(rl);
bool stop = false;
bool cantbuild = false;
for (Depends::iterator it = depend->begin(); it != depend->end(); ++it)
{
// if (!(*it)->GetOrdered())
{
if (rv = RunOne(*it, env, keepGoing))
{
stop = true;
if (!keepGoing)
{
Eval::PopruleStack();
return rv;
}
}
}
// else
// cantbuild |= !(*it)->GetRuleList()->IsBuilt();
}
if (stop)
{
Eval::PopruleStack();
return -1;
}
if (cantbuild || rl->IsBuilt())
{
Eval::PopruleStack();
return 0;
}
rl->SetBuilt();
if (touch)
{
rl->Touch(OS::GetCurrentTime());
}
bool sil = silent;
bool ig = ignoreResults;
if (!sil)
sil = OnList(depend->GetGoal(), ".SILENT");
if (!ig)
ig = OnList(depend->GetGoal(), ".IGNORE");
Spawner sp(env, ig, sil, displayOnly);
if (depend->GetRule() && depend->GetRule()->GetCommands())
rv = sp.Run(*depend->GetRule()->GetCommands(), rl, NULL);
if (rv)
{
std::strstream a;
a << rv;
std::string b;
a >> b;
if (RuleContainer::Instance()->Lookup(".DELETE_ON_ERROR"))
{
OS::RemoveFile(depend->GetGoal());
std::cout << std::endl;
Eval::error("commands returned error code " + b + " '" + depend->GetGoal() + "' removed");
}
else
{
std::cout << std::endl;
Eval::error("commands returned error code " + b);
}
}
Eval::PopruleStack();
return rv;
}
void Fuzzer::RunOneAndUpdateCorpus(const Unit &U) {
if (TotalNumberOfRuns >= Options.MaxNumberOfRuns)
return;
ReportNewCoverage(RunOne(U), U);
}