Beispiel #1
0
    void operator()() {
        if (work.empty())
            return;
        int specimen_id = work.front().specimen_id;

        // Database connections don't survive over fork() according to SqLite and PostgreSQL documentation, so open it again
        SqlDatabase::TransactionPtr tx = SqlDatabase::Connection::create(databaseUrl)->transaction();

        OutputGroups ogroups; // do not load from database (that might take a very long time)

        if (opt.verbosity>=LACONIC) {
            if (opt.verbosity>=EFFUSIVE)
                std::cerr <<argv0 <<": " <<std::string(100, '#') <<"\n";
            std::cerr <<argv0 <<": processing binary specimen \"" <<files.name(specimen_id) <<"\"\n";
        }

        // Parse the specimen
        SgProject *project = files.load_ast(tx, specimen_id);
        if (!project)
            project = open_specimen(tx, files, specimen_id, argv0);
        if (!project) {
            std::cerr <<argv0 <<": problems loading specimen\n";
            exit(1);
        }

        // Get list of specimen functions and initialize the instruction cache
        std::vector<SgAsmFunction*> all_functions = SageInterface::querySubTree<SgAsmFunction>(project);
        IdFunctionMap functions = existing_functions(tx, files, all_functions);
        FunctionIdMap function_ids;
        AddressIdMap entry2id;                              // maps function entry address to function ID
        for (IdFunctionMap::iterator fi=functions.begin(); fi!=functions.end(); ++fi) {
            function_ids[fi->second] = fi->first;
            entry2id[fi->second->get_entry_va()] = fi->first;
        }
        InstructionProvidor insns = InstructionProvidor(all_functions);

        // Split the work list into chunks, each containing testsPerChunk except the last, which may contain fewer.
        static const size_t testsPerChunk = 25;
        size_t nChunks = (work.size() + testsPerChunk - 1) / testsPerChunk;
        std::vector<SomeTests> jobs;
        for (size_t i=0; i<nChunks; ++i) {
            size_t beginWorkIdx = i * testsPerChunk;
            size_t endWorkIdx = std::min((i+1)*testsPerChunk, work.size());
            Work partWork(work.begin()+beginWorkIdx, work.begin()+endWorkIdx);
            jobs.push_back(SomeTests(partWork, databaseUrl, functions, function_ids, &insns, cmd_id, &entry2id));
        }

        // Run the parts in parallel using the maximum parallelism specified on the command-line.  We must commit our
        // transaction before forking, otherwise the children won't see the rows we've added to various tables.
        tx->commit();
        tx.reset();
        size_t nfailed = runInParallel(jobs, opt.nprocs);
        if (nfailed!=0) {
            std::cerr <<"SpecimenProcessor: " <<StringUtility::plural(nfailed, "jobs") <<" failed\n";
            exit(1);
        }
    }
Beispiel #2
0
int
main(int argc, char *argv[])
{
    // Parse command-line
    opt.nprocs = nProcessors();
    int argno = parse_commandline(argc, argv);
    if (argno+1!=argc)
        usage(1);
    std::string databaseUrl = argv[argno++];

    SqlDatabase::TransactionPtr tx = SqlDatabase::Connection::create(databaseUrl)->transaction();
    int64_t cmd_id = start_command(tx, argc, argv, "running tests");

    // Load worklist
    MultiWork work;
    load_sorted_work(work/*out*/);
    if (work.empty())
        return 0;


    // Load information about files.  The transaction is not saved anywhere.
    FilesTable files(tx);

    // We must commit our transaction before we fork, otherwise the child processes won't be able to see the rows we've
    // inserted. Specifically, the row in the semantic_history table that says who we are.  Destroy the smart pointer so that
    // the connection is even closed.
    tx->commit();
    tx.reset();

    // Process work items for each specimen sequentially
    BOOST_FOREACH (const Work &workForSpecimen, work)
    if (forkAndWait(SpecimenProcessor(workForSpecimen, files, databaseUrl, cmd_id)))
        exit(1);

    // Indicate that this command is finished
    tx = SqlDatabase::Connection::create(databaseUrl)->transaction();
    finish_command(tx, cmd_id, "ran tests");
    tx->commit();

    return 0;
}
Beispiel #3
0
    void operator()() {
        // Database connections don't survive over fork() according to SqLite and PostgreSQL documentation, so open it again
        SqlDatabase::TransactionPtr tx = SqlDatabase::Connection::create(databaseUrl)->transaction();

        // Use zero for the number of tests ran so that this child process doesn't try to update the semantic_history table.
        // If two or more processes try to change the same row (which they will if there's a non-zero number of tests) then
        // they will deadlock with each other.
        static const size_t NO_TESTS_RAN = 0;

        NameSet builtin_function_names;
        add_builtin_functions(builtin_function_names/*out*/);

        InputGroup igroup;
        WorkItem prevWorkItem;
        SgAsmInterpretation *prev_interp = NULL;
        MemoryMap ro_map;
        Disassembler::AddressSet whitelist_exports;         // dynamic functions that should be called
        PointerDetectors pointers;
        InsnCoverage insn_coverage;
        DynamicCallGraph dynamic_cg;
        Tracer tracer;
        ConsumedInputs consumed_inputs;
        FuncAnalyses funcinfo;
        OutputGroups ogroups; // do not load from database (that might take a very long time)
        time_t last_checkpoint = time(NULL);
        for (size_t workIdx=0; workIdx<work.size(); ++workIdx) {
            WorkItem &workItem = work[workIdx];

            // Load the input group from the database if necessary.
            if (workItem.igroup_id!=prevWorkItem.igroup_id) {
                if (!igroup.load(tx, workItem.igroup_id)) {
                    std::cerr <<argv0 <<": input group " <<workItem.igroup_id <<" is empty or does not exist\n";
                    exit(1);
                }
            }

            // Find the function to test
            IdFunctionMap::iterator func_found = functions.find(workItem.func_id);
            assert(func_found!=functions.end());
            SgAsmFunction *func = func_found->second;
            if (opt.verbosity>=LACONIC) {
                if (opt.verbosity>=EFFUSIVE)
                    std::cerr <<argv0 <<": " <<std::string(100, '=') <<"\n";
                std::cerr <<argv0 <<": processing function " <<function_to_str(func, function_ids) <<"\n";
            }
            SgAsmInterpretation *interp = SageInterface::getEnclosingNode<SgAsmInterpretation>(func);
            assert(interp!=NULL);

            // Do per-interpretation stuff
            if (interp!=prev_interp) {
                prev_interp = interp;
                assert(interp->get_map()!=NULL);
                ro_map = *interp->get_map();
                ro_map.require(MemoryMap::READABLE).prohibit(MemoryMap::WRITABLE).keep();
                Disassembler::AddressSet whitelist_imports = get_import_addresses(interp, builtin_function_names);
                whitelist_exports.clear(); // imports are addresses of import table slots; exports are functions
                overmap_dynlink_addresses(interp, *insns, opt.params.follow_calls, &ro_map, GOTPLT_VALUE,
                                          whitelist_imports, whitelist_exports/*out*/);
                if (opt.verbosity>=EFFUSIVE) {
                    std::cerr <<argv0 <<": memory map for SgAsmInterpretation:\n";
                    interp->get_map()->dump(std::cerr, argv0+":   ");
                }
            }

            // Run the test
            assert(insns!=NULL);
            assert(entry2id!=NULL);
            std::cerr <<"process " <<getpid() <<" about to run test " <<workIdx <<"/" <<work.size() <<" " <<workItem <<"\n";
            runOneTest(tx, workItem, pointers, func, function_ids, insn_coverage, dynamic_cg, tracer, consumed_inputs,
                       interp, whitelist_exports, cmd_id, igroup, funcinfo, *insns, &ro_map, *entry2id, ogroups);
            ++ntests_ran;

            // Checkpoint
            if (opt.checkpoint>0 && time(NULL)-last_checkpoint > opt.checkpoint) {
                if (!opt.dry_run)
                    tx = checkpoint(tx, ogroups, tracer, insn_coverage, dynamic_cg, consumed_inputs, NULL, NO_TESTS_RAN,
                                    cmd_id);
                last_checkpoint = time(NULL);
            }

            prevWorkItem = workItem;
        }
        std::cerr <<"process " <<getpid() <<" is done testing; now finishing up...\n";

        if (!tx->is_terminated()) {
            SqlDatabase::StatementPtr stmt = tx->statement("insert into semantic_funcpartials"
                                             " (func_id, ncalls, nretused, ntests, nvoids) values"
                                             " (?,       ?,      ?,        ?,      ?)");
            for (FuncAnalyses::iterator fi=funcinfo.begin(); fi!=funcinfo.end(); ++fi) {
                stmt->bind(0, fi->first);
                stmt->bind(1, fi->second.ncalls);
                stmt->bind(2, fi->second.nretused);
                stmt->bind(3, fi->second.ntests);
                stmt->bind(4, fi->second.nvoids);
                stmt->execute();
            }
        }

        // Cleanup
        if (!tx->is_terminated() && !opt.dry_run) {
            std::cerr <<"process " <<getpid() <<" is doing the final checkpoint\n";
            checkpoint(tx, ogroups, tracer, insn_coverage, dynamic_cg, consumed_inputs, NULL, NO_TESTS_RAN, cmd_id);
        }
        tx.reset();

        std::cerr <<"process " <<getpid() <<" finished\n";
    }