/** Returns a list of parts of a single section that have been referenced. The offsets are relative to the start of the * section. */ ExtentMap SgAsmGenericSection::get_referenced_extents() const { ExtentMap retval; if (0==get_size()) return retval; Extent s(get_offset(), get_size()); const ExtentMap &file_extents = get_file()->get_referenced_extents(); for (ExtentMap::const_iterator i=file_extents.begin(); i!=file_extents.end(); i++) { Extent e = i->first; if (e.contained_in(s)) { retval.insert(Extent(e.first()-get_offset(), e.size())); } else if (e.left_of(s) || e.right_of(s)) { /*void*/ } else if (e.contains(s)) { retval.insert(Extent(0, get_size())); } else if (e.begins_before(s)) { retval.insert(Extent(0, e.first()+e.size()-get_offset())); } else if (e.ends_after(s)) { retval.insert(Extent(e.first()-get_offset(), get_offset()+get_size()-e.first())); } else { assert(!"invalid extent overlap category"); abort(); } } return retval; }
int main(int argc, char **argv) { ExtentMap em; string prefix; if (argc > 2) usage(argv[0]); else if (argc == 2) prefix = argv[1]; else prefix = "BRM_state"; idbdatafile::IDBPolicy::configIDBPolicy(); try { em.load(prefix); cout << "OK." << endl; } catch (exception &e) { cout << "Load failed." << endl; return 1; } return 0; }
/** Write holes (unreferenced areas) back to the file */ void SgAsmGenericSection::unparse_holes(std::ostream &f) const { #if 0 /*DEBUGGING*/ ExtentMap holes = get_unreferenced_extents(); fprintf(stderr, "Section \"%s\", 0x%"PRIx64" bytes\n", get_name()->get_string(true).c_str(), get_size()); holes.dump_extents(stderr, " ", ""); #endif // unparse(f, get_unreferenced_extents()); }
int main(int argc, char** argv) { int c; string pname(argv[0]); opterr = 0; while ((c = getopt(argc, argv, "vdh")) != EOF) switch (c) { case 'v': vflg++; break; case 'd': dflg = true; break; case 'h': case '?': default: usage(pname); return (c == 'h' ? 0 : 1); break; } const Config* cf = Config::makeConfig(); DBRoot = cf->getConfig("SystemConfig", "DBRoot"); pattern = DBRoot + "/[0-9][0-9][0-9].dir/[0-9][0-9][0-9].dir/[0-9][0-9][0-9].dir/FILE[0-9][0-9][0-9].cdf"; if (vflg) { cout << "Using DBRoot " << DBRoot << endl; } if (access(DBRoot.c_str(), X_OK) != 0) { cerr << "Could not scan DBRoot " << DBRoot << '!' << endl; return 1; } ExtentMap em; extentSize = em.getExtentSize(); if (vflg) { cout << "System extent size is " << extentSize << " blocks" << endl; } if (nftw(DBRoot.c_str(), walkDB, 64, FTW_PHYS|FTW_ACTIONRETVAL) != 0) { cerr << "Error processing files in DBRoot " << DBRoot << '!' << endl; return 1; } return 0; }
/** Precipitates individual extents into larger extents by combining individual extents that are separated by an amount less * than or equal to some specified @p reagent value. Individual elements that would have been adjacent have already * been combined by the other modifying methods (insert, erase, etc). */ void ExtentMap::precipitate(rose_addr_t reagent) { abort(); // NOT IMPLEMENTED ExtentMap result; for (iterator i=begin(); i!=end(); /*void*/) { ExtentPair left = *i++; for (/*void*/; i!=end() && left.first+left.second+reagent >= i->first; i++) left.second = (i->first + i->second) - left.first; result.insert(left); } *this = result; }
/** Write just the specified regions back to the file */ void SgAsmGenericSection::unparse(std::ostream &f, const ExtentMap &map) const { for (ExtentMap::const_iterator i=map.begin(); i!=map.end(); ++i) { Extent e = i->first; assert(e.first()+e.size() <= get_size()); const unsigned char *extent_data; if (e.first() >= p_data.size()) { extent_data = NULL; } else if (e.first() + e.size() > p_data.size()) { extent_data = &p_data[e.first()]; } else { extent_data = &p_data[e.first()]; } if (extent_data) write(f, e.first(), e.size(), extent_data); } }
void addVectorToDatabase(const SqlDatabase::TransactionPtr &tx, const SignatureVector& vec, const std::string& functionName, size_t functionId, size_t indexWithinFunction, const std::string& normalizedUnparsedInstructions, SgAsmx86Instruction* firstInsn[], const std::string& filename, size_t windowSize, size_t stride) { ++numVectorsGenerated; vector<uint8_t> compressedCounts = compressVector(vec.getBase(), SignatureVector::Size); size_t vectorSum = 0; for (size_t i=0; i<SignatureVector::Size; ++i) vectorSum += vec[i]; ExtentMap extent; for (size_t i=0; i<windowSize; ++i) extent.insert(Extent(firstInsn[i]->get_address(), firstInsn[i]->get_size())); unsigned char md[16]; MD5((const unsigned char*)normalizedUnparsedInstructions.data(), normalizedUnparsedInstructions.size(), md); SqlDatabase::StatementPtr cmd = tx->statement("insert into vectors" // 0 1 2 3 4 5 " (id, function_id, index_within_function, line, last_insn_va, size," // 6 7 8 "sum_of_counts, counts_b64, instr_seq_b64)" " values (?,?,?,?,?,?,?,?,?)"); int vector_id = tx->statement("select coalesce(max(id),0)+1 from vectors")->execute_int(); // 1-origin cmd->bind(0, vector_id); cmd->bind(1, functionId); cmd->bind(2, indexWithinFunction); cmd->bind(3, firstInsn[0]->get_address()); cmd->bind(4, firstInsn[windowSize-1]->get_address()); cmd->bind(5, extent.size()); cmd->bind(6, vectorSum); cmd->bind(7, StringUtility::encode_base64(&compressedCounts[0], compressedCounts.size())); cmd->bind(8, StringUtility::encode_base64(md, 16)); cmd->execute(); }
/** Obtains the virtual address of the Hint/Name Table. The Hint/Name Table is an implicit table--the PE file format * specification talks about such a table, but it is not actually defined anywhere in the PE file. Instead, various Import * Lookup Table and Import Address Table entries might point to individual Hint/Name pairs, which collectively form an * implicit Hint/Name Table. There is no requirement that the Hint/Name pairs are contiguous in the address space, and indeed * they often are not. Therefore, the only way to describe the location of the Hint/Name Table is by a list of addresses. * * This function will scan this Import Directory's import items, observe which items make references to Hint/Name pairs that * have known addresses, and add those areas of virtual memory to the specified extent map. This function returns the number * of ILT entries that reference a Hint/Name pair. */ size_t SgAsmPEImportDirectory::hintname_table_extent(ExtentMap &extent/*in,out*/) const { size_t retval = 0; const SgAsmPEImportItemPtrList &imports = get_imports()->get_vector(); for (SgAsmPEImportItemPtrList::const_iterator ii=imports.begin(); ii!=imports.end(); ++ii) { SgAsmPEImportItem *import = *ii; if (!import->get_by_ordinal() && import->get_hintname_rva().get_rva()!=0 && import->get_hintname_nalloc()>0) { size_t nbytes = std::min(import->get_hintname_nalloc(), import->hintname_required_size()); extent.insert(Extent(import->get_hintname_rva().get_va(), nbytes)); ++retval; } } return retval; }
AddressIntervalSet toAddressIntervalSet(const ExtentMap &x) { AddressIntervalSet retval; for (ExtentMap::const_iterator iter=x.begin(); iter!=x.end(); ++iter) retval.insert(toAddressInterval(iter->first)); return retval; }
ExtentMap toExtentMap(const AddressIntervalSet &x) { ExtentMap retval; BOOST_FOREACH (const AddressInterval &interval, x.intervals()) retval.insert(toExtent(interval)); return retval; }
void extentMap_good_1() { ExtentMap em; int i, err, oid, iterations = 1300; // (EM_INITIAL_SIZE + 3*EM_INCREMENT) int caughtException = 0, allocdSize; uint32_t fbo, hwm; BRM::HWM_t hwm2; BRM::VER_t txnID; vector<LBID_t> lbids; const uint32_t extentSize = em.getExtentSize(); em.load(string("EMImage")); em.checkConsistency(); for (i = 0; i < iterations; i++) { err = em.lookup(static_cast<LBID_t>(i * extentSize), oid, fbo); CPPUNIT_ASSERT(err == 0); CPPUNIT_ASSERT(oid == i); CPPUNIT_ASSERT(fbo == 0); if (i != 0) { err = em.lookup(static_cast<LBID_t>(i * extentSize - 1), oid, fbo); CPPUNIT_ASSERT(err == 0); CPPUNIT_ASSERT(oid == i - 1); CPPUNIT_ASSERT(fbo == extentSize - 1); } if (i != iterations - 1) { err = em.lookup(static_cast<LBID_t>(i * extentSize + 1), oid, fbo); CPPUNIT_ASSERT(err == 0); CPPUNIT_ASSERT(oid == i); CPPUNIT_ASSERT(fbo == 1); } } em.checkConsistency(); err = em.lookup(static_cast<LBID_t>(i * extentSize), oid, fbo); CPPUNIT_ASSERT(err == -1); for (i = 0; i < iterations; i++) { err = em.getBulkInsertVars(static_cast<LBID_t>(i * extentSize), hwm2, txnID); CPPUNIT_ASSERT(err == 0); CPPUNIT_ASSERT(hwm2 == 0); CPPUNIT_ASSERT(txnID == 0); err = em.setBulkInsertVars(static_cast<LBID_t>(i * extentSize), i, i + 1); em.confirmChanges(); CPPUNIT_ASSERT(err == 0); err = em.getBulkInsertVars(static_cast<LBID_t>(i * extentSize), hwm2, txnID); CPPUNIT_ASSERT(err == 0); CPPUNIT_ASSERT(hwm2 == static_cast<LBID_t>(i)); CPPUNIT_ASSERT(txnID == static_cast<VER_t>(i + 1)); hwm = em.getHWM(i); CPPUNIT_ASSERT(hwm == 0); em.setHWM(i, (i > (extentSize - 1) ? extentSize - 1 : i)); em.confirmChanges(); hwm = em.getHWM(i); CPPUNIT_ASSERT(hwm == static_cast<uint32_t>(i > extentSize - 1 ? extentSize - 1 : i)); } em.checkConsistency(); #ifdef BRM_DEBUG caughtException = 0; try { em.setHWM(i, hwm); } catch (std::invalid_argument e) { caughtException = 1; } em.undoChanges(); CPPUNIT_ASSERT(caughtException == 1); #endif for (i = 0; i < iterations; i++) { em.deleteOID(i); em.confirmChanges(); } #ifdef BRM_DEBUG caughtException = 0; try { em.deleteOID(i); } catch (std::invalid_argument& e) { caughtException = 1; } em.undoChanges(); CPPUNIT_ASSERT(caughtException == 1); #endif em.checkConsistency(); }
rose_addr_t DwarfLineMapper::src2first_addr(const SrcInfo &srcinfo) const { ExtentMap ex = src2addr(srcinfo); return ex.empty() ? 0 : ex.min(); }
Partitioner::RegionStats * Partitioner::region_statistics(const ExtentMap &addresses) { RegionStats *stats = new_region_stats(); assert(stats!=NULL); size_t nbytes = addresses.size(); if (0==nbytes) return stats; stats->add_sample(RegionStats::RA_NBYTES, nbytes); ExtentMap not_addresses = addresses.invert<ExtentMap>(); Disassembler::AddressSet worklist; // addresses waiting to be disassembled recursively InstructionMap insns_found; // all the instructions we found herein ExtentMap insns_extent; // memory used by the instructions we've found ExtentMap pending = addresses; // addresses we haven't looked at yet /* Undirected local control flow graph used to count connected components */ typedef boost::adjacency_list<boost::vecS, boost::vecS, boost::undirectedS> CFG; typedef boost::graph_traits<CFG>::vertex_descriptor CFGVertex; typedef std::map<rose_addr_t, CFGVertex> Addr2Vertex; CFG cfg; Addr2Vertex va2id; /* Statistics */ size_t nstarts=0; // number of times the recursive disassembler was started size_t nfails=0; // number of disassembler failures size_t noverlaps=0; // instructions overlapping with a previously found instruction size_t nincomplete=0; // number of instructions with unknown successors size_t nfallthrough=0; // number of branches to fall-through address within our "addresses" size_t ncalls=0; // number of function calls outside our "addresses" size_t nnoncalls=0; // number of branches to non-functions outside our "addresses" size_t ninternal=0; // number of non-fallthrough internal branches while (!pending.empty()) { rose_addr_t start_va = pending.min(); worklist.insert(start_va); ++nstarts; while (!worklist.empty()) { rose_addr_t va = *worklist.begin(); worklist.erase(worklist.begin()); /* Obtain (disassemble) the instruction and make sure it falls entirely within the "addresses" */ Instruction *insn = find_instruction(va); if (!insn) { ++nfails; pending.erase(Extent(va)); continue; } Extent ie(va, insn->get_size()); if (not_addresses.overlaps(ie)) { ++nfails; pending.erase(Extent(va)); continue; } /* The disassembler can also return an "unknown" instruction when failing, depending on how it is invoked. */ if (insn->node->is_unknown()) { ++nfails; pending.erase(Extent(va, insn->get_size())); continue; } insns_found.insert(std::make_pair(va, insn)); rose_addr_t fall_through_va = va + insn->get_size(); /* Does this instruction overlap with any we've already found? */ if (insns_extent.overlaps(ie)) ++noverlaps; pending.erase(Extent(va, insn->get_size())); insns_extent.insert(ie); /* Find instruction successors by looking only at the instruction itself. This is simpler, but less rigorous * method than finding successors a basic block at a time. For instance, we'll find both sides of a branch * instruction even if the more rigorous method determined that one side or the other is always taken. But this is * probably what we want here anyway for determining whether something looks like code. */ bool complete; Disassembler::AddressSet succs = insn->get_successors(&complete); if (!complete) ++nincomplete; /* Add instruction as vertex to CFG */ std::pair<Addr2Vertex::iterator, bool> inserted = va2id.insert(std::make_pair(va, va2id.size())); if (inserted.second) { CFGVertex vertex __attribute__((unused)) = add_vertex(cfg); assert(vertex==inserted.first->second); } /* Classify the various successors. */ for (Disassembler::AddressSet::const_iterator si=succs.begin(); si!=succs.end(); ++si) { rose_addr_t succ_va = *si; if (succ_va==fall_through_va) { ++nfallthrough; if (pending.find(succ_va)!=pending.end()) worklist.insert(succ_va); /* Add edge to CFG graph */ va2id.insert(std::make_pair(succ_va, va2id.size())); add_edge(va2id[va], va2id[succ_va], cfg); } else if (addresses.find(succ_va)==addresses.end()) { /* A non-fallthrough branch to something outside this memory region */ if (functions.find(succ_va)!=functions.end()) { /* A branch to a function entry point we've previously discovered. */ ++ncalls; } else { ++nnoncalls; } } else { /* A non-fallthrough branch to something in our address range. */ ++ninternal; if (pending.find(succ_va)!=pending.end()) worklist.insert(succ_va); /* Add edge to CFG graph */ va2id.insert(std::make_pair(succ_va, va2id.size())); add_edge(va2id[va], va2id[succ_va], cfg); } } } } /* Statistics */ stats->add_sample(RegionStats::RA_NFAILS, nfails); stats->add_sample(RegionStats::RA_NINSNS, insns_found.size()); stats->add_sample(RegionStats::RA_NOVERLAPS, noverlaps); stats->add_sample(RegionStats::RA_NSTARTS, nstarts); stats->add_sample(RegionStats::RA_NCOVERAGE, insns_extent.size()); stats->add_sample(RegionStats::RA_NINCOMPLETE, nincomplete); stats->add_sample(RegionStats::RA_NBRANCHES, ncalls+nnoncalls+ninternal); stats->add_sample(RegionStats::RA_NCALLS, ncalls); stats->add_sample(RegionStats::RA_NNONCALLS, nnoncalls); stats->add_sample(RegionStats::RA_NINTERNAL, ninternal); stats->add_sample(RegionStats::RA_NICFGEDGES, ninternal + nfallthrough); stats->add_sample(RegionStats::RA_NIUNIQUE, count_kinds(insns_found)); stats->add_sample(RegionStats::RA_NPRIV, count_privileged(insns_found)); stats->add_sample(RegionStats::RA_NFLOAT, count_floating_point(insns_found)); double regsz, regvar; stats->add_sample(RegionStats::RA_NREGREFS, count_registers(insns_found, ®sz, ®var)); stats->add_sample(RegionStats::RA_REGSZ, regsz); stats->add_sample(RegionStats::RA_REGVAR, regvar); /* Count the number of connected components in the undirected CFG */ if (!va2id.empty()) { std::vector<int> component(num_vertices(cfg)); stats->add_sample(RegionStats::RA_NCOMPS, connected_components(cfg, &component[0])); } stats->compute_ratios(); return stats; }
void extentMap_freelist() { ExtentMap em; int i, allocdSize, iterations = 1400; // (EM_INITIAL_SIZE + 4*EM_INCREMENT) vector<LBID_t> lbids; const int extentSize = em.getExtentSize(); for (i = 0; i < iterations; i++) { em.createExtent(extentSize, i, lbids, allocdSize); em.confirmChanges(); CPPUNIT_ASSERT(lbids.back() == static_cast<LBID_t>(i * extentSize)); } em.checkConsistency(); //frag the lbid space to blow up the free list for (i = 0; i < iterations; i += 2) { em.deleteOID(i); em.confirmChanges(); } em.checkConsistency(); //fill in the holes for (i = 0; i < iterations; i += 2) { em.createExtent(extentSize, i, lbids, allocdSize); em.confirmChanges(); } for (i = 0; i < iterations; i += 2) { em.deleteOID(i); em.confirmChanges(); } for (i = 1; i < iterations; i += 2) { em.deleteOID(i); em.confirmChanges(); } em.checkConsistency(); }
void SgAsmGenericFile::shift_extend(SgAsmGenericSection *s, rose_addr_t sa, rose_addr_t sn, AddressSpace space, Elasticity elasticity) { ROSE_ASSERT(s!=NULL); ROSE_ASSERT(s->get_file()==this); ROSE_ASSERT((space & (ADDRSP_FILE|ADDRSP_MEMORY)) != 0); const bool debug = false; static size_t ncalls=0; char p[256]; if (debug) { const char *space_s="unknown"; if (space & ADDRSP_FILE) { space_s = "file"; } else if (space & ADDRSP_MEMORY) { space_s = "memory"; } sprintf(p, "SgAsmGenericFile::shift_extend[%" PRIuPTR "]: ", ncalls++); fprintf(stderr, "%s -- START --\n", p); fprintf(stderr, "%s S = [%d] \"%s\"\n", p, s->get_id(), s->get_name()->get_string(true).c_str()); fprintf(stderr, "%s %s Sa=0x%08" PRIx64 " (%" PRIu64 "), Sn=0x%08" PRIx64 " (%" PRIu64 ")\n", p, space_s, sa, sa, sn, sn); fprintf(stderr, "%s elasticity = %s\n", p, (ELASTIC_NONE==elasticity ? "none" : ELASTIC_UNREF==elasticity ? "unref" : ELASTIC_HOLE==elasticity ? "unref+holes" : "unknown")); } /* No-op case */ if (0==sa && 0==sn) { if (debug) { fprintf(stderr, "%s No change necessary.\n", p); fprintf(stderr, "%s -- END --\n", p); } return; } bool filespace = (space & ADDRSP_FILE)!=0; bool memspace = (space & ADDRSP_MEMORY)!=0; rose_addr_t align=1, aligned_sa, aligned_sasn; SgAsmGenericSectionPtrList neighbors, villagers; ExtentMap amap; /* address mappings for all extents */ Extent sp; /* Get a list of all sections that may need to be adjusted. */ SgAsmGenericSectionPtrList all; switch (elasticity) { case ELASTIC_NONE: case ELASTIC_UNREF: all = filespace ? get_sections() : get_mapped_sections(); break; case ELASTIC_HOLE: all = filespace ? get_sections(false) : get_mapped_sections(); break; } if (debug) { fprintf(stderr, "%s Following sections are in 'all' set:\n", p); for (size_t i=0; i<all.size(); i++) { Extent ep; if (filespace) { ep = all[i]->get_file_extent(); } else { ROSE_ASSERT(all[i]->is_mapped()); ep = all[i]->get_mapped_preferred_extent(); } fprintf(stderr, "%s 0x%08" PRIx64 " 0x%08" PRIx64 " 0x%08" PRIx64 " [%d] \"%s\"\n", p, ep.relaxed_first(), ep.size(), ep.relaxed_first()+ep.size(), all[i]->get_id(), all[i]->get_name()->get_string(true).c_str()); } } for (size_t pass=0; pass<2; pass++) { if (debug) { fprintf(stderr, "%s -- %s --\n", p, 0==pass?"FIRST PASS":"******"); } /* S offset and size in file or memory address space */ if (filespace) { sp = s->get_file_extent(); } else if (!memspace || !s->is_mapped()) { return; /*nothing to do*/ } else { sp = s->get_mapped_preferred_extent(); } /* Build address map */ for (size_t i=0; i<all.size(); i++) { if (filespace) { amap.insert(all[i]->get_file_extent()); } else { ROSE_ASSERT(all[i]->is_mapped()); amap.insert(all[i]->get_mapped_preferred_extent()); } } if (debug) { fprintf(stderr, "%s Address map:\n", p); amap.dump_extents(stderr, (std::string(p)+" ").c_str(), "amap"); fprintf(stderr, "%s Extent of S:\n", p); fprintf(stderr, "%s start=0x%08" PRIx64 " size=0x%08" PRIx64 " end=0x%08" PRIx64 "\n", p, sp.relaxed_first(), sp.size(), sp.relaxed_first()+sp.size()); } /* Neighborhood (nhs) of S is a single extent. However, if S is zero size then nhs might be empty. The neighborhood of * S is S plus all sections that overlap with S and all sections that are right-contiguous with S. */ ExtentMap nhs_map; for (ExtentMap::iterator amapi=amap.begin(); amapi!=amap.end(); ++amapi) { if (amapi->first.relaxed_first() <= sp.relaxed_first()+sp.size() && amapi->first.relaxed_first()+amapi->first.size() > sp.relaxed_first()) nhs_map.insert(amapi->first, amapi->second); } if (debug) { fprintf(stderr, "%s Neighborhood of S:\n", p); nhs_map.dump_extents(stderr, (std::string(p)+" ").c_str(), "nhs_map"); } Extent nhs; if (nhs_map.size()>0) { assert(nhs_map.nranges()==1); nhs = nhs_map.begin()->first; } else { nhs = sp; } /* What sections are in the neighborhood (including S), and right of the neighborhood? */ neighbors.clear(); /*sections in neighborhood*/ neighbors.push_back(s); villagers.clear(); /*sections right of neighborhood*/ if (debug) fprintf(stderr, "%s Ignoring left (L) sections:\n", p); for (size_t i=0; i<all.size(); i++) { SgAsmGenericSection *a = all[i]; if (a==s) continue; /*already pushed onto neighbors*/ Extent ap; if (filespace) { ap = a->get_file_extent(); } else if (!a->is_mapped()) { continue; } else { ap = a->get_mapped_preferred_extent(); } switch (ExtentMap::category(ap, nhs)) { case 'L': if (debug) fprintf(stderr, "%s L 0x%08" PRIx64 " 0x%08" PRIx64 " 0x%08" PRIx64 " [%d] \"%s\"\n", p, ap.relaxed_first(), ap.size(), ap.relaxed_first()+ap.size(), a->get_id(), a->get_name()->get_string(true).c_str()); break; case 'R': if (ap.relaxed_first()==nhs.relaxed_first()+nhs.size() && 0==ap.size()) { /* Empty sections immediately right of the neighborhood of S should actually be considered part of the * neighborhood rather than right of it. */ neighbors.push_back(a); } else if (elasticity!=ELASTIC_NONE) { /* If holes are elastic then treat things right of the hole as being part of the right village; otherwise * add those sections to the neighborhood of S even though they fall outside 'nhs' (it's OK because this * partitioning of sections is the only thing we use 'nhs' for anyway. */ villagers.push_back(a); } else if ('L'==ExtentMap::category(ap, sp)) { /*ignore sections left of S*/ } else { neighbors.push_back(a); } break; default: if ('L'!=ExtentMap::category(ap, sp)) /*ignore sections left of S*/ neighbors.push_back(a); break; } } if (debug) { fprintf(stderr, "%s Neighbors:\n", p); for (size_t i=0; i<neighbors.size(); i++) { SgAsmGenericSection *a = neighbors[i]; Extent ap = filespace ? a->get_file_extent() : a->get_mapped_preferred_extent(); rose_addr_t align = filespace ? a->get_file_alignment() : a->get_mapped_alignment(); char cat = ExtentMap::category(ap, sp); fprintf(stderr, "%s %c %c0x%08" PRIx64 " 0x%08" PRIx64 " 0x%08" PRIx64, p, cat, 0==ap.relaxed_first() % (align?align:1) ? ' ' : '!', ap.relaxed_first(), ap.size(), ap.relaxed_first()+ap.size()); if (strchr("RICE", cat)) { fprintf(stderr, " align=0x%08" PRIx64, align); } else { fputs(" ", stderr); } fprintf(stderr, " [%2d] \"%s\"\n", a->get_id(), a->get_name()->get_string(true).c_str()); } if (villagers.size()>0) fprintf(stderr, "%s Villagers:\n", p); for (size_t i=0; i<villagers.size(); i++) { SgAsmGenericSection *a = villagers[i]; Extent ap = filespace ? a->get_file_extent() : a->get_mapped_preferred_extent(); rose_addr_t align = filespace ? a->get_file_alignment() : a->get_mapped_alignment(); fprintf(stderr, "%s %c %c0x%08" PRIx64 " 0x%08" PRIx64 " 0x%08" PRIx64, p, ExtentMap::category(ap, sp), /*cat should always be R*/ 0==ap.relaxed_first() % (align?align:1) ? ' ' : '!', ap.relaxed_first(), ap.size(), ap.relaxed_first()+ap.size()); fputs(" ", stderr); fprintf(stderr, " [%2d] \"%s\"\n", a->get_id(), a->get_name()->get_string(true).c_str()); } } /* Adjust Sa to satisfy all alignment constraints in neighborhood(S) for sections that will move (cats R, I, C, and E). */ align = 1; for (size_t i=0; i<neighbors.size(); i++) { SgAsmGenericSection *a = neighbors[i]; Extent ap = filespace ? a->get_file_extent() : a->get_mapped_preferred_extent(); if (strchr("RICE", ExtentMap::category(ap, sp))) { rose_addr_t x = filespace ? a->get_file_alignment() : a->get_mapped_alignment(); #if BOOST_VERSION < 106900 align = boost::math::lcm(align, x?x:1); // deprecated in boost-1.69.0 #else align = boost::integer::lcm(align, x?x:1); // not present before boost-1.60.0 #endif } } aligned_sa = (sa/align + (sa%align?1:0))*align; aligned_sasn = ((sa+sn)/align + ((sa+sn)%align?1:0))*align; if (debug) { fprintf(stderr, "%s Alignment LCM = 0x%08" PRIx64 " (%" PRIu64 ")\n", p, align, align); fprintf(stderr, "%s Aligned Sa = 0x%08" PRIx64 " (%" PRIu64 ")\n", p, aligned_sa, aligned_sa); fprintf(stderr, "%s Aligned Sa+Sn = 0x%08" PRIx64 " (%" PRIu64 ")\n", p, aligned_sasn, aligned_sasn); } /* Are there any sections to the right of neighborhood(S)? If so, find the one with the lowest start address and use * that to define the size of the hole right of neighborhood(S). */ if (0==villagers.size()) break; SgAsmGenericSection *after_hole = NULL; Extent hp(0, 0); for (size_t i=0; i<villagers.size(); i++) { SgAsmGenericSection *a = villagers[i]; Extent ap = filespace ? a->get_file_extent() : a->get_mapped_preferred_extent(); if (!after_hole || ap.relaxed_first()<hp.relaxed_first()) { after_hole = a; hp = ap; } } ROSE_ASSERT(after_hole); ROSE_ASSERT(hp.relaxed_first() > nhs.relaxed_first()+nhs.size()); rose_addr_t hole_size = hp.relaxed_first() - (nhs.relaxed_first()+nhs.size()); if (debug) { fprintf(stderr, "%s hole size = 0x%08" PRIx64 " (%" PRIu64 "); need 0x%08" PRIx64 " (%" PRIu64 "); %s\n", p, hole_size, hole_size, aligned_sasn, aligned_sasn, hole_size>=aligned_sasn ? "large enough" : "not large enough"); } if (hole_size >= aligned_sasn) break; rose_addr_t need_more = aligned_sasn - hole_size; /* Hole is not large enough. We need to recursively move things that are right of our neighborhood, then recompute the * all-sections address map and neighborhood(S). */ ROSE_ASSERT(0==pass); /*logic problem since the recursive call should have enlarged the hole enough*/ if (debug) { fprintf(stderr, "%s Calling recursively to increase hole size by 0x%08" PRIx64 " (%" PRIu64 ") bytes\n", p, need_more, need_more); } shift_extend(after_hole, need_more, 0, space, elasticity); if (debug) fprintf(stderr, "%s Returned from recursive call\n", p); } /* Consider sections that are in the same neighborhood as S */ if (debug) fprintf(stderr, "%s -- ADJUSTING --\n", p); bool resized_mem = false; for (size_t i=0; i<neighbors.size(); i++) { SgAsmGenericSection *a = neighbors[i]; Extent ap = filespace ? a->get_file_extent() : a->get_mapped_preferred_extent(); switch (ExtentMap::category(ap, sp)) { case 'L': break; case 'R': if (filespace) { a->set_offset(a->get_offset()+aligned_sasn); } else { a->set_mapped_preferred_rva(a->get_mapped_preferred_rva()+aligned_sasn); } break; case 'C': /*including S itself*/ case 'E': if (filespace) { a->set_offset(a->get_offset()+aligned_sa); a->set_size(a->get_size()+sn); if (memspace && !resized_mem && a->is_mapped()) { shift_extend(a, 0, sn, ADDRSP_MEMORY, elasticity); resized_mem = true; } } else { a->set_mapped_preferred_rva(a->get_mapped_preferred_rva()+aligned_sa); a->set_mapped_size(a->get_mapped_size()+sn); } break; case 'O': if (ap.relaxed_first()==sp.relaxed_first()) { if (filespace) { a->set_offset(a->get_offset()+aligned_sa); a->set_size(a->get_size()+sn); } else { a->set_mapped_preferred_rva(a->get_mapped_preferred_rva()+aligned_sa); a->set_mapped_size(a->get_mapped_size()+sn); } } else { if (filespace) { a->set_size(a->get_size()+aligned_sasn); if (memspace && !resized_mem && a->is_mapped()) { shift_extend(a, 0, aligned_sasn, ADDRSP_MEMORY, elasticity); resized_mem = true; } } else { a->set_mapped_size(a->get_mapped_size()+aligned_sasn); } } break; case 'I': if (filespace) { a->set_offset(a->get_offset()+aligned_sa); } else { a->set_mapped_preferred_rva(a->get_mapped_preferred_rva()+aligned_sa); } break; case 'B': if (filespace) { a->set_size(a->get_size()+sn); if (memspace && !resized_mem && a->is_mapped()) { shift_extend(a, 0, sn, ADDRSP_MEMORY, elasticity); resized_mem = true; } } else { a->set_mapped_size(a->get_size()+sn); } break; default: ROSE_ASSERT(!"invalid extent category"); break; } if (debug) { const char *space_name = filespace ? "file" : "mem"; rose_addr_t x = filespace ? a->get_file_alignment() : a->get_mapped_alignment(); fprintf(stderr, "%s %4s-%c %c0x%08" PRIx64 " 0x%08" PRIx64 " 0x%08" PRIx64, p, space_name, ExtentMap::category(ap, sp), 0==ap.relaxed_first()%(x?x:1)?' ':'!', ap.relaxed_first(), ap.size(), ap.relaxed_first()+ap.size()); Extent newap = filespace ? a->get_file_extent() : a->get_mapped_preferred_extent(); fprintf(stderr, " -> %c0x%08" PRIx64 " 0x%08" PRIx64 " 0x%08" PRIx64, 0==newap.relaxed_first()%(x?x:1)?' ':'!', newap.relaxed_first(), newap.size(), newap.relaxed_first()+newap.size()); fprintf(stderr, " [%2d] \"%s\"\n", a->get_id(), a->get_name()->get_string(true).c_str()); } } if (debug) fprintf(stderr, "%s -- END --\n", p); }
int ExtentMapConverter::doCvt(unsigned oldExtentSize, unsigned newExtentSize, const string&filename) { int currentSize, loadSize[3]; ifstream in; em.grabEMEntryTable(ExtentMap::WRITE); try { em.grabFreeList(ExtentMap::WRITE); } catch(...) { em.releaseEMEntryTable(ExtentMap::WRITE); throw; } div_t d = div((int)oldExtentSize, (int)newExtentSize); idbassert(d.quot > 1); idbassert(d.rem == 0); const unsigned mult = d.quot; in.open(filename.c_str()); if (!in) { log_errno("ExtentMap::load(): open"); em.releaseFreeList(ExtentMap::WRITE); em.releaseEMEntryTable(ExtentMap::WRITE); throw std::ios_base::failure("ExtentMap::load(): open failed. Check the error log."); } in.exceptions(ios_base::badbit | ios_base::failbit); try { in.read((char *) &loadSize, 3*sizeof(int)); } catch(...) { in.close(); em.releaseFreeList(ExtentMap::WRITE); em.releaseEMEntryTable(ExtentMap::WRITE); throw; } const int emVersion = loadSize[0]; const int emNumElements = loadSize[1]; const int flNumElements = loadSize[2]; /* What's a safe upper limit on the # of EM and FL entries? */ #define EM_MAGIC_V3 0x76f78b1e if ( emVersion != EM_MAGIC_V3 || emNumElements < 0 || flNumElements < 0) { in.close(); em.releaseFreeList(ExtentMap::WRITE); em.releaseEMEntryTable(ExtentMap::WRITE); log("ExtentMap::load64(): That file is not a valid 64-bit ExtentMap image"); throw std::runtime_error("ExtentMap::load64(): That file is not a valid 64-bit ExtentMap image"); } memset(em.fExtentMap, 0, em.fEMShminfo->allocdSize); memset(em.fFreeList, 0, em.fFLShminfo->allocdSize); em.fEMShminfo->currentSize = 0; em.fFLShminfo->currentSize = 0; int j = 0; int maxLoops = (emNumElements * (signed)mult - em.fEMShminfo->allocdSize/sizeof(EMEntry)) / 100 + 1; int target = (int)(maxLoops * .02); if (maxLoops < 50) target = 1; // allocate shared memory for extent data for (currentSize = em.fEMShminfo->allocdSize/sizeof(EMEntry); currentSize < (emNumElements * (signed)mult); currentSize = em.fEMShminfo->allocdSize/sizeof(EMEntry)) { em.growEMShmseg(); if ((j % target) == 0) cout << '.' << flush; j++; } cout << endl; // allocate shared memory for freelist for (currentSize = em.fFLShminfo->allocdSize/sizeof(InlineLBIDRange); currentSize < flNumElements; currentSize = em.fFLShminfo->allocdSize/sizeof(InlineLBIDRange)) { em.growFLShmseg(); } try { typedef map<int, vector<int> > OIDMap_t; OIDMap_t OIDMap; u_int8_t buf[emNumElements * sizeof(EMEntry)]; u_int8_t buf2[flNumElements * sizeof(InlineLBIDRange)]; in.read((char *) buf, emNumElements * sizeof(EMEntry)); //memcpy(fExtentMap, buf, emNumElements * sizeof(EMEntry)); EMEntry* emSrc = reinterpret_cast<EMEntry*>(&buf[0]); j = 0; for (int i = 0; i < emNumElements; i++) { vector<int>& oidv = OIDMap[emSrc[i].fileID]; for (unsigned k = 0; k < mult; k++) { oidv.push_back(j); //em.fExtentMap[j].range.start = emSrc[i].range.start; em.fExtentMap[j].range.start = emSrc[i].range.start + (k * newExtentSize); //em.fExtentMap[j].range.size = emSrc[i].range.size; em.fExtentMap[j].range.size = newExtentSize / 1024; em.fExtentMap[j].fileID = emSrc[i].fileID; em.fExtentMap[j].blockOffset = emSrc[i].blockOffset + (k * newExtentSize); em.fExtentMap[j].HWM = emSrc[i].HWM; em.fExtentMap[j].txnID = emSrc[i].txnID; em.fExtentMap[j].secondHWM = emSrc[i].secondHWM; em.fExtentMap[j].nextHeader = emSrc[i].nextHeader; em.fExtentMap[j].partition.type = emSrc[i].partition.type; //em.fExtentMap[j].partition.cprange.hi_val = emSrc[i].partition.cprange.hi_val; em.fExtentMap[j].partition.cprange.hi_val = numeric_limits<int64_t>::min(); //em.fExtentMap[j].partition.cprange.lo_val = emSrc[i].partition.cprange.lo_val; em.fExtentMap[j].partition.cprange.lo_val = numeric_limits<int64_t>::max(); //em.fExtentMap[j].partition.cprange.sequenceNum = emSrc[i].partition.cprange.sequenceNum; em.fExtentMap[j].partition.cprange.sequenceNum = 0; //em.fExtentMap[j].partition.cprange.isValid = emSrc[i].partition.cprange.isValid; em.fExtentMap[j].partition.cprange.isValid = CP_INVALID; j++; } } em.fEMShminfo->currentSize = j * sizeof(EMEntry); cout << j << " total new em entries from " << emNumElements << " file entries" << endl; cout << OIDMap.size() << " OIDs added to em" << endl; OIDMap_t::const_iterator iter = OIDMap.begin(); OIDMap_t::const_iterator end = OIDMap.end(); int l = 0; while (iter != end) { const vector<int>& oidv = iter->second; vector<int>::const_reverse_iterator riter = oidv.rbegin(); vector<int>::const_reverse_iterator rend = oidv.rend(); HWM_t hwm = em.fExtentMap[*riter].HWM; while (riter != rend) { if (em.fExtentMap[*riter].blockOffset > hwm) { em.fExtentMap[*riter].fileID = numeric_limits<int>::max(); em.fExtentMap[*riter].blockOffset = 0; em.fExtentMap[*riter].HWM = 0; l++; } else break; ++riter; } ++iter; } cout << l << " entries moved to OID " << numeric_limits<int>::max() << endl; #if 0 int k = j; for (int j = 0; j < k; j++) cout << em.fExtentMap[j].range.start << '\t' << em.fExtentMap[j].range.size << '\t' << em.fExtentMap[j].fileID << '\t' << em.fExtentMap[j].blockOffset << '\t' << em.fExtentMap[j].HWM << '\t' << em.fExtentMap[j].txnID << '\t' << em.fExtentMap[j].secondHWM << '\t' << em.fExtentMap[j].nextHeader << endl; #endif in.read((char *) buf2, flNumElements * sizeof(InlineLBIDRange)); //memcpy(fFreeList, buf2, flNumElements * sizeof(InlineLBIDRange)); InlineLBIDRange* lrSrc = reinterpret_cast<InlineLBIDRange*>(&buf2[0]); j = 0; for (int i = 0; i < flNumElements; i++) { em.fFreeList[j].start = lrSrc[i].start; em.fFreeList[j].size = lrSrc[i].size; j++; } em.fFLShminfo->currentSize = j * sizeof(InlineLBIDRange); cout << j << " total new fl entries from " << flNumElements << " file entries" << endl; } catch(...) { in.close(); em.releaseFreeList(ExtentMap::WRITE); em.releaseEMEntryTable(ExtentMap::WRITE); throw; } in.close(); em.releaseFreeList(ExtentMap::WRITE); em.releaseEMEntryTable(ExtentMap::WRITE); return 0; }
static void* EMRunner(void *arg) { // keep track of LBID ranges allocated here and // randomly allocate, lookup, delete, get/set HWM, and // destroy the EM object. struct EMEntries { u_int64_t LBIDstart; u_int32_t size; int OID; u_int32_t FBO; u_int32_t HWM; u_int32_t secondHWM; u_int32_t txnID; struct EMEntries *next; EMEntries() { next = NULL; HWM = 0; secondHWM = 0; txnID = 0; } }; #ifdef BRM_VERBOSE int threadNum = reinterpret_cast<int>(arg); #endif int op, listSize = 0, i; uint randstate; struct EMEntries *head = NULL, *tmp; struct timeval tv; ExtentMap *em; vector<LBID_t> lbids; #ifdef BRM_VERBOSE cerr << "thread number " << threadNum << " started." << endl; #endif gettimeofday(&tv, NULL); randstate = static_cast<uint>(tv.tv_usec); em = new ExtentMap(); while (!threadStop) { op = rand_r(&randstate) % 10; #ifdef BRM_VERBOSE cerr << "next op is " << op << endl; #endif switch (op) { case 0: //allocate space for a new file { struct EMEntries *newEm; int size = rand_r(&randstate) % 102399 + 1; int entries, OID, allocdSize; pthread_mutex_lock(&mutex); OID = oid++; pthread_mutex_unlock(&mutex); em->createExtent(size, OID, lbids, allocdSize); em->confirmChanges(); entries = size/em->getExtentSize(); if ((size % em->getExtentSize()) != 0) entries++; CPPUNIT_ASSERT((uint)entries == lbids.size()); for (i = 0 ; i < entries; i++) { newEm = new EMEntries(); newEm->size = em->getExtentSize(); newEm->OID = OID; newEm->FBO = i * em->getExtentSize(); newEm->LBIDstart = lbids[i]; newEm->next = head; head = newEm; listSize++; } #ifdef BRM_VERBOSE cerr << "created new space for OID " << newEm->OID << endl; #endif em->checkConsistency(); break; } case 1: //allocate space for an existing file { if (listSize == 0) break; struct EMEntries *newEm, *tmp; int size = rand_r(&randstate) % 102399 + 1; int fileRand = rand_r(&randstate) % listSize; int i, lastExtent, blockEnd, oid; int tmpHWM, entries, allocdSize; vector<LBID_t> lbids; for (i = 0, tmp = head; i < fileRand; i++) tmp = tmp->next; oid = tmp->OID; for (lastExtent = 0, tmp = head; tmp != NULL; tmp = tmp->next) { if (tmp->OID != oid) continue; tmpHWM = tmp->HWM; blockEnd = tmp->FBO + tmp->size; if (lastExtent < blockEnd) lastExtent = blockEnd; } em->createExtent(size, oid, lbids, allocdSize); em->confirmChanges(); entries = size/em->getExtentSize(); if ((size % em->getExtentSize()) != 0) entries++; CPPUNIT_ASSERT((uint)entries == lbids.size()); for (i = 0; i < entries; i++) { newEm = new EMEntries(); if (i != entries) newEm->size = em->getExtentSize(); else newEm->size = size % em->getExtentSize(); newEm->OID = oid; newEm->FBO = lastExtent + (i*em->getExtentSize()); newEm->LBIDstart = lbids[i]; newEm->HWM = tmpHWM; newEm->next = head; head = newEm; listSize++; } #ifdef BRM_VERBOSE cerr << "created another extent for OID " << newEm->OID << endl; #endif em->checkConsistency(); break; } case 2: //delete an OID { if (listSize == 0) break; struct EMEntries *tmp, *prev; int fileRand = rand_r(&randstate) % listSize; int i, oid; for (i = 0, tmp = head; i < fileRand; i++) tmp = tmp->next; oid = tmp->OID; em->deleteOID(oid); em->confirmChanges(); for (tmp = head; tmp != NULL;) { if (tmp->OID == oid) { if (tmp == head) { head = head->next; delete tmp; tmp = head; } else { prev->next = tmp->next; delete tmp; tmp = prev->next; } listSize--; } else { prev = tmp; tmp = tmp->next; } } #ifdef BRM_VERBOSE cerr << "deleted OID " << oid << endl; #endif em->checkConsistency(); break; } case 3: //lookup by LBID { if (listSize == 0) break; int entryRand = rand_r(&randstate) % listSize; int i, err, offset, oid; struct EMEntries *tmp; LBID_t target; u_int32_t fbo; for (i = 0, tmp = head; i < entryRand; i++) tmp = tmp->next; offset = rand_r(&randstate) % tmp->size; target = tmp->LBIDstart + offset; err = em->lookup(target, oid, fbo); #ifdef BRM_VERBOSE cerr << "looked up LBID " << target << " got oid " << oid << " fbo " << fbo << endl; cerr << " oid should be " << tmp->OID << " fbo should be " << offset+tmp->FBO << endl; #endif CPPUNIT_ASSERT(err == 0); CPPUNIT_ASSERT(oid == tmp->OID); CPPUNIT_ASSERT(fbo == offset + tmp->FBO); em->checkConsistency(); break; } case 4: //lookup by OID, FBO { if (listSize == 0) break; int entryRand = rand_r(&randstate) % listSize; int i, oid, err, offset; struct EMEntries *tmp; LBID_t lbid; for (i = 0, tmp = head; i < entryRand; i++) tmp = tmp->next; offset = rand_r(&randstate) % tmp->size; oid = tmp->OID; err = em->lookup(oid, offset + tmp->FBO, lbid); #ifdef BRM_VERBOSE cerr << "looked up OID " << oid << " fbo " << offset + tmp->FBO << " got lbid " << lbid << endl; cerr << " lbid should be " << tmp->LBIDstart + offset << endl; #endif CPPUNIT_ASSERT(err == 0); CPPUNIT_ASSERT(lbid == tmp->LBIDstart + offset); em->checkConsistency(); break; } case 5: //getHWM { if (listSize == 0) break; int entryRand = rand_r(&randstate) % listSize; int i; struct EMEntries *tmp; u_int32_t hwm; for (i = 0, tmp = head; i < entryRand; i++) tmp = tmp->next; hwm = em->getHWM(tmp->OID); #ifdef BRM_VERBOSE cerr << "stored HWM for OID " << tmp->OID << " is " << tmp->HWM << " BRM says it's " << hwm << endl; #endif CPPUNIT_ASSERT(hwm == tmp->HWM); em->checkConsistency(); break; } case 6: //setHWM { if (listSize == 0) break; int entryRand = rand_r(&randstate) % listSize; int i, hwm, oid; struct EMEntries *tmp; for (i = 0, tmp = head; i < entryRand; i++) tmp = tmp->next; oid = tmp->OID; hwm = rand_r(&randstate) % (tmp->FBO + em->getExtentSize()); em->setHWM(oid, hwm); em->confirmChanges(); for (tmp = head; tmp != NULL; tmp = tmp->next) if (tmp->OID == oid) tmp->HWM = hwm; #ifdef BRM_VERBOSE cerr << "setHWM of OID " << oid << " to " << hwm << endl; #endif em->checkConsistency(); break; } case 7: // renew this EM object { delete em; em = new ExtentMap(); #ifdef BRM_VERBOSE cerr << "got a new EM instance" << endl; #endif em->checkConsistency(); break; } case 8: //getBulkInsertVars { if (listSize == 0) break; HWM_t hwm; VER_t txnID; int entryRand = rand_r(&randstate) % listSize; int i, err, offset; EMEntries *tmp; LBID_t lbid; for (i = 0, tmp = head; i < entryRand; i++) tmp = tmp->next; offset = rand_r(&randstate) % tmp->size; lbid = tmp->LBIDstart + offset; err = em->getBulkInsertVars(lbid, hwm, txnID); CPPUNIT_ASSERT(err == 0); CPPUNIT_ASSERT(hwm == tmp->secondHWM); CPPUNIT_ASSERT(txnID == tmp->txnID); break; } case 9: //setBulkInsertVars { if (listSize == 0) break; int entryRand = rand_r(&randstate) % listSize; int i, err, offset; EMEntries *tmp; for (i = 0, tmp = head; i < entryRand; i++) tmp = tmp->next; offset = rand_r(&randstate) % tmp->size; tmp->secondHWM = rand_r(&randstate) % MAXINT; tmp->txnID = rand_r(&randstate) % MAXINT; err = em->setBulkInsertVars(tmp->LBIDstart + offset, tmp->secondHWM, tmp->txnID); em->confirmChanges(); CPPUNIT_ASSERT(err == 0); break; } default: break; } } delete em; while (head != NULL) { tmp = head->next; delete head; head = tmp; } #ifdef BRM_VERBOSE cerr << "thread " << threadNum << " exiting" << endl; #endif return NULL; }