/*
* Unit
*/
void print(std::ostream& os, const IRUnit& unit,
const RegAllocInfo* regs, const AsmInfo* asmInfo,
const GuardConstraints* guards, bool dotBodies) {
// For nice-looking dumps, we want to remember curMarker between blocks.
BCMarker curMarker;
auto const layout = layoutBlocks(unit);
auto const& blocks = layout.blocks;
if (dumpIREnabled(kExtraExtraLevel)) printOpcodeStats(os, blocks);
// Print the block CFG above the actual code.
os << "digraph G {\n";
for (Block* block : blocks) {
if (block->empty()) continue;
if (dotBodies && block->hint() != Block::Hint::Unlikely &&
block->hint() != Block::Hint::Unused) {
// Include the IR in the body of the node
std::ostringstream out;
print(out, block, regs, asmInfo, guards, &curMarker);
auto bodyRaw = out.str();
std::string body;
body.reserve(bodyRaw.size() * 1.25);
for (auto c : bodyRaw) {
if (c == '\n') body += "\\n";
else if (c == '"') body += "\\\"";
else if (c == '\\') body += "\\\\";
else body += c;
}
os << folly::format("B{} [shape=\"box\" label=\"{}\"]\n",
block->id(), body);
}
auto* next = block->next();
auto* taken = block->taken();
if (!next && !taken) continue;
if (next) {
os << folly::format("B{} -> B{}", block->id(), next->id());
if (taken) os << "; ";
}
if (taken) os << folly::format("B{} -> B{}", block->id(), taken->id());
os << "\n";
}
os << "}\n";
curMarker = BCMarker();
for (auto it = blocks.begin(); it != blocks.end(); ++it) {
if (it == layout.acoldIt) {
os << folly::format("\n{:-^60}", "cold blocks");
}
if (it == layout.afrozenIt) {
os << folly::format("\n{:-^60}", "frozen blocks");
}
print(os, *it, regs, asmInfo, guards, &curMarker);
}
}
void genCodeImpl(CodeBlock& mainCode,
CodeBlock& stubsCode,
IRUnit& unit,
std::vector<TransBCMapping>* bcMap,
JIT::MCGenerator* mcg,
const RegAllocInfo& regs,
AsmInfo* asmInfo) {
LiveRegs live_regs = computeLiveRegs(unit, regs);
CodegenState state(unit, regs, live_regs, asmInfo);
// Returns: whether a block has already been emitted.
DEBUG_ONLY auto isEmitted = [&](Block* block) {
return state.addresses[block];
};
/*
* Emit the given block on the supplied assembler. The `nextLinear'
* is the next block that will be emitted on this assembler. If is
* not the next block in control flow order, then emit a patchable jump
* to the next flow block.
*/
auto emitBlock = [&](CodeBlock& cb, Block* block, Block* nextLinear) {
assert(!isEmitted(block));
FTRACE(6, "genBlock {} on {}\n", block->id(),
cb.base() == stubsCode.base() ? "astubs" : "a");
auto const aStart = cb.frontier();
auto const astubsStart = stubsCode.frontier();
mcg->backEnd().patchJumps(cb, state, block);
state.addresses[block] = aStart;
// If the block ends with a Jmp and the next block is going to be
// its target, we don't need to actually emit it.
IRInstruction* last = &block->back();
state.noTerminalJmp = last->op() == Jmp && nextLinear == last->taken();
if (state.asmInfo) {
state.asmInfo->asmRanges[block] = TcaRange(aStart, cb.frontier());
}
genBlock(unit, cb, stubsCode, mcg, state, block, bcMap);
auto nextFlow = block->next();
if (nextFlow && nextFlow != nextLinear) {
mcg->backEnd().emitFwdJmp(cb, nextFlow, state);
}
if (state.asmInfo) {
state.asmInfo->asmRanges[block] = TcaRange(aStart, cb.frontier());
if (cb.base() != stubsCode.base()) {
state.asmInfo->astubRanges[block] = TcaRange(astubsStart,
stubsCode.frontier());
}
}
};
if (RuntimeOption::EvalHHIRGenerateAsserts) {
mcg->backEnd().emitTraceCall(mainCode, unit.bcOff());
}
auto const linfo = layoutBlocks(unit);
for (auto it = linfo.blocks.begin(); it != linfo.astubsIt; ++it) {
Block* nextLinear = boost::next(it) != linfo.astubsIt
? *boost::next(it) : nullptr;
emitBlock(mainCode, *it, nextLinear);
}
for (auto it = linfo.astubsIt; it != linfo.blocks.end(); ++it) {
Block* nextLinear = boost::next(it) != linfo.blocks.end()
? *boost::next(it) : nullptr;
emitBlock(stubsCode, *it, nextLinear);
}
if (debug) {
for (Block* UNUSED block : linfo.blocks) {
assert(isEmitted(block));
}
}
}
static void genCodeImpl(IRUnit& unit, AsmInfo* asmInfo) {
auto regs = allocateRegs(unit);
assert(checkRegisters(unit, regs)); // calls checkCfg internally.
Timer _t(Timer::codeGen);
LiveRegs live_regs = computeLiveRegs(unit, regs);
CodegenState state(unit, regs, live_regs, asmInfo);
// Returns: whether a block has already been emitted.
DEBUG_ONLY auto isEmitted = [&](Block* block) {
return state.addresses[block];
};
CodeBlock& mainCodeIn = mcg->code.main();
CodeBlock& coldCodeIn = mcg->code.cold();
CodeBlock* frozenCode = &mcg->code.frozen();
CodeBlock mainCode;
CodeBlock coldCode;
bool relocate = false;
if (RuntimeOption::EvalJitRelocationSize &&
mcg->backEnd().supportsRelocation() &&
coldCodeIn.canEmit(RuntimeOption::EvalJitRelocationSize * 3)) {
/*
* This is mainly to exercise the relocator, and ensure that its
* not broken by new non-relocatable code. Later, it will be
* used to do some peephole optimizations, such as reducing branch
* sizes.
* Allocate enough space that the relocated cold code doesn't
* overlap the emitted cold code.
*/
static unsigned seed = 42;
auto off = rand_r(&seed) & (mcg->backEnd().cacheLineSize() - 1);
coldCode.init(coldCodeIn.frontier() +
RuntimeOption::EvalJitRelocationSize + off,
RuntimeOption::EvalJitRelocationSize - off, "cgRelocCold");
mainCode.init(coldCode.frontier() +
RuntimeOption::EvalJitRelocationSize + off,
RuntimeOption::EvalJitRelocationSize - off, "cgRelocMain");
relocate = true;
} else {
/*
* Use separate code blocks, so that attempts to use the mcg's
* code blocks directly will fail (eg by overwriting the same
* memory being written through these locals).
*/
coldCode.init(coldCodeIn.frontier(), coldCodeIn.available(),
coldCodeIn.name().c_str());
mainCode.init(mainCodeIn.frontier(), mainCodeIn.available(),
mainCodeIn.name().c_str());
}
if (frozenCode == &coldCodeIn) {
frozenCode = &coldCode;
}
auto frozenStart = frozenCode->frontier();
auto coldStart DEBUG_ONLY = coldCodeIn.frontier();
auto mainStart DEBUG_ONLY = mainCodeIn.frontier();
auto bcMap = &mcg->cgFixups().m_bcMap;
{
mcg->code.lock();
mcg->cgFixups().setBlocks(&mainCode, &coldCode, frozenCode);
SCOPE_EXIT {
mcg->cgFixups().setBlocks(nullptr, nullptr, nullptr);
mcg->code.unlock();
};
/*
* Emit the given block on the supplied assembler. The `nextLinear'
* is the next block that will be emitted on this assembler. If is
* not the next block in control flow order, then emit a patchable jump
* to the next flow block.
*/
auto emitBlock = [&](CodeBlock& cb, Block* block, Block* nextLinear) {
assert(!isEmitted(block));
FTRACE(6, "genBlock {} on {}\n", block->id(),
cb.base() == coldCode.base() ? "acold" : "a");
auto const aStart = cb.frontier();
auto const acoldStart = coldCode.frontier();
auto const afrozenStart = frozenCode->frontier();
mcg->backEnd().patchJumps(cb, state, block);
state.addresses[block] = aStart;
// If the block ends with a Jmp and the next block is going to be
// its target, we don't need to actually emit it.
IRInstruction* last = &block->back();
state.noTerminalJmp = last->op() == Jmp && nextLinear == last->taken();
if (state.asmInfo) {
state.asmInfo->asmRanges[block] = TcaRange(aStart, cb.frontier());
}
genBlock(unit, cb, coldCode, *frozenCode, state, block, bcMap);
auto nextFlow = block->next();
if (nextFlow && nextFlow != nextLinear) {
mcg->backEnd().emitFwdJmp(cb, nextFlow, state);
}
if (state.asmInfo) {
state.asmInfo->asmRanges[block] = TcaRange(aStart, cb.frontier());
if (cb.base() != coldCode.base() && frozenCode != &coldCode) {
state.asmInfo->acoldRanges[block] = TcaRange(acoldStart,
coldCode.frontier());
}
if (cb.base() != frozenCode->base()) {
state.asmInfo->afrozenRanges[block] = TcaRange(afrozenStart,
frozenCode->frontier());
}
}
};
if (RuntimeOption::EvalHHIRGenerateAsserts) {
mcg->backEnd().emitTraceCall(mainCode, unit.bcOff());
}
auto const linfo = layoutBlocks(unit);
for (auto it = linfo.blocks.begin(); it != linfo.acoldIt; ++it) {
Block* nextLinear = boost::next(it) != linfo.acoldIt
? *boost::next(it) : nullptr;
emitBlock(mainCode, *it, nextLinear);
}
for (auto it = linfo.acoldIt; it != linfo.afrozenIt; ++it) {
Block* nextLinear = boost::next(it) != linfo.afrozenIt
? *boost::next(it) : nullptr;
emitBlock(coldCode, *it, nextLinear);
}
for (auto it = linfo.afrozenIt; it != linfo.blocks.end(); ++it) {
Block* nextLinear = boost::next(it) != linfo.blocks.end()
? *boost::next(it) : nullptr;
emitBlock(*frozenCode, *it, nextLinear);
}
if (debug) {
for (Block* UNUSED block : linfo.blocks) {
assert(isEmitted(block));
}
}
}
assert(coldCodeIn.frontier() == coldStart);
assert(mainCodeIn.frontier() == mainStart);
if (relocate) {
if (asmInfo) {
printUnit(kRelocationLevel, unit, " before relocation ", ®s, asmInfo);
}
auto& be = mcg->backEnd();
RelocationInfo rel;
be.relocate(rel, mainCodeIn,
mainCode.base(), mainCode.frontier(),
mcg->cgFixups());
be.relocate(rel, coldCodeIn,
coldCode.base(), coldCode.frontier(),
mcg->cgFixups());
if (frozenCode != &coldCode) {
rel.recordRange(frozenStart, frozenCode->frontier(),
frozenStart, frozenCode->frontier());
}
be.adjustForRelocation(rel, mcg->cgFixups());
be.adjustForRelocation(rel, asmInfo, mcg->cgFixups());
if (asmInfo) {
static int64_t mainDeltaTot = 0, coldDeltaTot = 0;
int64_t mainDelta =
(mainCodeIn.frontier() - mainStart) -
(mainCode.frontier() - mainCode.base());
int64_t coldDelta =
(coldCodeIn.frontier() - coldStart) -
(coldCode.frontier() - coldCode.base());
mainDeltaTot += mainDelta;
HPHP::Trace::traceRelease("main delta after relocation: %" PRId64
" (%" PRId64 ")\n",
mainDelta, mainDeltaTot);
coldDeltaTot += coldDelta;
HPHP::Trace::traceRelease("cold delta after relocation: %" PRId64
" (%" PRId64 ")\n",
coldDelta, coldDeltaTot);
}
#ifndef NDEBUG
auto& ip = mcg->cgFixups().m_inProgressTailJumps;
for (size_t i = 0; i < ip.size(); ++i) {
const auto& ib = ip[i];
assert(!mainCode.contains(ib.toSmash()));
assert(!coldCode.contains(ib.toSmash()));
}
memset(mainCode.base(), 0xcc, mainCode.frontier() - mainCode.base());
memset(coldCode.base(), 0xcc, coldCode.frontier() - coldCode.base());
#endif
} else {
coldCodeIn.skip(coldCode.frontier() - coldCodeIn.frontier());
mainCodeIn.skip(mainCode.frontier() - mainCodeIn.frontier());
}
if (asmInfo) {
printUnit(kCodeGenLevel, unit, " after code gen ", ®s, asmInfo);
}
}
void BackEnd::genCodeImpl(IRUnit& unit, AsmInfo* asmInfo) {
ctr++;
auto regs = allocateRegs(unit);
assert(checkRegisters(unit, regs)); // calls checkCfg internally.
Timer _t(Timer::codeGen);
LiveRegs live_regs = computeLiveRegs(unit, regs);
CodegenState state(unit, regs, live_regs, asmInfo);
CodeBlock& mainCodeIn = mcg->code.main();
CodeBlock& coldCodeIn = mcg->code.cold();
CodeBlock* frozenCode = &mcg->code.frozen();
CodeBlock mainCode;
CodeBlock coldCode;
bool relocate = false;
if (RuntimeOption::EvalJitRelocationSize &&
supportsRelocation() &&
coldCodeIn.canEmit(RuntimeOption::EvalJitRelocationSize * 3)) {
/*
* This is mainly to exercise the relocator, and ensure that its
* not broken by new non-relocatable code. Later, it will be
* used to do some peephole optimizations, such as reducing branch
* sizes.
* Allocate enough space that the relocated cold code doesn't
* overlap the emitted cold code.
*/
static unsigned seed = 42;
auto off = rand_r(&seed) & (cacheLineSize() - 1);
coldCode.init(coldCodeIn.frontier() +
RuntimeOption::EvalJitRelocationSize + off,
RuntimeOption::EvalJitRelocationSize - off, "cgRelocCold");
mainCode.init(coldCode.frontier() +
RuntimeOption::EvalJitRelocationSize + off,
RuntimeOption::EvalJitRelocationSize - off, "cgRelocMain");
relocate = true;
} else {
/*
* Use separate code blocks, so that attempts to use the mcg's
* code blocks directly will fail (eg by overwriting the same
* memory being written through these locals).
*/
coldCode.init(coldCodeIn.frontier(), coldCodeIn.available(),
coldCodeIn.name().c_str());
mainCode.init(mainCodeIn.frontier(), mainCodeIn.available(),
mainCodeIn.name().c_str());
}
if (frozenCode == &coldCodeIn) {
frozenCode = &coldCode;
}
auto frozenStart = frozenCode->frontier();
auto coldStart DEBUG_ONLY = coldCodeIn.frontier();
auto mainStart DEBUG_ONLY = mainCodeIn.frontier();
size_t hhir_count{0};
{
mcg->code.lock();
mcg->cgFixups().setBlocks(&mainCode, &coldCode, frozenCode);
SCOPE_EXIT {
mcg->cgFixups().setBlocks(nullptr, nullptr, nullptr);
mcg->code.unlock();
};
if (RuntimeOption::EvalHHIRGenerateAsserts) {
emitTraceCall(mainCode, unit.bcOff());
}
auto const linfo = layoutBlocks(unit);
auto main_start = mainCode.frontier();
auto cold_start = coldCode.frontier();
auto frozen_start = frozenCode->frontier();
Vasm vasm(&state.meta);
auto& vunit = vasm.unit();
// create the initial set of vasm numbered the same as hhir blocks.
for (uint32_t i = 0, n = unit.numBlocks(); i < n; ++i) {
state.labels[i] = vunit.makeBlock(AreaIndex::Main);
}
vunit.roots.push_back(state.labels[unit.entry()]);
vasm.main(mainCode);
vasm.cold(coldCode);
vasm.frozen(*frozenCode);
for (auto it = linfo.blocks.begin(); it != linfo.blocks.end(); ++it) {
auto block = *it;
auto v = block->hint() == Block::Hint::Unlikely ? vasm.cold() :
block->hint() == Block::Hint::Unused ? vasm.frozen() :
vasm.main();
FTRACE(6, "genBlock {} on {}\n", block->id(),
area_names[(unsigned)v.area()]);
auto b = state.labels[block];
vunit.blocks[b].area = v.area();
v.use(b);
hhir_count += genBlock(unit, v, vasm, state, block);
assert(v.closed());
assert(vasm.main().empty() || vasm.main().closed());
assert(vasm.cold().empty() || vasm.cold().closed());
assert(vasm.frozen().empty() || vasm.frozen().closed());
}
printUnit("after code-gen", vasm.unit());
vasm.finish(vasm_abi);
if (state.asmInfo) {
auto block = unit.entry();
state.asmInfo->asmRanges[block] = {main_start, mainCode.frontier()};
if (mainCode.base() != coldCode.base() && frozenCode != &coldCode) {
state.asmInfo->acoldRanges[block] = {cold_start, coldCode.frontier()};
}
if (mainCode.base() != frozenCode->base()) {
state.asmInfo->afrozenRanges[block] = {frozen_start,
frozenCode->frontier()};
}
}
}
auto bcMap = &mcg->cgFixups().m_bcMap;
if (!bcMap->empty()) {
TRACE(1, "BCMAPS before relocation\n");
for (UNUSED auto& map : *bcMap) {
TRACE(1, "%s %-6d %p %p %p\n", map.md5.toString().c_str(),
map.bcStart, map.aStart, map.acoldStart, map.afrozenStart);
}
}
assert(coldCodeIn.frontier() == coldStart);
assert(mainCodeIn.frontier() == mainStart);
if (relocate) {
if (asmInfo) {
printUnit(kRelocationLevel, unit, " before relocation ", ®s, asmInfo);
}
auto& be = mcg->backEnd();
RelocationInfo rel;
size_t asm_count{0};
asm_count += be.relocate(rel, mainCodeIn,
mainCode.base(), mainCode.frontier(),
mcg->cgFixups());
asm_count += be.relocate(rel, coldCodeIn,
coldCode.base(), coldCode.frontier(),
mcg->cgFixups());
TRACE(1, "hhir-inst-count %ld asm %ld\n", hhir_count, asm_count);
if (frozenCode != &coldCode) {
rel.recordRange(frozenStart, frozenCode->frontier(),
frozenStart, frozenCode->frontier());
}
be.adjustForRelocation(rel, mcg->cgFixups());
be.adjustForRelocation(rel, asmInfo, mcg->cgFixups());
if (asmInfo) {
static int64_t mainDeltaTot = 0, coldDeltaTot = 0;
int64_t mainDelta =
(mainCodeIn.frontier() - mainStart) -
(mainCode.frontier() - mainCode.base());
int64_t coldDelta =
(coldCodeIn.frontier() - coldStart) -
(coldCode.frontier() - coldCode.base());
mainDeltaTot += mainDelta;
HPHP::Trace::traceRelease("main delta after relocation: %" PRId64
" (%" PRId64 ")\n",
mainDelta, mainDeltaTot);
coldDeltaTot += coldDelta;
HPHP::Trace::traceRelease("cold delta after relocation: %" PRId64
" (%" PRId64 ")\n",
coldDelta, coldDeltaTot);
}
#ifndef NDEBUG
auto& ip = mcg->cgFixups().m_inProgressTailJumps;
for (size_t i = 0; i < ip.size(); ++i) {
const auto& ib = ip[i];
assert(!mainCode.contains(ib.toSmash()));
assert(!coldCode.contains(ib.toSmash()));
}
memset(mainCode.base(), 0xcc, mainCode.frontier() - mainCode.base());
memset(coldCode.base(), 0xcc, coldCode.frontier() - coldCode.base());
#endif
} else {
coldCodeIn.skip(coldCode.frontier() - coldCodeIn.frontier());
mainCodeIn.skip(mainCode.frontier() - mainCodeIn.frontier());
}
if (asmInfo) {
printUnit(kCodeGenLevel, unit, " after code gen ", ®s, asmInfo);
}
}
