void SrcRec::replaceOldTranslations() {
// Everyone needs to give up on old translations; send them to the anchor,
// which is a REQ_RETRANSLATE.
m_translations.clear();
m_tailFallbackJumps.clear();
m_topTranslation.store(nullptr, std::memory_order_release);
/*
* It may seem a little weird that we're about to point every
* incoming branch at the anchor, since that's going to just
* unconditionally retranslate this SrcKey and never patch the
* incoming branch to do something else.
*
* The reason this is ok is this mechanism is only used in
* non-RepoAuthoritative mode, and the granularity of code
* invalidation there is such that we'll only have incoming branches
* like this basically within the same file since we don't have
* whole program analysis.
*
* This means all these incoming branches are about to go away
* anyway ...
*
* If we ever change that we'll have to change this to patch to
* some sort of rebind requests.
*/
assert(!RuntimeOption::RepoAuthoritative || RuntimeOption::EvalJitPGO);
patchIncomingBranches(m_anchorTranslation);
}
void SrcRec::replaceOldTranslations(Asm& a, Asm& astubs) {
// Everyone needs to give up on old translations; send them to the anchor,
// which is a REQ_RETRANSLATE
m_translations.clear();
m_tailFallbackJumps.clear();
atomic_release_store(&m_topTranslation, static_cast<TCA>(0));
patchIncomingBranches(a, astubs, m_anchorTranslation);
}
void SrcRec::addDebuggerGuard(TCA dbgGuard, TCA dbgBranchGuardSrc) {
assertx(!m_dbgBranchGuardSrc);
TRACE(1, "SrcRec(%p)::addDebuggerGuard @%p, "
"%zd incoming branches to rechain\n",
this, dbgGuard, m_incomingBranches.size());
patchIncomingBranches(dbgGuard);
// Set m_dbgBranchGuardSrc after patching, so we don't try to patch
// the debug guard.
m_dbgBranchGuardSrc = dbgBranchGuardSrc;
m_topTranslation = dbgGuard;
}
void SrcRec::addDebuggerGuard(Asm& a, Asm &astubs, TCA dbgGuard,
TCA dbgBranchGuardSrc) {
ASSERT(!m_dbgBranchGuardSrc);
TRACE(1, "SrcRec(%p)::addDebuggerGuard @%p, "
"%zd incoming branches to rechain\n",
this, dbgGuard, m_incomingBranches.size());
patchIncomingBranches(a, astubs, dbgGuard);
// Set m_dbgBranchGuardSrc after patching, so we don't try to patch
// the debug guard.
m_dbgBranchGuardSrc = dbgBranchGuardSrc;
atomic_release_store(&m_topTranslation, dbgGuard);
}
void SrcRec::replaceOldTranslations() {
// Everyone needs to give up on old translations; send them to the anchor,
// which is a REQ_RETRANSLATE.
auto translations = std::move(m_translations);
m_tailFallbackJumps.clear();
m_topTranslation = nullptr;
/*
* It may seem a little weird that we're about to point every
* incoming branch at the anchor, since that's going to just
* unconditionally retranslate this SrcKey and never patch the
* incoming branch to do something else.
*
* The reason this is ok is this mechanism is only used in
* non-RepoAuthoritative mode, and the granularity of code
* invalidation there is such that we'll only have incoming branches
* like this basically within the same file since we don't have
* whole program analysis.
*
* This means all these incoming branches are about to go away
* anyway ...
*
* If we ever change that we'll have to change this to patch to
* some sort of rebind requests.
*/
assertx(!RuntimeOption::RepoAuthoritative || RuntimeOption::EvalJitPGO);
patchIncomingBranches(m_anchorTranslation);
// Now that we've smashed all the IBs for these translations they should be
// unreachable-- to prevent a race we treadmill here and then reclaim their
// associated TC space
if (RuntimeOption::EvalEnableReusableTC) {
auto trans = folly::makeMoveWrapper(std::move(translations));
Treadmill::enqueue([trans]() mutable {
for (auto& loc : *trans) {
reclaimTranslation(loc);
}
trans->clear();
});
return;
}
translations.clear();
}
void SrcRec::newTranslation(TransLoc loc,
GrowableVector<IncomingBranch>& tailBranches) {
// When translation punts due to hitting limit, will generate one
// more translation that will call the interpreter.
assertx(m_translations.size() <=
std::max(RuntimeOption::EvalJitMaxProfileTranslations,
RuntimeOption::EvalJitMaxTranslations));
TRACE(1, "SrcRec(%p)::newTranslation @%p, ", this, loc.mainStart());
m_translations.push_back(loc);
if (!m_topTranslation.get()) {
m_topTranslation = loc.mainStart();
patchIncomingBranches(loc.mainStart());
}
/*
* Link all the jumps from the current tail translation to this new
* guy.
*
* It's (mostly) ok if someone is running in this code while we do
* this: we hold the write lease, they'll instead jump to the anchor
* and do REQ_RETRANSLATE and failing to get the write lease they'll
* interp. FIXME: Unfortunately, right now, in an unlikely race
* another thread could create another translation with the same
* type specialization that we just created in this case. (If we
* happen to release the write lease after they jump but before they
* get into REQ_RETRANSLATE, they'll acquire it and generate a
* translation possibly for this same situation.)
*/
for (auto& br : m_tailFallbackJumps) {
br.patch(loc.mainStart());
}
// This is the new tail translation, so store the fallback jump list
// in case we translate this again.
m_tailFallbackJumps.swap(tailBranches);
}
void SrcRec::newTranslation(Asm& a, Asm &astubs, TCA newStart) {
// When translation punts due to hitting limit, will generate one
// more translation that will call the interpreter.
ASSERT(m_translations.size() <= kMaxTranslations);
TRACE(1, "SrcRec(%p)::newTranslation @%p, ", this, newStart);
m_translations.push_back(newStart);
if (!m_topTranslation) {
atomic_release_store(&m_topTranslation, newStart);
patchIncomingBranches(a, astubs, newStart);
}
/*
* Link all the jumps from the current tail translation to this new
* guy.
*
* It's (mostly) ok if someone is running in this code while we do
* this: we hold the write lease, they'll instead jump to the anchor
* and do REQ_RETRANSLATE and failing to get the write lease they'll
* interp. FIXME: Unfortunately, right now, in an unlikely race
* another thread could create another translation with the same
* type specialization that we just created in this case. (If we
* happen to release the write lease after they jump but before they
* get into REQ_RETRANSLATE, they'll acquire it and generate a
* translation possibly for this same situation.)
*/
for (size_t i = 0; i < m_tailFallbackJumps.size(); ++i) {
Asm& as = Asm::Choose(a, astubs, m_tailFallbackJumps[i].m_src);
patch(&as, m_tailFallbackJumps[i], newStart);
}
// This is the new tail translation, so store the fallback jump list
// in case we translate this again.
m_tailFallbackJumps.swap(m_inProgressTailJumps);
m_inProgressTailJumps.clear();
}
