uint32_t
jit::Bailout(BailoutStack *sp, BaselineBailoutInfo **bailoutInfo)
{
JSContext *cx = GetJSContextFromJitCode();
JS_ASSERT(bailoutInfo);
// We don't have an exit frame.
MOZ_ASSERT(IsInRange(FAKE_JIT_TOP_FOR_BAILOUT, 0, 0x1000) &&
IsInRange(FAKE_JIT_TOP_FOR_BAILOUT + sizeof(IonCommonFrameLayout), 0, 0x1000),
"Fake jitTop pointer should be within the first page.");
cx->mainThread().jitTop = FAKE_JIT_TOP_FOR_BAILOUT;
gc::AutoSuppressGC suppress(cx);
JitActivationIterator jitActivations(cx->runtime());
IonBailoutIterator iter(jitActivations, sp);
JitActivation *activation = jitActivations->asJit();
TraceLogger *logger = TraceLoggerForMainThread(cx->runtime());
TraceLogTimestamp(logger, TraceLogger::Bailout);
JitSpew(JitSpew_IonBailouts, "Took bailout! Snapshot offset: %d", iter.snapshotOffset());
JS_ASSERT(IsBaselineEnabled(cx));
*bailoutInfo = nullptr;
uint32_t retval = BailoutIonToBaseline(cx, activation, iter, false, bailoutInfo);
JS_ASSERT(retval == BAILOUT_RETURN_OK ||
retval == BAILOUT_RETURN_FATAL_ERROR ||
retval == BAILOUT_RETURN_OVERRECURSED);
JS_ASSERT_IF(retval == BAILOUT_RETURN_OK, *bailoutInfo != nullptr);
if (retval != BAILOUT_RETURN_OK) {
// If the bailout failed, then bailout trampoline will pop the
// current frame and jump straight to exception handling code when
// this function returns. Any SPS entry pushed for this frame will
// be silently forgotten.
//
// We call ExitScript here to ensure that if the ionScript had SPS
// instrumentation, then the SPS entry for it is popped.
//
// However, if the bailout was during argument check, then a
// pseudostack frame would not have been pushed in the first
// place, so don't pop anything in that case.
bool popSPSFrame = iter.ionScript()->hasSPSInstrumentation() &&
(SnapshotIterator(iter).bailoutKind() != Bailout_ArgumentCheck);
JSScript *script = iter.script();
probes::ExitScript(cx, script, script->functionNonDelazifying(), popSPSFrame);
EnsureExitFrame(iter.jsFrame());
}
return retval;
}
uint32_t
jit::Bailout(BailoutStack* sp, BaselineBailoutInfo** bailoutInfo)
{
JSContext* cx = GetJSContextFromJitCode();
MOZ_ASSERT(bailoutInfo);
// We don't have an exit frame.
MOZ_ASSERT(IsInRange(FAKE_JIT_TOP_FOR_BAILOUT, 0, 0x1000) &&
IsInRange(FAKE_JIT_TOP_FOR_BAILOUT + sizeof(CommonFrameLayout), 0, 0x1000),
"Fake jitTop pointer should be within the first page.");
cx->runtime()->jitTop = FAKE_JIT_TOP_FOR_BAILOUT;
JitActivationIterator jitActivations(cx->runtime());
BailoutFrameInfo bailoutData(jitActivations, sp);
JitFrameIterator iter(jitActivations);
MOZ_ASSERT(!iter.ionScript()->invalidated());
CommonFrameLayout* currentFramePtr = iter.current();
TraceLoggerThread* logger = TraceLoggerForMainThread(cx->runtime());
TraceLogTimestamp(logger, TraceLogger_Bailout);
JitSpew(JitSpew_IonBailouts, "Took bailout! Snapshot offset: %d", iter.snapshotOffset());
MOZ_ASSERT(IsBaselineEnabled(cx));
*bailoutInfo = nullptr;
uint32_t retval = BailoutIonToBaseline(cx, bailoutData.activation(), iter, false, bailoutInfo,
/* excInfo = */ nullptr);
MOZ_ASSERT(retval == BAILOUT_RETURN_OK ||
retval == BAILOUT_RETURN_FATAL_ERROR ||
retval == BAILOUT_RETURN_OVERRECURSED);
MOZ_ASSERT_IF(retval == BAILOUT_RETURN_OK, *bailoutInfo != nullptr);
if (retval != BAILOUT_RETURN_OK) {
JSScript* script = iter.script();
probes::ExitScript(cx, script, script->functionNonDelazifying(),
/* popSPSFrame = */ false);
EnsureExitFrame(iter.jsFrame());
}
// This condition was wrong when we entered this bailout function, but it
// might be true now. A GC might have reclaimed all the Jit code and
// invalidated all frames which are currently on the stack. As we are
// already in a bailout, we could not switch to an invalidation
// bailout. When the code of an IonScript which is on the stack is
// invalidated (see InvalidateActivation), we remove references to it and
// increment the reference counter for each activation that appear on the
// stack. As the bailed frame is one of them, we have to decrement it now.
if (iter.ionScript()->invalidated())
iter.ionScript()->decrementInvalidationCount(cx->runtime()->defaultFreeOp());
// NB: Commentary on how |lastProfilingFrame| is set from bailouts.
//
// Once we return to jitcode, any following frames might get clobbered,
// but the current frame will not (as it will be clobbered "in-place"
// with a baseline frame that will share the same frame prefix).
// However, there may be multiple baseline frames unpacked from this
// single Ion frame, which means we will need to once again reset
// |lastProfilingFrame| to point to the correct unpacked last frame
// in |FinishBailoutToBaseline|.
//
// In the case of error, the jitcode will jump immediately to an
// exception handler, which will unwind the frames and properly set
// the |lastProfilingFrame| to point to the frame being resumed into
// (see |AutoResetLastProfilerFrameOnReturnFromException|).
//
// In both cases, we want to temporarily set the |lastProfilingFrame|
// to the current frame being bailed out, and then fix it up later.
if (cx->runtime()->jitRuntime()->isProfilerInstrumentationEnabled(cx->runtime()))
cx->runtime()->jitActivation->setLastProfilingFrame(currentFramePtr);
return retval;
}
uint32_t
jit::InvalidationBailout(InvalidationBailoutStack* sp, size_t* frameSizeOut,
BaselineBailoutInfo** bailoutInfo)
{
sp->checkInvariants();
JSContext* cx = GetJSContextFromJitCode();
// We don't have an exit frame.
cx->runtime()->jitTop = FAKE_JIT_TOP_FOR_BAILOUT;
JitActivationIterator jitActivations(cx->runtime());
BailoutFrameInfo bailoutData(jitActivations, sp);
JitFrameIterator iter(jitActivations);
CommonFrameLayout* currentFramePtr = iter.current();
TraceLoggerThread* logger = TraceLoggerForMainThread(cx->runtime());
TraceLogTimestamp(logger, TraceLogger_Invalidation);
JitSpew(JitSpew_IonBailouts, "Took invalidation bailout! Snapshot offset: %d", iter.snapshotOffset());
// Note: the frame size must be computed before we return from this function.
*frameSizeOut = iter.frameSize();
MOZ_ASSERT(IsBaselineEnabled(cx));
*bailoutInfo = nullptr;
uint32_t retval = BailoutIonToBaseline(cx, bailoutData.activation(), iter, true, bailoutInfo,
/* excInfo = */ nullptr);
MOZ_ASSERT(retval == BAILOUT_RETURN_OK ||
retval == BAILOUT_RETURN_FATAL_ERROR ||
retval == BAILOUT_RETURN_OVERRECURSED);
MOZ_ASSERT_IF(retval == BAILOUT_RETURN_OK, *bailoutInfo != nullptr);
if (retval != BAILOUT_RETURN_OK) {
// If the bailout failed, then bailout trampoline will pop the
// current frame and jump straight to exception handling code when
// this function returns. Any SPS entry pushed for this frame will
// be silently forgotten.
//
// We call ExitScript here to ensure that if the ionScript had SPS
// instrumentation, then the SPS entry for it is popped.
//
// However, if the bailout was during argument check, then a
// pseudostack frame would not have been pushed in the first
// place, so don't pop anything in that case.
JSScript* script = iter.script();
probes::ExitScript(cx, script, script->functionNonDelazifying(),
/* popSPSFrame = */ false);
JitFrameLayout* frame = iter.jsFrame();
JitSpew(JitSpew_IonInvalidate, "Bailout failed (%s): converting to exit frame",
(retval == BAILOUT_RETURN_FATAL_ERROR) ? "Fatal Error" : "Over Recursion");
JitSpew(JitSpew_IonInvalidate, " orig calleeToken %p", (void*) frame->calleeToken());
JitSpew(JitSpew_IonInvalidate, " orig frameSize %u", unsigned(frame->prevFrameLocalSize()));
JitSpew(JitSpew_IonInvalidate, " orig ra %p", (void*) frame->returnAddress());
frame->replaceCalleeToken(nullptr);
EnsureExitFrame(frame);
JitSpew(JitSpew_IonInvalidate, " new calleeToken %p", (void*) frame->calleeToken());
JitSpew(JitSpew_IonInvalidate, " new frameSize %u", unsigned(frame->prevFrameLocalSize()));
JitSpew(JitSpew_IonInvalidate, " new ra %p", (void*) frame->returnAddress());
}
iter.ionScript()->decrementInvalidationCount(cx->runtime()->defaultFreeOp());
// Make the frame being bailed out the top profiled frame.
if (cx->runtime()->jitRuntime()->isProfilerInstrumentationEnabled(cx->runtime()))
cx->runtime()->jitActivation->setLastProfilingFrame(currentFramePtr);
return retval;
}
static EnterJitStatus JS_HAZ_JSNATIVE_CALLER EnterJit(JSContext* cx,
RunState& state,
uint8_t* code) {
MOZ_ASSERT(state.script()->hasBaselineScript());
MOZ_ASSERT(code);
MOZ_ASSERT(IsBaselineEnabled(cx));
if (!CheckRecursionLimit(cx)) {
return EnterJitStatus::Error;
}
#ifdef DEBUG
// Assert we don't GC before entering JIT code. A GC could discard JIT code
// or move the function stored in the CalleeToken (it won't be traced at
// this point). We use Maybe<> here so we can call reset() to call the
// AutoAssertNoGC destructor before we enter JIT code.
mozilla::Maybe<JS::AutoAssertNoGC> nogc;
nogc.emplace(cx);
#endif
JSScript* script = state.script();
size_t numActualArgs;
bool constructing;
size_t maxArgc;
Value* maxArgv;
JSObject* envChain;
CalleeToken calleeToken;
if (state.isInvoke()) {
const CallArgs& args = state.asInvoke()->args();
numActualArgs = args.length();
if (TooManyActualArguments(numActualArgs)) {
// Too many arguments for Ion. Baseline supports more actual
// arguments, so in that case force Baseline code.
if (numActualArgs > BASELINE_MAX_ARGS_LENGTH) {
return EnterJitStatus::NotEntered;
}
code = script->baselineScript()->method()->raw();
}
constructing = state.asInvoke()->constructing();
maxArgc = args.length() + 1;
maxArgv = args.array() - 1; // -1 to include |this|
envChain = nullptr;
calleeToken = CalleeToToken(&args.callee().as<JSFunction>(), constructing);
unsigned numFormals = script->functionNonDelazifying()->nargs();
if (numFormals > numActualArgs) {
code = cx->runtime()->jitRuntime()->getArgumentsRectifier().value;
}
} else {
numActualArgs = 0;
constructing = false;
if (script->isDirectEvalInFunction()) {
if (state.asExecute()->newTarget().isNull()) {
ScriptFrameIter iter(cx);
state.asExecute()->setNewTarget(iter.newTarget());
}
maxArgc = 1;
maxArgv = state.asExecute()->addressOfNewTarget();
} else {
maxArgc = 0;
maxArgv = nullptr;
}
envChain = state.asExecute()->environmentChain();
calleeToken = CalleeToToken(state.script());
}
// Caller must construct |this| before invoking the function.
MOZ_ASSERT_IF(constructing, maxArgv[0].isObject() ||
maxArgv[0].isMagic(JS_UNINITIALIZED_LEXICAL));
RootedValue result(cx, Int32Value(numActualArgs));
{
AssertRealmUnchanged aru(cx);
ActivationEntryMonitor entryMonitor(cx, calleeToken);
JitActivation activation(cx);
EnterJitCode enter = cx->runtime()->jitRuntime()->enterJit();
#ifdef DEBUG
nogc.reset();
#endif
CALL_GENERATED_CODE(enter, code, maxArgc, maxArgv, /* osrFrame = */ nullptr,
calleeToken, envChain, /* osrNumStackValues = */ 0,
result.address());
}
MOZ_ASSERT(!cx->hasIonReturnOverride());
// Release temporary buffer used for OSR into Ion.
cx->freeOsrTempData();
if (result.isMagic()) {
MOZ_ASSERT(result.isMagic(JS_ION_ERROR));
return EnterJitStatus::Error;
}
// Jit callers wrap primitive constructor return, except for derived
// class constructors, which are forced to do it themselves.
if (constructing && result.isPrimitive()) {
MOZ_ASSERT(maxArgv[0].isObject());
result = maxArgv[0];
}
state.setReturnValue(result);
return EnterJitStatus::Ok;
}
uint32_t
jit::InvalidationBailout(InvalidationBailoutStack *sp, size_t *frameSizeOut,
BaselineBailoutInfo **bailoutInfo)
{
sp->checkInvariants();
JSContext *cx = GetJSContextFromJitCode();
// We don't have an exit frame.
cx->mainThread().jitTop = FAKE_JIT_TOP_FOR_BAILOUT;
gc::AutoSuppressGC suppress(cx);
JitActivationIterator jitActivations(cx->runtime());
IonBailoutIterator iter(jitActivations, sp);
JitActivation *activation = jitActivations->asJit();
TraceLogger *logger = TraceLoggerForMainThread(cx->runtime());
TraceLogTimestamp(logger, TraceLogger::Invalidation);
IonSpew(IonSpew_Bailouts, "Took invalidation bailout! Snapshot offset: %d", iter.snapshotOffset());
// Note: the frame size must be computed before we return from this function.
*frameSizeOut = iter.topFrameSize();
JS_ASSERT(IsBaselineEnabled(cx));
*bailoutInfo = nullptr;
uint32_t retval = BailoutIonToBaseline(cx, activation, iter, true, bailoutInfo);
JS_ASSERT(retval == BAILOUT_RETURN_OK ||
retval == BAILOUT_RETURN_FATAL_ERROR ||
retval == BAILOUT_RETURN_OVERRECURSED);
JS_ASSERT_IF(retval == BAILOUT_RETURN_OK, *bailoutInfo != nullptr);
if (retval != BAILOUT_RETURN_OK) {
// If the bailout failed, then bailout trampoline will pop the
// current frame and jump straight to exception handling code when
// this function returns. Any SPS entry pushed for this frame will
// be silently forgotten.
//
// We call ExitScript here to ensure that if the ionScript had SPS
// instrumentation, then the SPS entry for it is popped.
JSScript *script = iter.script();
probes::ExitScript(cx, script, script->functionNonDelazifying(),
iter.ionScript()->hasSPSInstrumentation());
IonJSFrameLayout *frame = iter.jsFrame();
IonSpew(IonSpew_Invalidate, "Bailout failed (%s): converting to exit frame",
(retval == BAILOUT_RETURN_FATAL_ERROR) ? "Fatal Error" : "Over Recursion");
IonSpew(IonSpew_Invalidate, " orig calleeToken %p", (void *) frame->calleeToken());
IonSpew(IonSpew_Invalidate, " orig frameSize %u", unsigned(frame->prevFrameLocalSize()));
IonSpew(IonSpew_Invalidate, " orig ra %p", (void *) frame->returnAddress());
frame->replaceCalleeToken(nullptr);
EnsureExitFrame(frame);
IonSpew(IonSpew_Invalidate, " new calleeToken %p", (void *) frame->calleeToken());
IonSpew(IonSpew_Invalidate, " new frameSize %u", unsigned(frame->prevFrameLocalSize()));
IonSpew(IonSpew_Invalidate, " new ra %p", (void *) frame->returnAddress());
}
iter.ionScript()->decref(cx->runtime()->defaultFreeOp());
return retval;
}
uint32_t
jit::Bailout(BailoutStack *sp, BaselineBailoutInfo **bailoutInfo)
{
JSContext *cx = GetJSContextFromJitCode();
MOZ_ASSERT(bailoutInfo);
// We don't have an exit frame.
MOZ_ASSERT(IsInRange(FAKE_JIT_TOP_FOR_BAILOUT, 0, 0x1000) &&
IsInRange(FAKE_JIT_TOP_FOR_BAILOUT + sizeof(CommonFrameLayout), 0, 0x1000),
"Fake jitTop pointer should be within the first page.");
cx->mainThread().jitTop = FAKE_JIT_TOP_FOR_BAILOUT;
JitActivationIterator jitActivations(cx->runtime());
BailoutFrameInfo bailoutData(jitActivations, sp);
JitFrameIterator iter(jitActivations);
MOZ_ASSERT(!iter.ionScript()->invalidated());
TraceLogger *logger = TraceLoggerForMainThread(cx->runtime());
TraceLogTimestamp(logger, TraceLogger::Bailout);
JitSpew(JitSpew_IonBailouts, "Took bailout! Snapshot offset: %d", iter.snapshotOffset());
MOZ_ASSERT(IsBaselineEnabled(cx));
*bailoutInfo = nullptr;
bool poppedLastSPSFrame = false;
uint32_t retval = BailoutIonToBaseline(cx, bailoutData.activation(), iter, false, bailoutInfo,
/* excInfo = */ nullptr, &poppedLastSPSFrame);
MOZ_ASSERT(retval == BAILOUT_RETURN_OK ||
retval == BAILOUT_RETURN_FATAL_ERROR ||
retval == BAILOUT_RETURN_OVERRECURSED);
MOZ_ASSERT_IF(retval == BAILOUT_RETURN_OK, *bailoutInfo != nullptr);
if (retval != BAILOUT_RETURN_OK) {
// If the bailout failed, then bailout trampoline will pop the
// current frame and jump straight to exception handling code when
// this function returns. Any SPS entry pushed for this frame will
// be silently forgotten.
//
// We call ExitScript here to ensure that if the ionScript had SPS
// instrumentation, then the SPS entry for it is popped.
//
// However, if the bailout was during argument check, then a
// pseudostack frame would not have been pushed in the first
// place, so don't pop anything in that case.
bool popSPSFrame = iter.ionScript()->hasSPSInstrumentation() &&
(SnapshotIterator(iter).bailoutKind() != Bailout_ArgumentCheck) &&
!poppedLastSPSFrame;
JSScript *script = iter.script();
probes::ExitScript(cx, script, script->functionNonDelazifying(), popSPSFrame);
EnsureExitFrame(iter.jsFrame());
}
// This condition was wrong when we entered this bailout function, but it
// might be true now. A GC might have reclaimed all the Jit code and
// invalidated all frames which are currently on the stack. As we are
// already in a bailout, we could not switch to an invalidation
// bailout. When the code of an IonScript which is on the stack is
// invalidated (see InvalidateActivation), we remove references to it and
// increment the reference counter for each activation that appear on the
// stack. As the bailed frame is one of them, we have to decrement it now.
if (iter.ionScript()->invalidated())
iter.ionScript()->decrementInvalidationCount(cx->runtime()->defaultFreeOp());
return retval;
}