const Vector<GPRReg>& gprsInPriorityOrder()
{
static Vector<GPRReg>* result;
static std::once_flag once;
std::call_once(
once,
[] {
result = new Vector<GPRReg>();
RegisterSet all = RegisterSet::allGPRs();
all.exclude(RegisterSet::stackRegisters());
all.exclude(RegisterSet::reservedHardwareRegisters());
RegisterSet calleeSave = RegisterSet::calleeSaveRegisters();
all.forEach(
[&] (Reg reg) {
if (!calleeSave.get(reg))
result->append(reg.gpr());
});
all.forEach(
[&] (Reg reg) {
if (calleeSave.get(reg))
result->append(reg.gpr());
});
});
return *result;
}
static void registerClobberCheck(AssemblyHelpers& jit, RegisterSet dontClobber)
{
if (!Options::clobberAllRegsInFTLICSlowPath())
return;
RegisterSet clobber = RegisterSet::allRegisters();
clobber.exclude(RegisterSet::reservedHardwareRegisters());
clobber.exclude(RegisterSet::stackRegisters());
clobber.exclude(RegisterSet::calleeSaveRegisters());
clobber.exclude(dontClobber);
GPRReg someGPR;
for (Reg reg = Reg::first(); reg <= Reg::last(); reg = reg.next()) {
if (!clobber.get(reg) || !reg.isGPR())
continue;
jit.move(AssemblyHelpers::TrustedImm32(0x1337beef), reg.gpr());
someGPR = reg.gpr();
}
for (Reg reg = Reg::first(); reg <= Reg::last(); reg = reg.next()) {
if (!clobber.get(reg) || !reg.isFPR())
continue;
jit.move64ToDouble(someGPR, reg.fpr());
}
}
RegisterSet RegisterSet::volatileRegistersForJSCall()
{
RegisterSet volatileRegisters = allRegisters();
volatileRegisters.exclude(RegisterSet::stackRegisters());
volatileRegisters.exclude(RegisterSet::reservedHardwareRegisters());
volatileRegisters.exclude(RegisterSet::vmCalleeSaveRegisters());
return volatileRegisters;
}
SlowPathCallContext::SlowPathCallContext(
RegisterSet usedRegisters, CCallHelpers& jit, unsigned numArgs, GPRReg returnRegister)
: m_jit(jit)
, m_numArgs(numArgs)
, m_returnRegister(returnRegister)
{
// We don't care that you're using callee-save, stack, or hardware registers.
usedRegisters.exclude(RegisterSet::stackRegisters());
usedRegisters.exclude(RegisterSet::reservedHardwareRegisters());
usedRegisters.exclude(RegisterSet::calleeSaveRegisters());
// The return register doesn't need to be saved.
if (m_returnRegister != InvalidGPRReg)
usedRegisters.clear(m_returnRegister);
size_t stackBytesNeededForReturnAddress = wordSize;
m_offsetToSavingArea =
(std::max(m_numArgs, NUMBER_OF_ARGUMENT_REGISTERS) - NUMBER_OF_ARGUMENT_REGISTERS) * wordSize;
for (unsigned i = std::min(NUMBER_OF_ARGUMENT_REGISTERS, numArgs); i--;)
m_argumentRegisters.set(GPRInfo::toArgumentRegister(i));
m_callingConventionRegisters.merge(m_argumentRegisters);
if (returnRegister != InvalidGPRReg)
m_callingConventionRegisters.set(GPRInfo::returnValueGPR);
m_callingConventionRegisters.filter(usedRegisters);
unsigned numberOfCallingConventionRegisters =
m_callingConventionRegisters.numberOfSetRegisters();
size_t offsetToThunkSavingArea =
m_offsetToSavingArea +
numberOfCallingConventionRegisters * wordSize;
m_stackBytesNeeded =
offsetToThunkSavingArea +
stackBytesNeededForReturnAddress +
(usedRegisters.numberOfSetRegisters() - numberOfCallingConventionRegisters) * wordSize;
m_stackBytesNeeded = (m_stackBytesNeeded + stackAlignmentBytes() - 1) & ~(stackAlignmentBytes() - 1);
m_jit.subPtr(CCallHelpers::TrustedImm32(m_stackBytesNeeded), CCallHelpers::stackPointerRegister);
m_thunkSaveSet = usedRegisters;
// This relies on all calling convention registers also being temp registers.
unsigned stackIndex = 0;
for (unsigned i = GPRInfo::numberOfRegisters; i--;) {
GPRReg reg = GPRInfo::toRegister(i);
if (!m_callingConventionRegisters.get(reg))
continue;
m_jit.storePtr(reg, CCallHelpers::Address(CCallHelpers::stackPointerRegister, m_offsetToSavingArea + (stackIndex++) * wordSize));
m_thunkSaveSet.clear(reg);
}
m_offset = offsetToThunkSavingArea;
}
void GetterSetterAccessCase::emitDOMJITGetter(AccessGenerationState& state, const DOMJIT::GetterSetter* domJIT, GPRReg baseForGetGPR)
{
CCallHelpers& jit = *state.jit;
StructureStubInfo& stubInfo = *state.stubInfo;
JSValueRegs valueRegs = state.valueRegs;
GPRReg baseGPR = state.baseGPR;
GPRReg scratchGPR = state.scratchGPR;
// We construct the environment that can execute the DOMJIT::Snippet here.
Ref<DOMJIT::CallDOMGetterSnippet> snippet = domJIT->compiler()();
Vector<GPRReg> gpScratch;
Vector<FPRReg> fpScratch;
Vector<SnippetParams::Value> regs;
ScratchRegisterAllocator allocator(stubInfo.patch.usedRegisters);
allocator.lock(baseGPR);
#if USE(JSVALUE32_64)
allocator.lock(static_cast<GPRReg>(stubInfo.patch.baseTagGPR));
#endif
allocator.lock(valueRegs);
allocator.lock(scratchGPR);
GPRReg paramBaseGPR = InvalidGPRReg;
GPRReg paramGlobalObjectGPR = InvalidGPRReg;
JSValueRegs paramValueRegs = valueRegs;
GPRReg remainingScratchGPR = InvalidGPRReg;
// valueRegs and baseForGetGPR may be the same. For example, in Baseline JIT, we pass the same regT0 for baseGPR and valueRegs.
// In FTL, there is no constraint that the baseForGetGPR interferes with the result. To make implementation simple in
// Snippet, Snippet assumes that result registers always early interfere with input registers, in this case,
// baseForGetGPR. So we move baseForGetGPR to the other register if baseForGetGPR == valueRegs.
if (baseForGetGPR != valueRegs.payloadGPR()) {
paramBaseGPR = baseForGetGPR;
if (!snippet->requireGlobalObject)
remainingScratchGPR = scratchGPR;
else
paramGlobalObjectGPR = scratchGPR;
} else {
jit.move(valueRegs.payloadGPR(), scratchGPR);
paramBaseGPR = scratchGPR;
if (snippet->requireGlobalObject)
paramGlobalObjectGPR = allocator.allocateScratchGPR();
}
JSGlobalObject* globalObjectForDOMJIT = structure()->globalObject();
regs.append(paramValueRegs);
regs.append(paramBaseGPR);
if (snippet->requireGlobalObject) {
ASSERT(paramGlobalObjectGPR != InvalidGPRReg);
regs.append(SnippetParams::Value(paramGlobalObjectGPR, globalObjectForDOMJIT));
}
if (snippet->numGPScratchRegisters) {
unsigned i = 0;
if (remainingScratchGPR != InvalidGPRReg) {
gpScratch.append(remainingScratchGPR);
++i;
}
for (; i < snippet->numGPScratchRegisters; ++i)
gpScratch.append(allocator.allocateScratchGPR());
}
for (unsigned i = 0; i < snippet->numFPScratchRegisters; ++i)
fpScratch.append(allocator.allocateScratchFPR());
// Let's store the reused registers to the stack. After that, we can use allocated scratch registers.
ScratchRegisterAllocator::PreservedState preservedState =
allocator.preserveReusedRegistersByPushing(jit, ScratchRegisterAllocator::ExtraStackSpace::SpaceForCCall);
if (verbose) {
dataLog("baseGPR = ", baseGPR, "\n");
dataLog("valueRegs = ", valueRegs, "\n");
dataLog("scratchGPR = ", scratchGPR, "\n");
dataLog("paramBaseGPR = ", paramBaseGPR, "\n");
if (paramGlobalObjectGPR != InvalidGPRReg)
dataLog("paramGlobalObjectGPR = ", paramGlobalObjectGPR, "\n");
dataLog("paramValueRegs = ", paramValueRegs, "\n");
for (unsigned i = 0; i < snippet->numGPScratchRegisters; ++i)
dataLog("gpScratch[", i, "] = ", gpScratch[i], "\n");
}
if (snippet->requireGlobalObject)
jit.move(CCallHelpers::TrustedImmPtr(globalObjectForDOMJIT), paramGlobalObjectGPR);
// We just spill the registers used in Snippet here. For not spilled registers here explicitly,
// they must be in the used register set passed by the callers (Baseline, DFG, and FTL) if they need to be kept.
// Some registers can be locked, but not in the used register set. For example, the caller could make baseGPR
// same to valueRegs, and not include it in the used registers since it will be changed.
RegisterSet registersToSpillForCCall;
for (auto& value : regs) {
SnippetReg reg = value.reg();
if (reg.isJSValueRegs())
registersToSpillForCCall.set(reg.jsValueRegs());
else if (reg.isGPR())
registersToSpillForCCall.set(reg.gpr());
else
registersToSpillForCCall.set(reg.fpr());
}
for (GPRReg reg : gpScratch)
registersToSpillForCCall.set(reg);
for (FPRReg reg : fpScratch)
registersToSpillForCCall.set(reg);
registersToSpillForCCall.exclude(RegisterSet::registersToNotSaveForCCall());
AccessCaseSnippetParams params(state.m_vm, WTFMove(regs), WTFMove(gpScratch), WTFMove(fpScratch));
snippet->generator()->run(jit, params);
allocator.restoreReusedRegistersByPopping(jit, preservedState);
state.succeed();
CCallHelpers::JumpList exceptions = params.emitSlowPathCalls(state, registersToSpillForCCall, jit);
if (!exceptions.empty()) {
exceptions.link(&jit);
allocator.restoreReusedRegistersByPopping(jit, preservedState);
state.emitExplicitExceptionHandler();
}
}
void fixPartialRegisterStalls(Code& code)
{
if (!isX86())
return;
PhaseScope phaseScope(code, "fixPartialRegisterStalls");
Vector<BasicBlock*> candidates;
for (BasicBlock* block : code) {
for (const Inst& inst : *block) {
if (hasPartialXmmRegUpdate(inst)) {
candidates.append(block);
break;
}
}
}
// Fortunately, Partial Stalls are rarely used. Return early if no block
// cares about them.
if (candidates.isEmpty())
return;
// For each block, this provides the distance to the last instruction setting each register
// on block *entry*.
IndexMap<BasicBlock, FPDefDistance> lastDefDistance(code.size());
// Blocks with dirty distance at head.
IndexSet<BasicBlock> dirty;
// First, we compute the local distance for each block and push it to the successors.
for (BasicBlock* block : code) {
FPDefDistance localDistance;
unsigned distanceToBlockEnd = block->size();
for (Inst& inst : *block)
updateDistances(inst, localDistance, distanceToBlockEnd);
for (BasicBlock* successor : block->successorBlocks()) {
if (lastDefDistance[successor].updateFromPrecessor(localDistance))
dirty.add(successor);
}
}
// Now we propagate the minimums accross blocks.
bool changed;
do {
changed = false;
for (BasicBlock* block : code) {
if (!dirty.remove(block))
continue;
// Little shortcut: if the block is big enough, propagating it won't add any information.
if (block->size() >= minimumSafeDistance)
continue;
unsigned blockSize = block->size();
FPDefDistance& blockDistance = lastDefDistance[block];
for (BasicBlock* successor : block->successorBlocks()) {
if (lastDefDistance[successor].updateFromPrecessor(blockDistance, blockSize)) {
dirty.add(successor);
changed = true;
}
}
}
} while (changed);
// Finally, update each block as needed.
InsertionSet insertionSet(code);
for (BasicBlock* block : candidates) {
unsigned distanceToBlockEnd = block->size();
FPDefDistance& localDistance = lastDefDistance[block];
for (unsigned i = 0; i < block->size(); ++i) {
Inst& inst = block->at(i);
if (hasPartialXmmRegUpdate(inst)) {
RegisterSet defs;
RegisterSet uses;
inst.forEachTmp([&] (Tmp& tmp, Arg::Role role, Arg::Type) {
if (tmp.isFPR()) {
if (Arg::isDef(role))
defs.set(tmp.fpr());
if (Arg::isAnyUse(role))
uses.set(tmp.fpr());
}
});
// We only care about values we define but not use. Otherwise we have to wait
// for the value to be resolved anyway.
defs.exclude(uses);
defs.forEach([&] (Reg reg) {
if (localDistance.distance[MacroAssembler::fpRegisterIndex(reg.fpr())] < minimumSafeDistance)
insertionSet.insert(i, MoveZeroToDouble, inst.origin, Tmp(reg));
});
}
updateDistances(inst, localDistance, distanceToBlockEnd);
}
insertionSet.execute(block);
}
}
RegisterSet ScratchRegisterAllocator::usedRegistersForCall() const
{
RegisterSet result = m_usedRegisters;
result.exclude(RegisterSet::registersToNotSaveForJSCall());
return result;
}
void handleCalleeSaves(Code& code)
{
PhaseScope phaseScope(code, "handleCalleeSaves");
RegisterSet usedCalleeSaves;
for (BasicBlock* block : code) {
for (Inst& inst : *block) {
inst.forEachTmpFast(
[&] (Tmp& tmp) {
// At first we just record all used regs.
usedCalleeSaves.set(tmp.reg());
});
if (inst.hasSpecial())
usedCalleeSaves.merge(inst.extraClobberedRegs());
}
}
// Now we filter to really get the callee saves.
usedCalleeSaves.filter(RegisterSet::calleeSaveRegisters());
usedCalleeSaves.exclude(RegisterSet::stackRegisters()); // We don't need to save FP here.
if (!usedCalleeSaves.numberOfSetRegisters())
return;
code.calleeSaveRegisters() = RegisterAtOffsetList(usedCalleeSaves);
size_t byteSize = 0;
for (const RegisterAtOffset& entry : code.calleeSaveRegisters())
byteSize = std::max(static_cast<size_t>(-entry.offset()), byteSize);
StackSlot* savesArea = code.addStackSlot(byteSize, StackSlotKind::Locked);
// This is a bit weird since we could have already pinned a different stack slot to this
// area. Also, our runtime does not require us to pin the saves area. Maybe we shouldn't pin it?
savesArea->setOffsetFromFP(-byteSize);
auto argFor = [&] (const RegisterAtOffset& entry) -> Arg {
return Arg::stack(savesArea, entry.offset() + byteSize);
};
InsertionSet insertionSet(code);
// First insert saving code in the prologue.
for (const RegisterAtOffset& entry : code.calleeSaveRegisters()) {
insertionSet.insert(
0, entry.reg().isGPR() ? Move : MoveDouble, code[0]->at(0).origin,
Tmp(entry.reg()), argFor(entry));
}
insertionSet.execute(code[0]);
// Now insert restore code at epilogues.
for (BasicBlock* block : code) {
Inst& last = block->last();
if (!isReturn(last.opcode))
continue;
for (const RegisterAtOffset& entry : code.calleeSaveRegisters()) {
insertionSet.insert(
block->size() - 1, entry.reg().isGPR() ? Move : MoveDouble, last.origin,
argFor(entry), Tmp(entry.reg()));
}
insertionSet.execute(block);
}
}
