void
CodeGeneratorX64::emitSimdLoad(LAsmJSLoadHeap *ins)
{
MAsmJSLoadHeap *mir = ins->mir();
Scalar::Type type = mir->accessType();
const LAllocation *ptr = ins->ptr();
FloatRegister out = ToFloatRegister(ins->output());
Operand srcAddr(HeapReg);
if (ptr->isConstant()) {
int32_t ptrImm = ptr->toConstant()->toInt32();
MOZ_ASSERT(ptrImm >= 0);
srcAddr = Operand(HeapReg, ptrImm);
} else {
srcAddr = Operand(HeapReg, ToRegister(ptr), TimesOne);
}
uint32_t maybeCmpOffset = AsmJSHeapAccess::NoLengthCheck;
if (mir->needsBoundsCheck()) {
maybeCmpOffset = masm.cmp32WithPatch(ToRegister(ptr), Imm32(0)).offset();
masm.j(Assembler::AboveOrEqual, mir->outOfBoundsLabel()); // Throws RangeError
}
unsigned numElems = mir->numSimdElems();
if (numElems == 3) {
MOZ_ASSERT(type == Scalar::Int32x4 || type == Scalar::Float32x4);
Operand shiftedOffset(HeapReg);
if (ptr->isConstant())
shiftedOffset = Operand(HeapReg, ptr->toConstant()->toInt32() + 2 * sizeof(float));
else
shiftedOffset = Operand(HeapReg, ToRegister(ptr), TimesOne, 2 * sizeof(float));
// Load XY
uint32_t before = masm.size();
loadSimd(type, 2, srcAddr, out);
uint32_t after = masm.size();
// We're noting a load of 3 elements, so that the bounds check checks
// for 3 elements.
masm.append(AsmJSHeapAccess(before, after, 3, type, maybeCmpOffset));
// Load Z (W is zeroed)
before = after;
loadSimd(type, 1, shiftedOffset, ScratchSimdReg);
after = masm.size();
masm.append(AsmJSHeapAccess(before, after, 1, type));
// Move ZW atop XY
masm.vmovlhps(ScratchSimdReg, out, out);
return;
}
uint32_t before = masm.size();
loadSimd(type, numElems, srcAddr, out);
uint32_t after = masm.size();
masm.append(AsmJSHeapAccess(before, after, numElems, type, maybeCmpOffset));
}
void
CodeGeneratorX64::visitAsmJSLoadHeap(LAsmJSLoadHeap *ins)
{
MAsmJSLoadHeap *mir = ins->mir();
Scalar::Type vt = mir->accessType();
const LAllocation *ptr = ins->ptr();
const LDefinition *out = ins->output();
Operand srcAddr(HeapReg);
if (Scalar::isSimdType(vt))
return emitSimdLoad(ins);
if (ptr->isConstant()) {
int32_t ptrImm = ptr->toConstant()->toInt32();
MOZ_ASSERT(ptrImm >= 0);
srcAddr = Operand(HeapReg, ptrImm);
} else {
srcAddr = Operand(HeapReg, ToRegister(ptr), TimesOne);
}
memoryBarrier(ins->mir()->barrierBefore());
OutOfLineLoadTypedArrayOutOfBounds *ool = nullptr;
uint32_t maybeCmpOffset = AsmJSHeapAccess::NoLengthCheck;
if (mir->needsBoundsCheck()) {
CodeOffsetLabel cmp = masm.cmp32WithPatch(ToRegister(ptr), Imm32(0));
ool = new(alloc()) OutOfLineLoadTypedArrayOutOfBounds(ToAnyRegister(out), vt);
addOutOfLineCode(ool, ins->mir());
masm.j(Assembler::AboveOrEqual, ool->entry());
maybeCmpOffset = cmp.offset();
}
uint32_t before = masm.size();
switch (vt) {
case Scalar::Int8: masm.movsbl(srcAddr, ToRegister(out)); break;
case Scalar::Uint8: masm.movzbl(srcAddr, ToRegister(out)); break;
case Scalar::Int16: masm.movswl(srcAddr, ToRegister(out)); break;
case Scalar::Uint16: masm.movzwl(srcAddr, ToRegister(out)); break;
case Scalar::Int32:
case Scalar::Uint32: masm.movl(srcAddr, ToRegister(out)); break;
case Scalar::Float32: masm.loadFloat32(srcAddr, ToFloatRegister(out)); break;
case Scalar::Float64: masm.loadDouble(srcAddr, ToFloatRegister(out)); break;
case Scalar::Float32x4:
case Scalar::Int32x4: MOZ_CRASH("SIMD loads should be handled in emitSimdLoad");
case Scalar::Uint8Clamped:
case Scalar::MaxTypedArrayViewType:
MOZ_CRASH("unexpected array type");
}
uint32_t after = masm.size();
verifyHeapAccessDisassembly(before, after, /*isLoad=*/true, vt, srcAddr, *out->output());
if (ool)
masm.bind(ool->rejoin());
memoryBarrier(ins->mir()->barrierAfter());
masm.append(AsmJSHeapAccess(before, after, vt, ToAnyRegister(out), maybeCmpOffset));
}
bool
CodeGeneratorX64::visitAsmJSLoadHeap(LAsmJSLoadHeap *ins)
{
MAsmJSLoadHeap *mir = ins->mir();
AsmJSHeapAccess::ViewType vt = mir->viewType();
const LAllocation *ptr = ins->ptr();
const LDefinition *out = ins->output();
Operand srcAddr(HeapReg);
if (ptr->isConstant()) {
int32_t ptrImm = ptr->toConstant()->toInt32();
MOZ_ASSERT(ptrImm >= 0);
srcAddr = Operand(HeapReg, ptrImm);
} else {
srcAddr = Operand(HeapReg, ToRegister(ptr), TimesOne);
}
memoryBarrier(ins->mir()->barrierBefore());
OutOfLineLoadTypedArrayOutOfBounds *ool = nullptr;
uint32_t maybeCmpOffset = AsmJSHeapAccess::NoLengthCheck;
if (mir->needsBoundsCheck()) {
ool = new(alloc()) OutOfLineLoadTypedArrayOutOfBounds(ToAnyRegister(out), vt);
if (!addOutOfLineCode(ool, ins->mir()))
return false;
CodeOffsetLabel cmp = masm.cmplWithPatch(ToRegister(ptr), Imm32(0));
masm.j(Assembler::AboveOrEqual, ool->entry());
maybeCmpOffset = cmp.offset();
}
uint32_t before = masm.size();
switch (vt) {
case AsmJSHeapAccess::Int8: masm.movsbl(srcAddr, ToRegister(out)); break;
case AsmJSHeapAccess::Uint8: masm.movzbl(srcAddr, ToRegister(out)); break;
case AsmJSHeapAccess::Int16: masm.movswl(srcAddr, ToRegister(out)); break;
case AsmJSHeapAccess::Uint16: masm.movzwl(srcAddr, ToRegister(out)); break;
case AsmJSHeapAccess::Int32:
case AsmJSHeapAccess::Uint32: masm.movl(srcAddr, ToRegister(out)); break;
case AsmJSHeapAccess::Float32: masm.loadFloat32(srcAddr, ToFloatRegister(out)); break;
case AsmJSHeapAccess::Float64: masm.loadDouble(srcAddr, ToFloatRegister(out)); break;
case AsmJSHeapAccess::Float32x4: masm.loadUnalignedFloat32x4(srcAddr, ToFloatRegister(out)); break;
case AsmJSHeapAccess::Int32x4: masm.loadUnalignedInt32x4(srcAddr, ToFloatRegister(out)); break;
case AsmJSHeapAccess::Uint8Clamped: MOZ_CRASH("unexpected array type");
}
uint32_t after = masm.size();
if (ool)
masm.bind(ool->rejoin());
memoryBarrier(ins->mir()->barrierAfter());
masm.append(AsmJSHeapAccess(before, after, vt, ToAnyRegister(out), maybeCmpOffset));
return true;
}