SDNode *AMDGPUDAGToDAGISel::Select(SDNode *N) {
const R600InstrInfo *TII =
static_cast<const R600InstrInfo*>(TM.getInstrInfo());
unsigned int Opc = N->getOpcode();
if (N->isMachineOpcode()) {
return NULL; // Already selected.
}
switch (Opc) {
default: break;
case AMDGPUISD::CONST_ADDRESS: {
for (SDNode::use_iterator I = N->use_begin(), Next = llvm::next(I);
I != SDNode::use_end(); I = Next) {
Next = llvm::next(I);
if (!I->isMachineOpcode()) {
continue;
}
unsigned Opcode = I->getMachineOpcode();
bool HasDst = TII->getOperandIdx(Opcode, AMDGPU::OpName::dst) > -1;
int SrcIdx = I.getOperandNo();
int SelIdx;
// Unlike MachineInstrs, SDNodes do not have results in their operand
// list, so we need to increment the SrcIdx, since
// R600InstrInfo::getOperandIdx is based on the MachineInstr indices.
if (HasDst) {
SrcIdx++;
}
SelIdx = TII->getSelIdx(I->getMachineOpcode(), SrcIdx);
if (SelIdx < 0) {
continue;
}
SDValue CstOffset;
if (N->getValueType(0).isVector() ||
!SelectGlobalValueConstantOffset(N->getOperand(0), CstOffset))
continue;
// Gather constants values
int SrcIndices[] = {
TII->getOperandIdx(Opcode, AMDGPU::OpName::src0),
TII->getOperandIdx(Opcode, AMDGPU::OpName::src1),
TII->getOperandIdx(Opcode, AMDGPU::OpName::src2),
TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_X),
TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_Y),
TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_Z),
TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_W),
TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_X),
TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_Y),
TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_Z),
TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_W)
};
std::vector<unsigned> Consts;
for (unsigned i = 0; i < sizeof(SrcIndices) / sizeof(int); i++) {
int OtherSrcIdx = SrcIndices[i];
int OtherSelIdx = TII->getSelIdx(Opcode, OtherSrcIdx);
if (OtherSrcIdx < 0 || OtherSelIdx < 0) {
continue;
}
if (HasDst) {
OtherSrcIdx--;
OtherSelIdx--;
}
if (RegisterSDNode *Reg =
dyn_cast<RegisterSDNode>(I->getOperand(OtherSrcIdx))) {
if (Reg->getReg() == AMDGPU::ALU_CONST) {
ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(I->getOperand(OtherSelIdx));
Consts.push_back(Cst->getZExtValue());
}
}
}
ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(CstOffset);
Consts.push_back(Cst->getZExtValue());
if (!TII->fitsConstReadLimitations(Consts))
continue;
// Convert back to SDNode indices
if (HasDst) {
SrcIdx--;
SelIdx--;
}
std::vector<SDValue> Ops;
for (int i = 0, e = I->getNumOperands(); i != e; ++i) {
if (i == SrcIdx) {
Ops.push_back(CurDAG->getRegister(AMDGPU::ALU_CONST, MVT::f32));
} else if (i == SelIdx) {
Ops.push_back(CstOffset);
} else {
Ops.push_back(I->getOperand(i));
}
}
CurDAG->UpdateNodeOperands(*I, Ops.data(), Ops.size());
}
break;
}
case ISD::BUILD_VECTOR: {
const AMDGPUSubtarget &ST = TM.getSubtarget<AMDGPUSubtarget>();
if (ST.getGeneration() > AMDGPUSubtarget::NORTHERN_ISLANDS) {
break;
}
unsigned RegClassID;
switch(N->getValueType(0).getVectorNumElements()) {
case 2: RegClassID = AMDGPU::R600_Reg64RegClassID; break;
case 4: RegClassID = AMDGPU::R600_Reg128RegClassID; break;
default: llvm_unreachable("Do not know how to lower this BUILD_VECTOR");
}
// BUILD_VECTOR is usually lowered into an IMPLICIT_DEF + 4 INSERT_SUBREG
// that adds a 128 bits reg copy when going through TwoAddressInstructions
// pass. We want to avoid 128 bits copies as much as possible because they
// can't be bundled by our scheduler.
SDValue RegSeqArgs[9] = {
CurDAG->getTargetConstant(RegClassID, MVT::i32),
SDValue(), CurDAG->getTargetConstant(AMDGPU::sub0, MVT::i32),
SDValue(), CurDAG->getTargetConstant(AMDGPU::sub1, MVT::i32),
SDValue(), CurDAG->getTargetConstant(AMDGPU::sub2, MVT::i32),
SDValue(), CurDAG->getTargetConstant(AMDGPU::sub3, MVT::i32)
};
bool IsRegSeq = true;
for (unsigned i = 0; i < N->getNumOperands(); i++) {
if (dyn_cast<RegisterSDNode>(N->getOperand(i))) {
IsRegSeq = false;
break;
}
RegSeqArgs[2 * i + 1] = N->getOperand(i);
}
if (!IsRegSeq)
break;
return CurDAG->SelectNodeTo(N, AMDGPU::REG_SEQUENCE, N->getVTList(),
RegSeqArgs, 2 * N->getNumOperands() + 1);
}
case ISD::BUILD_PAIR: {
SDValue RC, SubReg0, SubReg1;
const AMDGPUSubtarget &ST = TM.getSubtarget<AMDGPUSubtarget>();
if (ST.getGeneration() <= AMDGPUSubtarget::NORTHERN_ISLANDS) {
break;
}
if (N->getValueType(0) == MVT::i128) {
RC = CurDAG->getTargetConstant(AMDGPU::SReg_128RegClassID, MVT::i32);
SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0_sub1, MVT::i32);
SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub2_sub3, MVT::i32);
} else if (N->getValueType(0) == MVT::i64) {
RC = CurDAG->getTargetConstant(AMDGPU::VSrc_64RegClassID, MVT::i32);
SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0, MVT::i32);
SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub1, MVT::i32);
} else {
llvm_unreachable("Unhandled value type for BUILD_PAIR");
}
const SDValue Ops[] = { RC, N->getOperand(0), SubReg0,
N->getOperand(1), SubReg1 };
return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE,
SDLoc(N), N->getValueType(0), Ops);
}
case ISD::ConstantFP:
case ISD::Constant: {
const AMDGPUSubtarget &ST = TM.getSubtarget<AMDGPUSubtarget>();
// XXX: Custom immediate lowering not implemented yet. Instead we use
// pseudo instructions defined in SIInstructions.td
if (ST.getGeneration() > AMDGPUSubtarget::NORTHERN_ISLANDS) {
break;
}
uint64_t ImmValue = 0;
unsigned ImmReg = AMDGPU::ALU_LITERAL_X;
if (N->getOpcode() == ISD::ConstantFP) {
// XXX: 64-bit Immediates not supported yet
assert(N->getValueType(0) != MVT::f64);
ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N);
APFloat Value = C->getValueAPF();
float FloatValue = Value.convertToFloat();
if (FloatValue == 0.0) {
ImmReg = AMDGPU::ZERO;
} else if (FloatValue == 0.5) {
ImmReg = AMDGPU::HALF;
} else if (FloatValue == 1.0) {
ImmReg = AMDGPU::ONE;
} else {
ImmValue = Value.bitcastToAPInt().getZExtValue();
}
} else {
// XXX: 64-bit Immediates not supported yet
assert(N->getValueType(0) != MVT::i64);
ConstantSDNode *C = dyn_cast<ConstantSDNode>(N);
if (C->getZExtValue() == 0) {
ImmReg = AMDGPU::ZERO;
} else if (C->getZExtValue() == 1) {
ImmReg = AMDGPU::ONE_INT;
} else {
ImmValue = C->getZExtValue();
}
}
for (SDNode::use_iterator Use = N->use_begin(), Next = llvm::next(Use);
Use != SDNode::use_end(); Use = Next) {
Next = llvm::next(Use);
std::vector<SDValue> Ops;
for (unsigned i = 0; i < Use->getNumOperands(); ++i) {
Ops.push_back(Use->getOperand(i));
}
if (!Use->isMachineOpcode()) {
if (ImmReg == AMDGPU::ALU_LITERAL_X) {
// We can only use literal constants (e.g. AMDGPU::ZERO,
// AMDGPU::ONE, etc) in machine opcodes.
continue;
}
} else {
if (!TII->isALUInstr(Use->getMachineOpcode()) ||
(TII->get(Use->getMachineOpcode()).TSFlags &
R600_InstFlag::VECTOR)) {
continue;
}
int ImmIdx = TII->getOperandIdx(Use->getMachineOpcode(),
AMDGPU::OpName::literal);
if (ImmIdx == -1) {
continue;
}
if (TII->getOperandIdx(Use->getMachineOpcode(),
AMDGPU::OpName::dst) != -1) {
// subtract one from ImmIdx, because the DST operand is usually index
// 0 for MachineInstrs, but we have no DST in the Ops vector.
ImmIdx--;
}
// Check that we aren't already using an immediate.
// XXX: It's possible for an instruction to have more than one
// immediate operand, but this is not supported yet.
if (ImmReg == AMDGPU::ALU_LITERAL_X) {
ConstantSDNode *C = dyn_cast<ConstantSDNode>(Use->getOperand(ImmIdx));
assert(C);
if (C->getZExtValue() != 0) {
// This instruction is already using an immediate.
continue;
}
// Set the immediate value
Ops[ImmIdx] = CurDAG->getTargetConstant(ImmValue, MVT::i32);
}
}
// Set the immediate register
Ops[Use.getOperandNo()] = CurDAG->getRegister(ImmReg, MVT::i32);
CurDAG->UpdateNodeOperands(*Use, Ops.data(), Use->getNumOperands());
}
break;
}
}
SDNode *Result = SelectCode(N);
// Fold operands of selected node
const AMDGPUSubtarget &ST = TM.getSubtarget<AMDGPUSubtarget>();
if (ST.getGeneration() <= AMDGPUSubtarget::NORTHERN_ISLANDS) {
const R600InstrInfo *TII =
static_cast<const R600InstrInfo*>(TM.getInstrInfo());
if (Result && Result->isMachineOpcode() && Result->getMachineOpcode() == AMDGPU::DOT_4) {
bool IsModified = false;
do {
std::vector<SDValue> Ops;
for(SDNode::op_iterator I = Result->op_begin(), E = Result->op_end();
I != E; ++I)
Ops.push_back(*I);
IsModified = FoldDotOperands(Result->getMachineOpcode(), TII, Ops);
if (IsModified) {
Result = CurDAG->UpdateNodeOperands(Result, Ops.data(), Ops.size());
}
} while (IsModified);
}
if (Result && Result->isMachineOpcode() &&
!(TII->get(Result->getMachineOpcode()).TSFlags & R600_InstFlag::VECTOR)
&& TII->hasInstrModifiers(Result->getMachineOpcode())) {
// Fold FNEG/FABS
// TODO: Isel can generate multiple MachineInst, we need to recursively
// parse Result
bool IsModified = false;
do {
std::vector<SDValue> Ops;
for(SDNode::op_iterator I = Result->op_begin(), E = Result->op_end();
I != E; ++I)
Ops.push_back(*I);
IsModified = FoldOperands(Result->getMachineOpcode(), TII, Ops);
if (IsModified) {
Result = CurDAG->UpdateNodeOperands(Result, Ops.data(), Ops.size());
}
} while (IsModified);
// If node has a single use which is CLAMP_R600, folds it
if (Result->hasOneUse() && Result->isMachineOpcode()) {
SDNode *PotentialClamp = *Result->use_begin();
if (PotentialClamp->isMachineOpcode() &&
PotentialClamp->getMachineOpcode() == AMDGPU::CLAMP_R600) {
unsigned ClampIdx =
TII->getOperandIdx(Result->getMachineOpcode(), AMDGPU::OpName::clamp);
std::vector<SDValue> Ops;
unsigned NumOp = Result->getNumOperands();
for (unsigned i = 0; i < NumOp; ++i) {
Ops.push_back(Result->getOperand(i));
}
Ops[ClampIdx - 1] = CurDAG->getTargetConstant(1, MVT::i32);
Result = CurDAG->SelectNodeTo(PotentialClamp,
Result->getMachineOpcode(), PotentialClamp->getVTList(),
Ops.data(), NumOp);
}
}
}
}
return Result;
}
SDNode *AMDGPUDAGToDAGISel::Select(SDNode *N) {
unsigned int Opc = N->getOpcode();
if (N->isMachineOpcode()) {
return NULL; // Already selected.
}
switch (Opc) {
default: break;
case ISD::BUILD_VECTOR: {
const AMDGPUSubtarget &ST = TM.getSubtarget<AMDGPUSubtarget>();
if (ST.device()->getGeneration() > AMDGPUDeviceInfo::HD6XXX) {
break;
}
// BUILD_VECTOR is usually lowered into an IMPLICIT_DEF + 4 INSERT_SUBREG
// that adds a 128 bits reg copy when going through TwoAddressInstructions
// pass. We want to avoid 128 bits copies as much as possible because they
// can't be bundled by our scheduler.
SDValue RegSeqArgs[9] = {
CurDAG->getTargetConstant(AMDGPU::R600_Reg128RegClassID, MVT::i32),
SDValue(), CurDAG->getTargetConstant(AMDGPU::sub0, MVT::i32),
SDValue(), CurDAG->getTargetConstant(AMDGPU::sub1, MVT::i32),
SDValue(), CurDAG->getTargetConstant(AMDGPU::sub2, MVT::i32),
SDValue(), CurDAG->getTargetConstant(AMDGPU::sub3, MVT::i32)
};
bool IsRegSeq = true;
for (unsigned i = 0; i < N->getNumOperands(); i++) {
if (dyn_cast<RegisterSDNode>(N->getOperand(i))) {
IsRegSeq = false;
break;
}
RegSeqArgs[2 * i + 1] = N->getOperand(i);
}
if (!IsRegSeq)
break;
return CurDAG->SelectNodeTo(N, AMDGPU::REG_SEQUENCE, N->getVTList(),
RegSeqArgs, 2 * N->getNumOperands() + 1);
}
case ISD::ConstantFP:
case ISD::Constant: {
const AMDGPUSubtarget &ST = TM.getSubtarget<AMDGPUSubtarget>();
// XXX: Custom immediate lowering not implemented yet. Instead we use
// pseudo instructions defined in SIInstructions.td
if (ST.device()->getGeneration() > AMDGPUDeviceInfo::HD6XXX) {
break;
}
const R600InstrInfo *TII = static_cast<const R600InstrInfo*>(TM.getInstrInfo());
uint64_t ImmValue = 0;
unsigned ImmReg = AMDGPU::ALU_LITERAL_X;
if (N->getOpcode() == ISD::ConstantFP) {
// XXX: 64-bit Immediates not supported yet
assert(N->getValueType(0) != MVT::f64);
ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N);
APFloat Value = C->getValueAPF();
float FloatValue = Value.convertToFloat();
if (FloatValue == 0.0) {
ImmReg = AMDGPU::ZERO;
} else if (FloatValue == 0.5) {
ImmReg = AMDGPU::HALF;
} else if (FloatValue == 1.0) {
ImmReg = AMDGPU::ONE;
} else {
ImmValue = Value.bitcastToAPInt().getZExtValue();
}
} else {
// XXX: 64-bit Immediates not supported yet
assert(N->getValueType(0) != MVT::i64);
ConstantSDNode *C = dyn_cast<ConstantSDNode>(N);
if (C->getZExtValue() == 0) {
ImmReg = AMDGPU::ZERO;
} else if (C->getZExtValue() == 1) {
ImmReg = AMDGPU::ONE_INT;
} else {
ImmValue = C->getZExtValue();
}
}
for (SDNode::use_iterator Use = N->use_begin(), Next = llvm::next(Use);
Use != SDNode::use_end(); Use = Next) {
Next = llvm::next(Use);
std::vector<SDValue> Ops;
for (unsigned i = 0; i < Use->getNumOperands(); ++i) {
Ops.push_back(Use->getOperand(i));
}
if (!Use->isMachineOpcode()) {
if (ImmReg == AMDGPU::ALU_LITERAL_X) {
// We can only use literal constants (e.g. AMDGPU::ZERO,
// AMDGPU::ONE, etc) in machine opcodes.
continue;
}
} else {
if (!TII->isALUInstr(Use->getMachineOpcode()) ||
(TII->get(Use->getMachineOpcode()).TSFlags &
R600_InstFlag::VECTOR)) {
continue;
}
int ImmIdx = TII->getOperandIdx(Use->getMachineOpcode(), R600Operands::IMM);
assert(ImmIdx != -1);
// subtract one from ImmIdx, because the DST operand is usually index
// 0 for MachineInstrs, but we have no DST in the Ops vector.
ImmIdx--;
// Check that we aren't already using an immediate.
// XXX: It's possible for an instruction to have more than one
// immediate operand, but this is not supported yet.
if (ImmReg == AMDGPU::ALU_LITERAL_X) {
ConstantSDNode *C = dyn_cast<ConstantSDNode>(Use->getOperand(ImmIdx));
assert(C);
if (C->getZExtValue() != 0) {
// This instruction is already using an immediate.
continue;
}
// Set the immediate value
Ops[ImmIdx] = CurDAG->getTargetConstant(ImmValue, MVT::i32);
}
}
// Set the immediate register
Ops[Use.getOperandNo()] = CurDAG->getRegister(ImmReg, MVT::i32);
CurDAG->UpdateNodeOperands(*Use, Ops.data(), Use->getNumOperands());
}
break;
}
}
SDNode *Result = SelectCode(N);
// Fold operands of selected node
const AMDGPUSubtarget &ST = TM.getSubtarget<AMDGPUSubtarget>();
if (ST.device()->getGeneration() <= AMDGPUDeviceInfo::HD6XXX) {
const R600InstrInfo *TII =
static_cast<const R600InstrInfo*>(TM.getInstrInfo());
if (Result && Result->isMachineOpcode() &&
!(TII->get(Result->getMachineOpcode()).TSFlags & R600_InstFlag::VECTOR)
&& TII->isALUInstr(Result->getMachineOpcode())) {
// Fold FNEG/FABS/CONST_ADDRESS
// TODO: Isel can generate multiple MachineInst, we need to recursively
// parse Result
bool IsModified = false;
do {
std::vector<SDValue> Ops;
for(SDNode::op_iterator I = Result->op_begin(), E = Result->op_end();
I != E; ++I)
Ops.push_back(*I);
IsModified = FoldOperands(Result->getMachineOpcode(), TII, Ops);
if (IsModified) {
Result = CurDAG->UpdateNodeOperands(Result, Ops.data(), Ops.size());
}
} while (IsModified);
// If node has a single use which is CLAMP_R600, folds it
if (Result->hasOneUse() && Result->isMachineOpcode()) {
SDNode *PotentialClamp = *Result->use_begin();
if (PotentialClamp->isMachineOpcode() &&
PotentialClamp->getMachineOpcode() == AMDGPU::CLAMP_R600) {
unsigned ClampIdx =
TII->getOperandIdx(Result->getMachineOpcode(), R600Operands::CLAMP);
std::vector<SDValue> Ops;
unsigned NumOp = Result->getNumOperands();
for (unsigned i = 0; i < NumOp; ++i) {
Ops.push_back(Result->getOperand(i));
}
Ops[ClampIdx - 1] = CurDAG->getTargetConstant(1, MVT::i32);
Result = CurDAG->SelectNodeTo(PotentialClamp,
Result->getMachineOpcode(), PotentialClamp->getVTList(),
Ops.data(), NumOp);
}
}
}
}
return Result;
}
void SelectionDAGBuilder::LowerStatepoint(
ImmutableStatepoint ISP, MachineBasicBlock *LandingPad /*=nullptr*/) {
// The basic scheme here is that information about both the original call and
// the safepoint is encoded in the CallInst. We create a temporary call and
// lower it, then reverse engineer the calling sequence.
NumOfStatepoints++;
// Clear state
StatepointLowering.startNewStatepoint(*this);
ImmutableCallSite CS(ISP.getCallSite());
#ifndef NDEBUG
// Consistency check. Don't do this for invokes. It would be too
// expensive to preserve this information across different basic blocks
if (!CS.isInvoke()) {
for (const User *U : CS->users()) {
const CallInst *Call = cast<CallInst>(U);
if (isGCRelocate(Call))
StatepointLowering.scheduleRelocCall(*Call);
}
}
#endif
#ifndef NDEBUG
// If this is a malformed statepoint, report it early to simplify debugging.
// This should catch any IR level mistake that's made when constructing or
// transforming statepoints.
ISP.verify();
// Check that the associated GCStrategy expects to encounter statepoints.
assert(GFI->getStrategy().useStatepoints() &&
"GCStrategy does not expect to encounter statepoints");
#endif
// Lower statepoint vmstate and gcstate arguments
SmallVector<SDValue, 10> LoweredMetaArgs;
lowerStatepointMetaArgs(LoweredMetaArgs, ISP, *this);
// Get call node, we will replace it later with statepoint
SDNode *CallNode =
lowerCallFromStatepoint(ISP, LandingPad, *this, PendingExports);
// Construct the actual GC_TRANSITION_START, STATEPOINT, and GC_TRANSITION_END
// nodes with all the appropriate arguments and return values.
// Call Node: Chain, Target, {Args}, RegMask, [Glue]
SDValue Chain = CallNode->getOperand(0);
SDValue Glue;
bool CallHasIncomingGlue = CallNode->getGluedNode();
if (CallHasIncomingGlue) {
// Glue is always last operand
Glue = CallNode->getOperand(CallNode->getNumOperands() - 1);
}
// Build the GC_TRANSITION_START node if necessary.
//
// The operands to the GC_TRANSITION_{START,END} nodes are laid out in the
// order in which they appear in the call to the statepoint intrinsic. If
// any of the operands is a pointer-typed, that operand is immediately
// followed by a SRCVALUE for the pointer that may be used during lowering
// (e.g. to form MachinePointerInfo values for loads/stores).
const bool IsGCTransition =
(ISP.getFlags() & (uint64_t)StatepointFlags::GCTransition) ==
(uint64_t)StatepointFlags::GCTransition;
if (IsGCTransition) {
SmallVector<SDValue, 8> TSOps;
// Add chain
TSOps.push_back(Chain);
// Add GC transition arguments
for (const Value *V : ISP.gc_transition_args()) {
TSOps.push_back(getValue(V));
if (V->getType()->isPointerTy())
TSOps.push_back(DAG.getSrcValue(V));
}
// Add glue if necessary
if (CallHasIncomingGlue)
TSOps.push_back(Glue);
SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
SDValue GCTransitionStart =
DAG.getNode(ISD::GC_TRANSITION_START, getCurSDLoc(), NodeTys, TSOps);
Chain = GCTransitionStart.getValue(0);
Glue = GCTransitionStart.getValue(1);
}
// TODO: Currently, all of these operands are being marked as read/write in
// PrologEpilougeInserter.cpp, we should special case the VMState arguments
// and flags to be read-only.
SmallVector<SDValue, 40> Ops;
// Add the <id> and <numBytes> constants.
Ops.push_back(DAG.getTargetConstant(ISP.getID(), getCurSDLoc(), MVT::i64));
Ops.push_back(
DAG.getTargetConstant(ISP.getNumPatchBytes(), getCurSDLoc(), MVT::i32));
// Calculate and push starting position of vmstate arguments
// Get number of arguments incoming directly into call node
unsigned NumCallRegArgs =
CallNode->getNumOperands() - (CallHasIncomingGlue ? 4 : 3);
Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, getCurSDLoc(), MVT::i32));
// Add call target
SDValue CallTarget = SDValue(CallNode->getOperand(1).getNode(), 0);
Ops.push_back(CallTarget);
// Add call arguments
// Get position of register mask in the call
SDNode::op_iterator RegMaskIt;
if (CallHasIncomingGlue)
RegMaskIt = CallNode->op_end() - 2;
else
RegMaskIt = CallNode->op_end() - 1;
Ops.insert(Ops.end(), CallNode->op_begin() + 2, RegMaskIt);
// Add a constant argument for the calling convention
pushStackMapConstant(Ops, *this, CS.getCallingConv());
// Add a constant argument for the flags
uint64_t Flags = ISP.getFlags();
assert(
((Flags & ~(uint64_t)StatepointFlags::MaskAll) == 0)
&& "unknown flag used");
pushStackMapConstant(Ops, *this, Flags);
// Insert all vmstate and gcstate arguments
Ops.insert(Ops.end(), LoweredMetaArgs.begin(), LoweredMetaArgs.end());
// Add register mask from call node
Ops.push_back(*RegMaskIt);
// Add chain
Ops.push_back(Chain);
// Same for the glue, but we add it only if original call had it
if (Glue.getNode())
Ops.push_back(Glue);
// Compute return values. Provide a glue output since we consume one as
// input. This allows someone else to chain off us as needed.
SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
SDNode *StatepointMCNode =
DAG.getMachineNode(TargetOpcode::STATEPOINT, getCurSDLoc(), NodeTys, Ops);
SDNode *SinkNode = StatepointMCNode;
// Build the GC_TRANSITION_END node if necessary.
//
// See the comment above regarding GC_TRANSITION_START for the layout of
// the operands to the GC_TRANSITION_END node.
if (IsGCTransition) {
SmallVector<SDValue, 8> TEOps;
// Add chain
TEOps.push_back(SDValue(StatepointMCNode, 0));
// Add GC transition arguments
for (const Value *V : ISP.gc_transition_args()) {
TEOps.push_back(getValue(V));
if (V->getType()->isPointerTy())
TEOps.push_back(DAG.getSrcValue(V));
}
// Add glue
TEOps.push_back(SDValue(StatepointMCNode, 1));
SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
SDValue GCTransitionStart =
DAG.getNode(ISD::GC_TRANSITION_END, getCurSDLoc(), NodeTys, TEOps);
SinkNode = GCTransitionStart.getNode();
}
// Replace original call
DAG.ReplaceAllUsesWith(CallNode, SinkNode); // This may update Root
// Remove originall call node
DAG.DeleteNode(CallNode);
// DON'T set the root - under the assumption that it's already set past the
// inserted node we created.
// TODO: A better future implementation would be to emit a single variable
// argument, variable return value STATEPOINT node here and then hookup the
// return value of each gc.relocate to the respective output of the
// previously emitted STATEPOINT value. Unfortunately, this doesn't appear
// to actually be possible today.
}
SDValue SelectionDAGBuilder::LowerAsSTATEPOINT(
SelectionDAGBuilder::StatepointLoweringInfo &SI) {
// The basic scheme here is that information about both the original call and
// the safepoint is encoded in the CallInst. We create a temporary call and
// lower it, then reverse engineer the calling sequence.
NumOfStatepoints++;
// Clear state
StatepointLowering.startNewStatepoint(*this);
#ifndef NDEBUG
// We schedule gc relocates before removeDuplicateGCPtrs since we _will_
// encounter the duplicate gc relocates we elide in removeDuplicateGCPtrs.
for (auto *Reloc : SI.GCRelocates)
if (Reloc->getParent() == SI.StatepointInstr->getParent())
StatepointLowering.scheduleRelocCall(*Reloc);
#endif
// Remove any redundant llvm::Values which map to the same SDValue as another
// input. Also has the effect of removing duplicates in the original
// llvm::Value input list as well. This is a useful optimization for
// reducing the size of the StackMap section. It has no other impact.
removeDuplicateGCPtrs(SI.Bases, SI.Ptrs, SI.GCRelocates, *this,
FuncInfo.StatepointSpillMaps[SI.StatepointInstr]);
assert(SI.Bases.size() == SI.Ptrs.size() &&
SI.Ptrs.size() == SI.GCRelocates.size());
// Lower statepoint vmstate and gcstate arguments
SmallVector<SDValue, 10> LoweredMetaArgs;
lowerStatepointMetaArgs(LoweredMetaArgs, SI, *this);
// Now that we've emitted the spills, we need to update the root so that the
// call sequence is ordered correctly.
SI.CLI.setChain(getRoot());
// Get call node, we will replace it later with statepoint
SDValue ReturnVal;
SDNode *CallNode;
std::tie(ReturnVal, CallNode) =
lowerCallFromStatepointLoweringInfo(SI, *this, PendingExports);
// Construct the actual GC_TRANSITION_START, STATEPOINT, and GC_TRANSITION_END
// nodes with all the appropriate arguments and return values.
// Call Node: Chain, Target, {Args}, RegMask, [Glue]
SDValue Chain = CallNode->getOperand(0);
SDValue Glue;
bool CallHasIncomingGlue = CallNode->getGluedNode();
if (CallHasIncomingGlue) {
// Glue is always last operand
Glue = CallNode->getOperand(CallNode->getNumOperands() - 1);
}
// Build the GC_TRANSITION_START node if necessary.
//
// The operands to the GC_TRANSITION_{START,END} nodes are laid out in the
// order in which they appear in the call to the statepoint intrinsic. If
// any of the operands is a pointer-typed, that operand is immediately
// followed by a SRCVALUE for the pointer that may be used during lowering
// (e.g. to form MachinePointerInfo values for loads/stores).
const bool IsGCTransition =
(SI.StatepointFlags & (uint64_t)StatepointFlags::GCTransition) ==
(uint64_t)StatepointFlags::GCTransition;
if (IsGCTransition) {
SmallVector<SDValue, 8> TSOps;
// Add chain
TSOps.push_back(Chain);
// Add GC transition arguments
for (const Value *V : SI.GCTransitionArgs) {
TSOps.push_back(getValue(V));
if (V->getType()->isPointerTy())
TSOps.push_back(DAG.getSrcValue(V));
}
// Add glue if necessary
if (CallHasIncomingGlue)
TSOps.push_back(Glue);
SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
SDValue GCTransitionStart =
DAG.getNode(ISD::GC_TRANSITION_START, getCurSDLoc(), NodeTys, TSOps);
Chain = GCTransitionStart.getValue(0);
Glue = GCTransitionStart.getValue(1);
}
// TODO: Currently, all of these operands are being marked as read/write in
// PrologEpilougeInserter.cpp, we should special case the VMState arguments
// and flags to be read-only.
SmallVector<SDValue, 40> Ops;
// Add the <id> and <numBytes> constants.
Ops.push_back(DAG.getTargetConstant(SI.ID, getCurSDLoc(), MVT::i64));
Ops.push_back(
DAG.getTargetConstant(SI.NumPatchBytes, getCurSDLoc(), MVT::i32));
// Calculate and push starting position of vmstate arguments
// Get number of arguments incoming directly into call node
unsigned NumCallRegArgs =
CallNode->getNumOperands() - (CallHasIncomingGlue ? 4 : 3);
Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, getCurSDLoc(), MVT::i32));
// Add call target
SDValue CallTarget = SDValue(CallNode->getOperand(1).getNode(), 0);
Ops.push_back(CallTarget);
// Add call arguments
// Get position of register mask in the call
SDNode::op_iterator RegMaskIt;
if (CallHasIncomingGlue)
RegMaskIt = CallNode->op_end() - 2;
else
RegMaskIt = CallNode->op_end() - 1;
Ops.insert(Ops.end(), CallNode->op_begin() + 2, RegMaskIt);
// Add a constant argument for the calling convention
pushStackMapConstant(Ops, *this, SI.CLI.CallConv);
// Add a constant argument for the flags
uint64_t Flags = SI.StatepointFlags;
assert(((Flags & ~(uint64_t)StatepointFlags::MaskAll) == 0) &&
"Unknown flag used");
pushStackMapConstant(Ops, *this, Flags);
// Insert all vmstate and gcstate arguments
Ops.insert(Ops.end(), LoweredMetaArgs.begin(), LoweredMetaArgs.end());
// Add register mask from call node
Ops.push_back(*RegMaskIt);
// Add chain
Ops.push_back(Chain);
// Same for the glue, but we add it only if original call had it
if (Glue.getNode())
Ops.push_back(Glue);
// Compute return values. Provide a glue output since we consume one as
// input. This allows someone else to chain off us as needed.
SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
SDNode *StatepointMCNode =
DAG.getMachineNode(TargetOpcode::STATEPOINT, getCurSDLoc(), NodeTys, Ops);
SDNode *SinkNode = StatepointMCNode;
// Build the GC_TRANSITION_END node if necessary.
//
// See the comment above regarding GC_TRANSITION_START for the layout of
// the operands to the GC_TRANSITION_END node.
if (IsGCTransition) {
SmallVector<SDValue, 8> TEOps;
// Add chain
TEOps.push_back(SDValue(StatepointMCNode, 0));
// Add GC transition arguments
for (const Value *V : SI.GCTransitionArgs) {
TEOps.push_back(getValue(V));
if (V->getType()->isPointerTy())
TEOps.push_back(DAG.getSrcValue(V));
}
// Add glue
TEOps.push_back(SDValue(StatepointMCNode, 1));
SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
SDValue GCTransitionStart =
DAG.getNode(ISD::GC_TRANSITION_END, getCurSDLoc(), NodeTys, TEOps);
SinkNode = GCTransitionStart.getNode();
}
// Replace original call
DAG.ReplaceAllUsesWith(CallNode, SinkNode); // This may update Root
// Remove original call node
DAG.DeleteNode(CallNode);
// DON'T set the root - under the assumption that it's already set past the
// inserted node we created.
// TODO: A better future implementation would be to emit a single variable
// argument, variable return value STATEPOINT node here and then hookup the
// return value of each gc.relocate to the respective output of the
// previously emitted STATEPOINT value. Unfortunately, this doesn't appear
// to actually be possible today.
return ReturnVal;
}
