/// Select instructions not customized! Used for
/// expanded, promoted and normal instructions
bool Mips16DAGToDAGISel::trySelect(SDNode *Node) {
unsigned Opcode = Node->getOpcode();
SDLoc DL(Node);
///
// Instruction Selection not handled by the auto-generated
// tablegen selection should be handled here.
///
EVT NodeTy = Node->getValueType(0);
unsigned MultOpc;
switch (Opcode) {
default:
break;
case ISD::SUBE:
case ISD::ADDE: {
SDValue InFlag = Node->getOperand(2), CmpLHS;
unsigned Opc = InFlag.getOpcode();
(void)Opc;
assert(((Opc == ISD::ADDC || Opc == ISD::ADDE) ||
(Opc == ISD::SUBC || Opc == ISD::SUBE)) &&
"(ADD|SUB)E flag operand must come from (ADD|SUB)C/E insn");
unsigned MOp;
if (Opcode == ISD::ADDE) {
CmpLHS = InFlag.getValue(0);
MOp = Mips::AdduRxRyRz16;
} else {
CmpLHS = InFlag.getOperand(0);
MOp = Mips::SubuRxRyRz16;
}
SDValue Ops[] = {CmpLHS, InFlag.getOperand(1)};
SDValue LHS = Node->getOperand(0);
SDValue RHS = Node->getOperand(1);
EVT VT = LHS.getValueType();
unsigned Sltu_op = Mips::SltuRxRyRz16;
SDNode *Carry = CurDAG->getMachineNode(Sltu_op, DL, VT, Ops);
unsigned Addu_op = Mips::AdduRxRyRz16;
SDNode *AddCarry =
CurDAG->getMachineNode(Addu_op, DL, VT, SDValue(Carry, 0), RHS);
CurDAG->SelectNodeTo(Node, MOp, VT, MVT::Glue, LHS, SDValue(AddCarry, 0));
return true;
}
/// Mul with two results
case ISD::SMUL_LOHI:
case ISD::UMUL_LOHI: {
MultOpc = (Opcode == ISD::UMUL_LOHI ? Mips::MultuRxRy16 : Mips::MultRxRy16);
std::pair<SDNode *, SDNode *> LoHi =
selectMULT(Node, MultOpc, DL, NodeTy, true, true);
if (!SDValue(Node, 0).use_empty())
ReplaceUses(SDValue(Node, 0), SDValue(LoHi.first, 0));
if (!SDValue(Node, 1).use_empty())
ReplaceUses(SDValue(Node, 1), SDValue(LoHi.second, 0));
CurDAG->RemoveDeadNode(Node);
return true;
}
case ISD::MULHS:
case ISD::MULHU: {
MultOpc = (Opcode == ISD::MULHU ? Mips::MultuRxRy16 : Mips::MultRxRy16);
auto LoHi = selectMULT(Node, MultOpc, DL, NodeTy, false, true);
ReplaceNode(Node, LoHi.second);
return true;
}
}
return false;
}
SDValue
X86SelectionDAGInfo::EmitTargetCodeForMemset(SelectionDAG &DAG, SDLoc dl,
SDValue Chain,
SDValue Dst, SDValue Src,
SDValue Size, unsigned Align,
bool isVolatile,
MachinePointerInfo DstPtrInfo) const {
ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
// If to a segment-relative address space, use the default lowering.
if (DstPtrInfo.getAddrSpace() >= 256)
return SDValue();
// If not DWORD aligned or size is more than the threshold, call the library.
// The libc version is likely to be faster for these cases. It can use the
// address value and run time information about the CPU.
if ((Align & 3) != 0 ||
!ConstantSize ||
ConstantSize->getZExtValue() >
Subtarget->getMaxInlineSizeThreshold()) {
SDValue InFlag(0, 0);
// Check to see if there is a specialized entry-point for memory zeroing.
ConstantSDNode *V = dyn_cast<ConstantSDNode>(Src);
if (const char *bzeroEntry = V &&
V->isNullValue() ? Subtarget->getBZeroEntry() : 0) {
EVT IntPtr = TLI.getPointerTy();
Type *IntPtrTy = getDataLayout()->getIntPtrType(*DAG.getContext());
TargetLowering::ArgListTy Args;
TargetLowering::ArgListEntry Entry;
Entry.Node = Dst;
Entry.Ty = IntPtrTy;
Args.push_back(Entry);
Entry.Node = Size;
Args.push_back(Entry);
TargetLowering::
CallLoweringInfo CLI(Chain, Type::getVoidTy(*DAG.getContext()),
false, false, false, false,
0, CallingConv::C, /*isTailCall=*/false,
/*doesNotRet=*/false, /*isReturnValueUsed=*/false,
DAG.getExternalSymbol(bzeroEntry, IntPtr), Args,
DAG, dl);
std::pair<SDValue,SDValue> CallResult =
TLI.LowerCallTo(CLI);
return CallResult.second;
}
// Otherwise have the target-independent code call memset.
return SDValue();
}
uint64_t SizeVal = ConstantSize->getZExtValue();
SDValue InFlag(0, 0);
EVT AVT;
SDValue Count;
ConstantSDNode *ValC = dyn_cast<ConstantSDNode>(Src);
unsigned BytesLeft = 0;
bool TwoRepStos = false;
if (ValC) {
unsigned ValReg;
uint64_t Val = ValC->getZExtValue() & 255;
// If the value is a constant, then we can potentially use larger sets.
switch (Align & 3) {
case 2: // WORD aligned
AVT = MVT::i16;
ValReg = X86::AX;
Val = (Val << 8) | Val;
break;
case 0: // DWORD aligned
AVT = MVT::i32;
ValReg = X86::EAX;
Val = (Val << 8) | Val;
Val = (Val << 16) | Val;
if (Subtarget->is64Bit() && ((Align & 0x7) == 0)) { // QWORD aligned
AVT = MVT::i64;
ValReg = X86::RAX;
Val = (Val << 32) | Val;
}
break;
default: // Byte aligned
AVT = MVT::i8;
ValReg = X86::AL;
Count = DAG.getIntPtrConstant(SizeVal);
break;
}
if (AVT.bitsGT(MVT::i8)) {
unsigned UBytes = AVT.getSizeInBits() / 8;
Count = DAG.getIntPtrConstant(SizeVal / UBytes);
BytesLeft = SizeVal % UBytes;
}
Chain = DAG.getCopyToReg(Chain, dl, ValReg, DAG.getConstant(Val, AVT),
InFlag);
InFlag = Chain.getValue(1);
} else {
AVT = MVT::i8;
Count = DAG.getIntPtrConstant(SizeVal);
Chain = DAG.getCopyToReg(Chain, dl, X86::AL, Src, InFlag);
InFlag = Chain.getValue(1);
}
Chain = DAG.getCopyToReg(Chain, dl, Subtarget->is64Bit() ? X86::RCX :
X86::ECX,
Count, InFlag);
InFlag = Chain.getValue(1);
Chain = DAG.getCopyToReg(Chain, dl, Subtarget->is64Bit() ? X86::RDI :
X86::EDI,
Dst, InFlag);
InFlag = Chain.getValue(1);
SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
SDValue Ops[] = { Chain, DAG.getValueType(AVT), InFlag };
Chain = DAG.getNode(X86ISD::REP_STOS, dl, Tys, Ops, array_lengthof(Ops));
if (TwoRepStos) {
InFlag = Chain.getValue(1);
Count = Size;
EVT CVT = Count.getValueType();
SDValue Left = DAG.getNode(ISD::AND, dl, CVT, Count,
DAG.getConstant((AVT == MVT::i64) ? 7 : 3, CVT));
Chain = DAG.getCopyToReg(Chain, dl, (CVT == MVT::i64) ? X86::RCX :
X86::ECX,
Left, InFlag);
InFlag = Chain.getValue(1);
Tys = DAG.getVTList(MVT::Other, MVT::Glue);
SDValue Ops[] = { Chain, DAG.getValueType(MVT::i8), InFlag };
Chain = DAG.getNode(X86ISD::REP_STOS, dl, Tys, Ops, array_lengthof(Ops));
} else if (BytesLeft) {
// Handle the last 1 - 7 bytes.
unsigned Offset = SizeVal - BytesLeft;
EVT AddrVT = Dst.getValueType();
EVT SizeVT = Size.getValueType();
Chain = DAG.getMemset(Chain, dl,
DAG.getNode(ISD::ADD, dl, AddrVT, Dst,
DAG.getConstant(Offset, AddrVT)),
Src,
DAG.getConstant(BytesLeft, SizeVT),
Align, isVolatile, DstPtrInfo.getWithOffset(Offset));
}
// TODO: Use a Tokenfactor, as in memcpy, instead of a single chain.
return Chain;
}
/// EmitNode - Generate machine code for a node and needed dependencies.
///
void InstrEmitter::EmitNode(SDNode *Node, bool IsClone, bool IsCloned,
DenseMap<SDValue, unsigned> &VRBaseMap,
DenseMap<MachineBasicBlock*, MachineBasicBlock*> *EM) {
// If machine instruction
if (Node->isMachineOpcode()) {
unsigned Opc = Node->getMachineOpcode();
// Handle subreg insert/extract specially
if (Opc == TargetOpcode::EXTRACT_SUBREG ||
Opc == TargetOpcode::INSERT_SUBREG ||
Opc == TargetOpcode::SUBREG_TO_REG) {
EmitSubregNode(Node, VRBaseMap);
return;
}
// Handle COPY_TO_REGCLASS specially.
if (Opc == TargetOpcode::COPY_TO_REGCLASS) {
EmitCopyToRegClassNode(Node, VRBaseMap);
return;
}
if (Opc == TargetOpcode::IMPLICIT_DEF)
// We want a unique VR for each IMPLICIT_DEF use.
return;
const TargetInstrDesc &II = TII->get(Opc);
unsigned NumResults = CountResults(Node);
unsigned NodeOperands = CountOperands(Node);
bool HasPhysRegOuts = (NumResults > II.getNumDefs()) &&
II.getImplicitDefs() != 0;
#ifndef NDEBUG
unsigned NumMIOperands = NodeOperands + NumResults;
assert((II.getNumOperands() == NumMIOperands ||
HasPhysRegOuts || II.isVariadic()) &&
"#operands for dag node doesn't match .td file!");
#endif
// Create the new machine instruction.
MachineInstr *MI = BuildMI(*MF, Node->getDebugLoc(), II);
// Add result register values for things that are defined by this
// instruction.
if (NumResults)
CreateVirtualRegisters(Node, MI, II, IsClone, IsCloned, VRBaseMap);
// Emit all of the actual operands of this instruction, adding them to the
// instruction as appropriate.
bool HasOptPRefs = II.getNumDefs() > NumResults;
assert((!HasOptPRefs || !HasPhysRegOuts) &&
"Unable to cope with optional defs and phys regs defs!");
unsigned NumSkip = HasOptPRefs ? II.getNumDefs() - NumResults : 0;
for (unsigned i = NumSkip; i != NodeOperands; ++i)
AddOperand(MI, Node->getOperand(i), i-NumSkip+II.getNumDefs(), &II,
VRBaseMap);
// Transfer all of the memory reference descriptions of this instruction.
MI->setMemRefs(cast<MachineSDNode>(Node)->memoperands_begin(),
cast<MachineSDNode>(Node)->memoperands_end());
if (II.usesCustomInsertionHook()) {
// Insert this instruction into the basic block using a target
// specific inserter which may returns a new basic block.
MBB = TLI->EmitInstrWithCustomInserter(MI, MBB, EM);
InsertPos = MBB->end();
} else {
MBB->insert(InsertPos, MI);
}
// Additional results must be an physical register def.
if (HasPhysRegOuts) {
for (unsigned i = II.getNumDefs(); i < NumResults; ++i) {
unsigned Reg = II.getImplicitDefs()[i - II.getNumDefs()];
if (Node->hasAnyUseOfValue(i))
EmitCopyFromReg(Node, i, IsClone, IsCloned, Reg, VRBaseMap);
// If there are no uses, mark the register as dead now, so that
// MachineLICM/Sink can see that it's dead. Don't do this if the
// node has a Flag value, for the benefit of targets still using
// Flag for values in physregs.
else if (Node->getValueType(Node->getNumValues()-1) != MVT::Flag)
MI->addRegisterDead(Reg, TRI);
}
}
return;
}
switch (Node->getOpcode()) {
default:
#ifndef NDEBUG
Node->dump();
#endif
llvm_unreachable("This target-independent node should have been selected!");
break;
case ISD::EntryToken:
llvm_unreachable("EntryToken should have been excluded from the schedule!");
break;
case ISD::MERGE_VALUES:
case ISD::TokenFactor: // fall thru
break;
case ISD::CopyToReg: {
unsigned SrcReg;
SDValue SrcVal = Node->getOperand(2);
if (RegisterSDNode *R = dyn_cast<RegisterSDNode>(SrcVal))
SrcReg = R->getReg();
else
SrcReg = getVR(SrcVal, VRBaseMap);
unsigned DestReg = cast<RegisterSDNode>(Node->getOperand(1))->getReg();
if (SrcReg == DestReg) // Coalesced away the copy? Ignore.
break;
const TargetRegisterClass *SrcTRC = 0, *DstTRC = 0;
// Get the register classes of the src/dst.
if (TargetRegisterInfo::isVirtualRegister(SrcReg))
SrcTRC = MRI->getRegClass(SrcReg);
else
SrcTRC = TRI->getPhysicalRegisterRegClass(SrcReg,SrcVal.getValueType());
if (TargetRegisterInfo::isVirtualRegister(DestReg))
DstTRC = MRI->getRegClass(DestReg);
else
DstTRC = TRI->getPhysicalRegisterRegClass(DestReg,
Node->getOperand(1).getValueType());
bool Emitted = TII->copyRegToReg(*MBB, InsertPos, DestReg, SrcReg,
DstTRC, SrcTRC);
assert(Emitted && "Unable to issue a copy instruction!\n");
(void) Emitted;
break;
}
case ISD::CopyFromReg: {
unsigned SrcReg = cast<RegisterSDNode>(Node->getOperand(1))->getReg();
EmitCopyFromReg(Node, 0, IsClone, IsCloned, SrcReg, VRBaseMap);
break;
}
case ISD::INLINEASM: {
unsigned NumOps = Node->getNumOperands();
if (Node->getOperand(NumOps-1).getValueType() == MVT::Flag)
--NumOps; // Ignore the flag operand.
// Create the inline asm machine instruction.
MachineInstr *MI = BuildMI(*MF, Node->getDebugLoc(),
TII->get(TargetOpcode::INLINEASM));
// Add the asm string as an external symbol operand.
const char *AsmStr =
cast<ExternalSymbolSDNode>(Node->getOperand(1))->getSymbol();
MI->addOperand(MachineOperand::CreateES(AsmStr));
// Add all of the operand registers to the instruction.
for (unsigned i = 2; i != NumOps;) {
unsigned Flags =
cast<ConstantSDNode>(Node->getOperand(i))->getZExtValue();
unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags);
MI->addOperand(MachineOperand::CreateImm(Flags));
++i; // Skip the ID value.
switch (Flags & 7) {
default: llvm_unreachable("Bad flags!");
case 2: // Def of register.
for (; NumVals; --NumVals, ++i) {
unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg();
MI->addOperand(MachineOperand::CreateReg(Reg, true));
}
break;
case 6: // Def of earlyclobber register.
for (; NumVals; --NumVals, ++i) {
unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg();
MI->addOperand(MachineOperand::CreateReg(Reg, true, false, false,
false, false, true));
}
break;
case 1: // Use of register.
case 3: // Immediate.
case 4: // Addressing mode.
// The addressing mode has been selected, just add all of the
// operands to the machine instruction.
for (; NumVals; --NumVals, ++i)
AddOperand(MI, Node->getOperand(i), 0, 0, VRBaseMap);
break;
}
}
MBB->insert(InsertPos, MI);
break;
}
}
}
SDValue IA64TargetLowering::
LowerOperation(SDValue Op, SelectionDAG &DAG) {
switch (Op.getOpcode()) {
default: assert(0 && "Should not custom lower this!");
case ISD::GlobalTLSAddress:
assert(0 && "TLS not implemented for IA64.");
case ISD::RET: {
SDValue AR_PFSVal, Copy;
switch(Op.getNumOperands()) {
default:
assert(0 && "Do not know how to return this many arguments!");
abort();
case 1:
AR_PFSVal = DAG.getCopyFromReg(Op.getOperand(0), VirtGPR, MVT::i64);
AR_PFSVal = DAG.getCopyToReg(AR_PFSVal.getValue(1), IA64::AR_PFS,
AR_PFSVal);
return DAG.getNode(IA64ISD::RET_FLAG, MVT::Other, AR_PFSVal);
case 3: {
// Copy the result into the output register & restore ar.pfs
MVT ArgVT = Op.getOperand(1).getValueType();
unsigned ArgReg = ArgVT.isInteger() ? IA64::r8 : IA64::F8;
AR_PFSVal = DAG.getCopyFromReg(Op.getOperand(0), VirtGPR, MVT::i64);
Copy = DAG.getCopyToReg(AR_PFSVal.getValue(1), ArgReg, Op.getOperand(1),
SDValue());
AR_PFSVal = DAG.getCopyToReg(Copy.getValue(0), IA64::AR_PFS, AR_PFSVal,
Copy.getValue(1));
return DAG.getNode(IA64ISD::RET_FLAG, MVT::Other,
AR_PFSVal, AR_PFSVal.getValue(1));
}
}
return SDValue();
}
case ISD::VAARG: {
MVT VT = getPointerTy();
const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
SDValue VAList = DAG.getLoad(VT, Op.getOperand(0), Op.getOperand(1),
SV, 0);
// Increment the pointer, VAList, to the next vaarg
SDValue VAIncr = DAG.getNode(ISD::ADD, VT, VAList,
DAG.getConstant(VT.getSizeInBits()/8,
VT));
// Store the incremented VAList to the legalized pointer
VAIncr = DAG.getStore(VAList.getValue(1), VAIncr,
Op.getOperand(1), SV, 0);
// Load the actual argument out of the pointer VAList
return DAG.getLoad(Op.getValueType(), VAIncr, VAList, NULL, 0);
}
case ISD::VASTART: {
// vastart just stores the address of the VarArgsFrameIndex slot into the
// memory location argument.
SDValue FR = DAG.getFrameIndex(VarArgsFrameIndex, MVT::i64);
const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
return DAG.getStore(Op.getOperand(0), FR, Op.getOperand(1), SV, 0);
}
// Frame & Return address. Currently unimplemented
case ISD::RETURNADDR: break;
case ISD::FRAMEADDR: break;
}
return SDValue();
}
/// Lower deopt state and gc pointer arguments of the statepoint. The actual
/// lowering is described in lowerIncomingStatepointValue. This function is
/// responsible for lowering everything in the right position and playing some
/// tricks to avoid redundant stack manipulation where possible. On
/// completion, 'Ops' will contain ready to use operands for machine code
/// statepoint. The chain nodes will have already been created and the DAG root
/// will be set to the last value spilled (if any were).
static void lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
ImmutableStatepoint StatepointSite,
SelectionDAGBuilder &Builder) {
// Lower the deopt and gc arguments for this statepoint. Layout will
// be: deopt argument length, deopt arguments.., gc arguments...
SmallVector<const Value *, 64> Bases, Ptrs, Relocations;
getIncomingStatepointGCValues(Bases, Ptrs, Relocations, StatepointSite,
Builder);
#ifndef NDEBUG
// Check that each of the gc pointer and bases we've gotten out of the
// safepoint is something the strategy thinks might be a pointer into the GC
// heap. This is basically just here to help catch errors during statepoint
// insertion. TODO: This should actually be in the Verifier, but we can't get
// to the GCStrategy from there (yet).
GCStrategy &S = Builder.GFI->getStrategy();
for (const Value *V : Bases) {
auto Opt = S.isGCManagedPointer(V);
if (Opt.hasValue()) {
assert(Opt.getValue() &&
"non gc managed base pointer found in statepoint");
}
}
for (const Value *V : Ptrs) {
auto Opt = S.isGCManagedPointer(V);
if (Opt.hasValue()) {
assert(Opt.getValue() &&
"non gc managed derived pointer found in statepoint");
}
}
for (const Value *V : Relocations) {
auto Opt = S.isGCManagedPointer(V);
if (Opt.hasValue()) {
assert(Opt.getValue() && "non gc managed pointer relocated");
}
}
#endif
// Before we actually start lowering (and allocating spill slots for values),
// reserve any stack slots which we judge to be profitable to reuse for a
// particular value. This is purely an optimization over the code below and
// doesn't change semantics at all. It is important for performance that we
// reserve slots for both deopt and gc values before lowering either.
for (const Value *V : StatepointSite.vm_state_args()) {
reservePreviousStackSlotForValue(V, Builder);
}
for (unsigned i = 0; i < Bases.size(); ++i) {
reservePreviousStackSlotForValue(Bases[i], Builder);
reservePreviousStackSlotForValue(Ptrs[i], Builder);
}
// First, prefix the list with the number of unique values to be
// lowered. Note that this is the number of *Values* not the
// number of SDValues required to lower them.
const int NumVMSArgs = StatepointSite.getNumTotalVMSArgs();
pushStackMapConstant(Ops, Builder, NumVMSArgs);
assert(NumVMSArgs == std::distance(StatepointSite.vm_state_begin(),
StatepointSite.vm_state_end()));
// The vm state arguments are lowered in an opaque manner. We do
// not know what type of values are contained within. We skip the
// first one since that happens to be the total number we lowered
// explicitly just above. We could have left it in the loop and
// not done it explicitly, but it's far easier to understand this
// way.
for (const Value *V : StatepointSite.vm_state_args()) {
SDValue Incoming = Builder.getValue(V);
lowerIncomingStatepointValue(Incoming, Ops, Builder);
}
// Finally, go ahead and lower all the gc arguments. There's no prefixed
// length for this one. After lowering, we'll have the base and pointer
// arrays interwoven with each (lowered) base pointer immediately followed by
// it's (lowered) derived pointer. i.e
// (base[0], ptr[0], base[1], ptr[1], ...)
for (unsigned i = 0; i < Bases.size(); ++i) {
const Value *Base = Bases[i];
lowerIncomingStatepointValue(Builder.getValue(Base), Ops, Builder);
const Value *Ptr = Ptrs[i];
lowerIncomingStatepointValue(Builder.getValue(Ptr), Ops, Builder);
}
// If there are any explicit spill slots passed to the statepoint, record
// them, but otherwise do not do anything special. These are user provided
// allocas and give control over placement to the consumer. In this case,
// it is the contents of the slot which may get updated, not the pointer to
// the alloca
for (Value *V : StatepointSite.gc_args()) {
SDValue Incoming = Builder.getValue(V);
if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
// This handles allocas as arguments to the statepoint
Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
Incoming.getValueType()));
}
}
// Record computed locations for all lowered values.
// This can not be embedded in lowering loops as we need to record *all*
// values, while previous loops account only values with unique SDValues.
const Instruction *StatepointInstr =
StatepointSite.getCallSite().getInstruction();
FunctionLoweringInfo::StatepointSpilledValueMapTy &SpillMap =
Builder.FuncInfo.StatepointRelocatedValues[StatepointInstr];
for (GCRelocateOperands RelocateOpers : StatepointSite.getRelocates()) {
const Value *V = RelocateOpers.getDerivedPtr();
SDValue SDV = Builder.getValue(V);
SDValue Loc = Builder.StatepointLowering.getLocation(SDV);
if (Loc.getNode()) {
SpillMap[V] = cast<FrameIndexSDNode>(Loc)->getIndex();
} else {
// Record value as visited, but not spilled. This is case for allocas
// and constants. For this values we can avoid emiting spill load while
// visiting corresponding gc_relocate.
// Actually we do not need to record them in this map at all.
// We do this only to check that we are not relocating any unvisited value.
SpillMap[V] = None;
// Default llvm mechanisms for exporting values which are used in
// different basic blocks does not work for gc relocates.
// Note that it would be incorrect to teach llvm that all relocates are
// uses of the corresponging values so that it would automatically
// export them. Relocates of the spilled values does not use original
// value.
if (StatepointSite.getCallSite().isInvoke())
Builder.ExportFromCurrentBlock(V);
}
}
}
SDValue SystemZTargetLowering::EmitCmp(SDValue LHS, SDValue RHS,
ISD::CondCode CC, SDValue &SystemZCC,
SelectionDAG &DAG) const {
// FIXME: Emit a test if RHS is zero
bool isUnsigned = false;
SystemZCC::CondCodes TCC;
switch (CC) {
default:
llvm_unreachable("Invalid integer condition!");
case ISD::SETEQ:
case ISD::SETOEQ:
TCC = SystemZCC::E;
break;
case ISD::SETUEQ:
TCC = SystemZCC::NLH;
break;
case ISD::SETNE:
case ISD::SETONE:
TCC = SystemZCC::NE;
break;
case ISD::SETUNE:
TCC = SystemZCC::LH;
break;
case ISD::SETO:
TCC = SystemZCC::O;
break;
case ISD::SETUO:
TCC = SystemZCC::NO;
break;
case ISD::SETULE:
if (LHS.getValueType().isFloatingPoint()) {
TCC = SystemZCC::NH;
break;
}
isUnsigned = true; // FALLTHROUGH
case ISD::SETLE:
case ISD::SETOLE:
TCC = SystemZCC::LE;
break;
case ISD::SETUGE:
if (LHS.getValueType().isFloatingPoint()) {
TCC = SystemZCC::NL;
break;
}
isUnsigned = true; // FALLTHROUGH
case ISD::SETGE:
case ISD::SETOGE:
TCC = SystemZCC::HE;
break;
case ISD::SETUGT:
if (LHS.getValueType().isFloatingPoint()) {
TCC = SystemZCC::NLE;
break;
}
isUnsigned = true; // FALLTHROUGH
case ISD::SETGT:
case ISD::SETOGT:
TCC = SystemZCC::H;
break;
case ISD::SETULT:
if (LHS.getValueType().isFloatingPoint()) {
TCC = SystemZCC::NHE;
break;
}
isUnsigned = true; // FALLTHROUGH
case ISD::SETLT:
case ISD::SETOLT:
TCC = SystemZCC::L;
break;
}
SystemZCC = DAG.getConstant(TCC, MVT::i32);
DebugLoc dl = LHS.getDebugLoc();
return DAG.getNode((isUnsigned ? SystemZISD::UCMP : SystemZISD::CMP),
dl, MVT::i64, LHS, RHS);
}
SDValue X86SelectionDAGInfo::EmitTargetCodeForMemcpy(
SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
SDValue Size, unsigned Align, bool isVolatile, bool AlwaysInline,
MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
// This requires the copy size to be a constant, preferably
// within a subtarget-specific limit.
ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
const X86Subtarget &Subtarget =
DAG.getMachineFunction().getSubtarget<X86Subtarget>();
if (!ConstantSize)
return SDValue();
RepMovsRepeats Repeats(ConstantSize->getZExtValue());
if (!AlwaysInline && Repeats.Size > Subtarget.getMaxInlineSizeThreshold())
return SDValue();
/// If not DWORD aligned, it is more efficient to call the library. However
/// if calling the library is not allowed (AlwaysInline), then soldier on as
/// the code generated here is better than the long load-store sequence we
/// would otherwise get.
if (!AlwaysInline && (Align & 3) != 0)
return SDValue();
// If to a segment-relative address space, use the default lowering.
if (DstPtrInfo.getAddrSpace() >= 256 ||
SrcPtrInfo.getAddrSpace() >= 256)
return SDValue();
// If the base register might conflict with our physical registers, bail out.
const MCPhysReg ClobberSet[] = {X86::RCX, X86::RSI, X86::RDI,
X86::ECX, X86::ESI, X86::EDI};
if (isBaseRegConflictPossible(DAG, ClobberSet))
return SDValue();
// If the target has enhanced REPMOVSB, then it's at least as fast to use
// REP MOVSB instead of REP MOVS{W,D,Q}, and it avoids having to handle
// BytesLeft.
if (!Subtarget.hasERMSB() && !(Align & 1)) {
if (Align & 2)
// WORD aligned
Repeats.AVT = MVT::i16;
else if (Align & 4)
// DWORD aligned
Repeats.AVT = MVT::i32;
else
// QWORD aligned
Repeats.AVT = Subtarget.is64Bit() ? MVT::i64 : MVT::i32;
if (Repeats.BytesLeft() > 0 &&
DAG.getMachineFunction().getFunction().optForMinSize()) {
// When aggressively optimizing for size, avoid generating the code to
// handle BytesLeft.
Repeats.AVT = MVT::i8;
}
}
bool Use64BitRegs = Subtarget.isTarget64BitLP64();
SDValue InFlag;
Chain = DAG.getCopyToReg(Chain, dl, Use64BitRegs ? X86::RCX : X86::ECX,
DAG.getIntPtrConstant(Repeats.Count(), dl), InFlag);
InFlag = Chain.getValue(1);
Chain = DAG.getCopyToReg(Chain, dl, Use64BitRegs ? X86::RDI : X86::EDI,
Dst, InFlag);
InFlag = Chain.getValue(1);
Chain = DAG.getCopyToReg(Chain, dl, Use64BitRegs ? X86::RSI : X86::ESI,
Src, InFlag);
InFlag = Chain.getValue(1);
SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
SDValue Ops[] = { Chain, DAG.getValueType(Repeats.AVT), InFlag };
SDValue RepMovs = DAG.getNode(X86ISD::REP_MOVS, dl, Tys, Ops);
SmallVector<SDValue, 4> Results;
Results.push_back(RepMovs);
if (Repeats.BytesLeft()) {
// Handle the last 1 - 7 bytes.
unsigned Offset = Repeats.Size - Repeats.BytesLeft();
EVT DstVT = Dst.getValueType();
EVT SrcVT = Src.getValueType();
EVT SizeVT = Size.getValueType();
Results.push_back(DAG.getMemcpy(Chain, dl,
DAG.getNode(ISD::ADD, dl, DstVT, Dst,
DAG.getConstant(Offset, dl,
DstVT)),
DAG.getNode(ISD::ADD, dl, SrcVT, Src,
DAG.getConstant(Offset, dl,
SrcVT)),
DAG.getConstant(Repeats.BytesLeft(), dl,
SizeVT),
Align, isVolatile, AlwaysInline, false,
DstPtrInfo.getWithOffset(Offset),
SrcPtrInfo.getWithOffset(Offset)));
}
return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Results);
}
void DAGTypeLegalizer::ExpandRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi) {
EVT OutVT = N->getValueType(0);
EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
SDValue InOp = N->getOperand(0);
EVT InVT = InOp.getValueType();
SDLoc dl(N);
// Handle some special cases efficiently.
switch (getTypeAction(InVT)) {
case TargetLowering::TypeLegal:
case TargetLowering::TypePromoteInteger:
break;
case TargetLowering::TypePromoteFloat:
llvm_unreachable("Bitcast of a promotion-needing float should never need"
"expansion");
case TargetLowering::TypeSoftenFloat:
// Convert the integer operand instead.
SplitInteger(GetSoftenedFloat(InOp), Lo, Hi);
Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
return;
case TargetLowering::TypeExpandInteger:
case TargetLowering::TypeExpandFloat:
// Convert the expanded pieces of the input.
GetExpandedOp(InOp, Lo, Hi);
if (TLI.hasBigEndianPartOrdering(InVT) !=
TLI.hasBigEndianPartOrdering(OutVT))
std::swap(Lo, Hi);
Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
return;
case TargetLowering::TypeSplitVector:
GetSplitVector(InOp, Lo, Hi);
if (TLI.hasBigEndianPartOrdering(OutVT))
std::swap(Lo, Hi);
Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
return;
case TargetLowering::TypeScalarizeVector:
// Convert the element instead.
SplitInteger(BitConvertToInteger(GetScalarizedVector(InOp)), Lo, Hi);
Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
return;
case TargetLowering::TypeWidenVector: {
assert(!(InVT.getVectorNumElements() & 1) && "Unsupported BITCAST");
InOp = GetWidenedVector(InOp);
EVT LoVT, HiVT;
std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(InVT);
std::tie(Lo, Hi) = DAG.SplitVector(InOp, dl, LoVT, HiVT);
if (TLI.hasBigEndianPartOrdering(OutVT))
std::swap(Lo, Hi);
Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
return;
}
}
if (InVT.isVector() && OutVT.isInteger()) {
// Handle cases like i64 = BITCAST v1i64 on x86, where the operand
// is legal but the result is not.
unsigned NumElems = 2;
EVT ElemVT = NOutVT;
EVT NVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElems);
// If <ElemVT * N> is not a legal type, try <ElemVT/2 * (N*2)>.
while (!isTypeLegal(NVT)) {
unsigned NewSizeInBits = ElemVT.getSizeInBits() / 2;
// If the element size is smaller than byte, bail.
if (NewSizeInBits < 8)
break;
NumElems *= 2;
ElemVT = EVT::getIntegerVT(*DAG.getContext(), NewSizeInBits);
NVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElems);
}
if (isTypeLegal(NVT)) {
SDValue CastInOp = DAG.getNode(ISD::BITCAST, dl, NVT, InOp);
SmallVector<SDValue, 8> Vals;
for (unsigned i = 0; i < NumElems; ++i)
Vals.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ElemVT,
CastInOp, DAG.getConstant(i,
TLI.getVectorIdxTy())));
// Build Lo, Hi pair by pairing extracted elements if needed.
unsigned Slot = 0;
for (unsigned e = Vals.size(); e - Slot > 2; Slot += 2, e += 1) {
// Each iteration will BUILD_PAIR two nodes and append the result until
// there are only two nodes left, i.e. Lo and Hi.
SDValue LHS = Vals[Slot];
SDValue RHS = Vals[Slot + 1];
if (TLI.isBigEndian())
std::swap(LHS, RHS);
Vals.push_back(DAG.getNode(ISD::BUILD_PAIR, dl,
EVT::getIntegerVT(
*DAG.getContext(),
LHS.getValueType().getSizeInBits() << 1),
LHS, RHS));
}
Lo = Vals[Slot++];
Hi = Vals[Slot++];
if (TLI.isBigEndian())
std::swap(Lo, Hi);
return;
}
}
// Lower the bit-convert to a store/load from the stack.
assert(NOutVT.isByteSized() && "Expanded type not byte sized!");
// Create the stack frame object. Make sure it is aligned for both
// the source and expanded destination types.
unsigned Alignment =
TLI.getDataLayout()->getPrefTypeAlignment(NOutVT.
getTypeForEVT(*DAG.getContext()));
SDValue StackPtr = DAG.CreateStackTemporary(InVT, Alignment);
int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(SPFI);
// Emit a store to the stack slot.
SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, InOp, StackPtr, PtrInfo,
false, false, 0);
// Load the first half from the stack slot.
Lo = DAG.getLoad(NOutVT, dl, Store, StackPtr, PtrInfo,
false, false, false, 0);
// Increment the pointer to the other half.
unsigned IncrementSize = NOutVT.getSizeInBits() / 8;
StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
DAG.getConstant(IncrementSize,
StackPtr.getValueType()));
// Load the second half from the stack slot.
Hi = DAG.getLoad(NOutVT, dl, Store, StackPtr,
PtrInfo.getWithOffset(IncrementSize), false,
false, false, MinAlign(Alignment, IncrementSize));
// Handle endianness of the load.
if (TLI.hasBigEndianPartOrdering(OutVT))
std::swap(Lo, Hi);
}
static SDValue EmitCMP(SDValue &LHS, SDValue &RHS, SDValue &TargetCC,
ISD::CondCode CC,
DebugLoc dl, SelectionDAG &DAG) {
// FIXME: Handle bittests someday
assert(!LHS.getValueType().isFloatingPoint() && "We don't handle FP yet");
// FIXME: Handle jump negative someday
MSP430CC::CondCodes TCC = MSP430CC::COND_INVALID;
switch (CC) {
default: llvm_unreachable("Invalid integer condition!");
case ISD::SETEQ:
TCC = MSP430CC::COND_E; // aka COND_Z
// Minor optimization: if LHS is a constant, swap operands, then the
// constant can be folded into comparison.
if (LHS.getOpcode() == ISD::Constant)
std::swap(LHS, RHS);
break;
case ISD::SETNE:
TCC = MSP430CC::COND_NE; // aka COND_NZ
// Minor optimization: if LHS is a constant, swap operands, then the
// constant can be folded into comparison.
if (LHS.getOpcode() == ISD::Constant)
std::swap(LHS, RHS);
break;
case ISD::SETULE:
std::swap(LHS, RHS); // FALLTHROUGH
case ISD::SETUGE:
// Turn lhs u>= rhs with lhs constant into rhs u< lhs+1, this allows us to
// fold constant into instruction.
if (const ConstantSDNode * C = dyn_cast<ConstantSDNode>(LHS)) {
LHS = RHS;
RHS = DAG.getConstant(C->getSExtValue() + 1, C->getValueType(0));
TCC = MSP430CC::COND_LO;
break;
}
TCC = MSP430CC::COND_HS; // aka COND_C
break;
case ISD::SETUGT:
std::swap(LHS, RHS); // FALLTHROUGH
case ISD::SETULT:
// Turn lhs u< rhs with lhs constant into rhs u>= lhs+1, this allows us to
// fold constant into instruction.
if (const ConstantSDNode * C = dyn_cast<ConstantSDNode>(LHS)) {
LHS = RHS;
RHS = DAG.getConstant(C->getSExtValue() + 1, C->getValueType(0));
TCC = MSP430CC::COND_HS;
break;
}
TCC = MSP430CC::COND_LO; // aka COND_NC
break;
case ISD::SETLE:
std::swap(LHS, RHS); // FALLTHROUGH
case ISD::SETGE:
// Turn lhs >= rhs with lhs constant into rhs < lhs+1, this allows us to
// fold constant into instruction.
if (const ConstantSDNode * C = dyn_cast<ConstantSDNode>(LHS)) {
LHS = RHS;
RHS = DAG.getConstant(C->getSExtValue() + 1, C->getValueType(0));
TCC = MSP430CC::COND_L;
break;
}
TCC = MSP430CC::COND_GE;
break;
case ISD::SETGT:
std::swap(LHS, RHS); // FALLTHROUGH
case ISD::SETLT:
// Turn lhs < rhs with lhs constant into rhs >= lhs+1, this allows us to
// fold constant into instruction.
if (const ConstantSDNode * C = dyn_cast<ConstantSDNode>(LHS)) {
LHS = RHS;
RHS = DAG.getConstant(C->getSExtValue() + 1, C->getValueType(0));
TCC = MSP430CC::COND_GE;
break;
}
TCC = MSP430CC::COND_L;
break;
}
TargetCC = DAG.getConstant(TCC, MVT::i8);
return DAG.getNode(MSP430ISD::CMP, dl, MVT::Glue, LHS, RHS);
}
SDValue MSP430TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
SDValue LHS = Op.getOperand(0);
SDValue RHS = Op.getOperand(1);
DebugLoc dl = Op.getDebugLoc();
// If we are doing an AND and testing against zero, then the CMP
// will not be generated. The AND (or BIT) will generate the condition codes,
// but they are different from CMP.
// FIXME: since we're doing a post-processing, use a pseudoinstr here, so
// lowering & isel wouldn't diverge.
bool andCC = false;
if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
if (RHSC->isNullValue() && LHS.hasOneUse() &&
(LHS.getOpcode() == ISD::AND ||
(LHS.getOpcode() == ISD::TRUNCATE &&
LHS.getOperand(0).getOpcode() == ISD::AND))) {
andCC = true;
}
}
ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
SDValue TargetCC;
SDValue Flag = EmitCMP(LHS, RHS, TargetCC, CC, dl, DAG);
// Get the condition codes directly from the status register, if its easy.
// Otherwise a branch will be generated. Note that the AND and BIT
// instructions generate different flags than CMP, the carry bit can be used
// for NE/EQ.
bool Invert = false;
bool Shift = false;
bool Convert = true;
switch (cast<ConstantSDNode>(TargetCC)->getZExtValue()) {
default:
Convert = false;
break;
case MSP430CC::COND_HS:
// Res = SRW & 1, no processing is required
break;
case MSP430CC::COND_LO:
// Res = ~(SRW & 1)
Invert = true;
break;
case MSP430CC::COND_NE:
if (andCC) {
// C = ~Z, thus Res = SRW & 1, no processing is required
} else {
// Res = ~((SRW >> 1) & 1)
Shift = true;
Invert = true;
}
break;
case MSP430CC::COND_E:
Shift = true;
// C = ~Z for AND instruction, thus we can put Res = ~(SRW & 1), however,
// Res = (SRW >> 1) & 1 is 1 word shorter.
break;
}
EVT VT = Op.getValueType();
SDValue One = DAG.getConstant(1, VT);
if (Convert) {
SDValue SR = DAG.getCopyFromReg(DAG.getEntryNode(), dl, MSP430::SRW,
MVT::i16, Flag);
if (Shift)
// FIXME: somewhere this is turned into a SRL, lower it MSP specific?
SR = DAG.getNode(ISD::SRA, dl, MVT::i16, SR, One);
SR = DAG.getNode(ISD::AND, dl, MVT::i16, SR, One);
if (Invert)
SR = DAG.getNode(ISD::XOR, dl, MVT::i16, SR, One);
return SR;
} else {
SDValue Zero = DAG.getConstant(0, VT);
SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue);
SmallVector<SDValue, 4> Ops;
Ops.push_back(One);
Ops.push_back(Zero);
Ops.push_back(TargetCC);
Ops.push_back(Flag);
return DAG.getNode(MSP430ISD::SELECT_CC, dl, VTs, &Ops[0], Ops.size());
}
}
bool MSP430TargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
return isZExtFree(Val.getValueType(), VT2);
}
void Mips16TargetLowering::
getOpndList(SmallVectorImpl<SDValue> &Ops,
std::deque< std::pair<unsigned, SDValue> > &RegsToPass,
bool IsPICCall, bool GlobalOrExternal, bool InternalLinkage,
CallLoweringInfo &CLI, SDValue Callee, SDValue Chain) const {
SelectionDAG &DAG = CLI.DAG;
MachineFunction &MF = DAG.getMachineFunction();
MipsFunctionInfo *FuncInfo = MF.getInfo<MipsFunctionInfo>();
const char* Mips16HelperFunction = nullptr;
bool NeedMips16Helper = false;
if (Subtarget.inMips16HardFloat()) {
//
// currently we don't have symbols tagged with the mips16 or mips32
// qualifier so we will assume that we don't know what kind it is.
// and generate the helper
//
bool LookupHelper = true;
if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(CLI.Callee)) {
Mips16Libcall Find = { RTLIB::UNKNOWN_LIBCALL, S->getSymbol() };
if (std::binary_search(std::begin(HardFloatLibCalls),
std::end(HardFloatLibCalls), Find))
LookupHelper = false;
else {
const char *Symbol = S->getSymbol();
Mips16IntrinsicHelperType IntrinsicFind = { Symbol, "" };
const Mips16HardFloatInfo::FuncSignature *Signature =
Mips16HardFloatInfo::findFuncSignature(Symbol);
if (!IsPICCall && (Signature && (FuncInfo->StubsNeeded.find(Symbol) ==
FuncInfo->StubsNeeded.end()))) {
FuncInfo->StubsNeeded[Symbol] = Signature;
//
// S2 is normally saved if the stub is for a function which
// returns a float or double value and is not otherwise. This is
// because more work is required after the function the stub
// is calling completes, and so the stub cannot directly return
// and the stub has no stack space to store the return address so
// S2 is used for that purpose.
// In order to take advantage of not saving S2, we need to also
// optimize the call in the stub and this requires some further
// functionality in MipsAsmPrinter which we don't have yet.
// So for now we always save S2. The optimization will be done
// in a follow-on patch.
//
if (1 || (Signature->RetSig != Mips16HardFloatInfo::NoFPRet))
FuncInfo->setSaveS2();
}
// one more look at list of intrinsics
const Mips16IntrinsicHelperType *Helper =
std::lower_bound(std::begin(Mips16IntrinsicHelper),
std::end(Mips16IntrinsicHelper), IntrinsicFind);
if (Helper != std::end(Mips16IntrinsicHelper) &&
*Helper == IntrinsicFind) {
Mips16HelperFunction = Helper->Helper;
NeedMips16Helper = true;
LookupHelper = false;
}
}
} else if (GlobalAddressSDNode *G =
dyn_cast<GlobalAddressSDNode>(CLI.Callee)) {
Mips16Libcall Find = { RTLIB::UNKNOWN_LIBCALL,
G->getGlobal()->getName().data() };
if (std::binary_search(std::begin(HardFloatLibCalls),
std::end(HardFloatLibCalls), Find))
LookupHelper = false;
}
if (LookupHelper)
Mips16HelperFunction =
getMips16HelperFunction(CLI.RetTy, CLI.getArgs(), NeedMips16Helper);
}
SDValue JumpTarget = Callee;
// T9 should contain the address of the callee function if
// -reloction-model=pic or it is an indirect call.
if (IsPICCall || !GlobalOrExternal) {
unsigned V0Reg = Mips::V0;
if (NeedMips16Helper) {
RegsToPass.push_front(std::make_pair(V0Reg, Callee));
JumpTarget = DAG.getExternalSymbol(Mips16HelperFunction, getPointerTy());
ExternalSymbolSDNode *S = cast<ExternalSymbolSDNode>(JumpTarget);
JumpTarget = getAddrGlobal(S, JumpTarget.getValueType(), DAG,
MipsII::MO_GOT, Chain,
FuncInfo->callPtrInfo(S->getSymbol()));
} else
RegsToPass.push_front(std::make_pair((unsigned)Mips::T9, Callee));
}
Ops.push_back(JumpTarget);
MipsTargetLowering::getOpndList(Ops, RegsToPass, IsPICCall, GlobalOrExternal,
InternalLinkage, CLI, Callee, Chain);
}
/// AddRegisterOperand - Add the specified register as an operand to the
/// specified machine instr. Insert register copies if the register is
/// not in the required register class.
void
InstrEmitter::AddRegisterOperand(MachineInstrBuilder &MIB,
SDValue Op,
unsigned IIOpNum,
const MCInstrDesc *II,
DenseMap<SDValue, unsigned> &VRBaseMap,
bool IsDebug, bool IsClone, bool IsCloned) {
assert(Op.getValueType() != MVT::Other &&
Op.getValueType() != MVT::Glue &&
"Chain and glue operands should occur at end of operand list!");
// Get/emit the operand.
unsigned VReg = getVR(Op, VRBaseMap);
const MCInstrDesc &MCID = MIB->getDesc();
bool isOptDef = IIOpNum < MCID.getNumOperands() &&
MCID.OpInfo[IIOpNum].isOptionalDef();
// If the instruction requires a register in a different class, create
// a new virtual register and copy the value into it, but first attempt to
// shrink VReg's register class within reason. For example, if VReg == GR32
// and II requires a GR32_NOSP, just constrain VReg to GR32_NOSP.
if (II) {
const TargetRegisterClass *OpRC = nullptr;
if (IIOpNum < II->getNumOperands())
OpRC = TII->getRegClass(*II, IIOpNum, TRI, *MF);
if (OpRC) {
const TargetRegisterClass *ConstrainedRC
= MRI->constrainRegClass(VReg, OpRC, MinRCSize);
if (!ConstrainedRC) {
OpRC = TRI->getAllocatableClass(OpRC);
assert(OpRC && "Constraints cannot be fulfilled for allocation");
unsigned NewVReg = MRI->createVirtualRegister(OpRC);
BuildMI(*MBB, InsertPos, Op.getNode()->getDebugLoc(),
TII->get(TargetOpcode::COPY), NewVReg).addReg(VReg);
VReg = NewVReg;
} else {
assert(ConstrainedRC->isAllocatable() &&
"Constraining an allocatable VReg produced an unallocatable class?");
}
}
}
// If this value has only one use, that use is a kill. This is a
// conservative approximation. InstrEmitter does trivial coalescing
// with CopyFromReg nodes, so don't emit kill flags for them.
// Avoid kill flags on Schedule cloned nodes, since there will be
// multiple uses.
// Tied operands are never killed, so we need to check that. And that
// means we need to determine the index of the operand.
bool isKill = Op.hasOneUse() &&
Op.getNode()->getOpcode() != ISD::CopyFromReg &&
!IsDebug &&
!(IsClone || IsCloned);
if (isKill) {
unsigned Idx = MIB->getNumOperands();
while (Idx > 0 &&
MIB->getOperand(Idx-1).isReg() &&
MIB->getOperand(Idx-1).isImplicit())
--Idx;
bool isTied = MCID.getOperandConstraint(Idx, MCOI::TIED_TO) != -1;
if (isTied)
isKill = false;
}
MIB.addReg(VReg, getDefRegState(isOptDef) | getKillRegState(isKill) |
getDebugRegState(IsDebug));
}
bool Mips16DAGToDAGISel::selectAddr16(
SDNode *Parent, SDValue Addr, SDValue &Base, SDValue &Offset,
SDValue &Alias) {
EVT ValTy = Addr.getValueType();
Alias = CurDAG->getTargetConstant(0, ValTy);
// if Address is FI, get the TargetFrameIndex.
if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), ValTy);
Offset = CurDAG->getTargetConstant(0, ValTy);
getMips16SPRefReg(Parent, Alias);
return true;
}
// on PIC code Load GA
if (Addr.getOpcode() == MipsISD::Wrapper) {
Base = Addr.getOperand(0);
Offset = Addr.getOperand(1);
return true;
}
if (TM.getRelocationModel() != Reloc::PIC_) {
if ((Addr.getOpcode() == ISD::TargetExternalSymbol ||
Addr.getOpcode() == ISD::TargetGlobalAddress))
return false;
}
// Addresses of the form FI+const or FI|const
if (CurDAG->isBaseWithConstantOffset(Addr)) {
ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1));
if (isInt<16>(CN->getSExtValue())) {
// If the first operand is a FI, get the TargetFI Node
if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>
(Addr.getOperand(0))) {
Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), ValTy);
getMips16SPRefReg(Parent, Alias);
}
else
Base = Addr.getOperand(0);
Offset = CurDAG->getTargetConstant(CN->getZExtValue(), ValTy);
return true;
}
}
// Operand is a result from an ADD.
if (Addr.getOpcode() == ISD::ADD) {
// When loading from constant pools, load the lower address part in
// the instruction itself. Example, instead of:
// lui $2, %hi($CPI1_0)
// addiu $2, $2, %lo($CPI1_0)
// lwc1 $f0, 0($2)
// Generate:
// lui $2, %hi($CPI1_0)
// lwc1 $f0, %lo($CPI1_0)($2)
if (Addr.getOperand(1).getOpcode() == MipsISD::Lo ||
Addr.getOperand(1).getOpcode() == MipsISD::GPRel) {
SDValue Opnd0 = Addr.getOperand(1).getOperand(0);
if (isa<ConstantPoolSDNode>(Opnd0) || isa<GlobalAddressSDNode>(Opnd0) ||
isa<JumpTableSDNode>(Opnd0)) {
Base = Addr.getOperand(0);
Offset = Opnd0;
return true;
}
}
// If an indexed floating point load/store can be emitted, return false.
const LSBaseSDNode *LS = dyn_cast<LSBaseSDNode>(Parent);
if (LS) {
if (LS->getMemoryVT() == MVT::f32 && Subtarget->hasMips4_32r2())
return false;
if (LS->getMemoryVT() == MVT::f64 && Subtarget->hasMips4_32r2())
return false;
}
}
Base = Addr;
Offset = CurDAG->getTargetConstant(0, ValTy);
return true;
}
SDValue X86SelectionDAGInfo::EmitTargetCodeForMemset(
SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Val,
SDValue Size, unsigned Align, bool isVolatile,
MachinePointerInfo DstPtrInfo) const {
ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
const X86Subtarget &Subtarget =
DAG.getMachineFunction().getSubtarget<X86Subtarget>();
#ifndef NDEBUG
// If the base register might conflict with our physical registers, bail out.
const MCPhysReg ClobberSet[] = {X86::RCX, X86::RAX, X86::RDI,
X86::ECX, X86::EAX, X86::EDI};
assert(!isBaseRegConflictPossible(DAG, ClobberSet));
#endif
// If to a segment-relative address space, use the default lowering.
if (DstPtrInfo.getAddrSpace() >= 256)
return SDValue();
// If not DWORD aligned or size is more than the threshold, call the library.
// The libc version is likely to be faster for these cases. It can use the
// address value and run time information about the CPU.
if ((Align & 3) != 0 || !ConstantSize ||
ConstantSize->getZExtValue() > Subtarget.getMaxInlineSizeThreshold()) {
// Check to see if there is a specialized entry-point for memory zeroing.
ConstantSDNode *ValC = dyn_cast<ConstantSDNode>(Val);
if (const char *bzeroName = (ValC && ValC->isNullValue())
? DAG.getTargetLoweringInfo().getLibcallName(RTLIB::BZERO)
: nullptr) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout());
Type *IntPtrTy = DAG.getDataLayout().getIntPtrType(*DAG.getContext());
TargetLowering::ArgListTy Args;
TargetLowering::ArgListEntry Entry;
Entry.Node = Dst;
Entry.Ty = IntPtrTy;
Args.push_back(Entry);
Entry.Node = Size;
Args.push_back(Entry);
TargetLowering::CallLoweringInfo CLI(DAG);
CLI.setDebugLoc(dl)
.setChain(Chain)
.setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
DAG.getExternalSymbol(bzeroName, IntPtr),
std::move(Args))
.setDiscardResult();
std::pair<SDValue,SDValue> CallResult = TLI.LowerCallTo(CLI);
return CallResult.second;
}
// Otherwise have the target-independent code call memset.
return SDValue();
}
uint64_t SizeVal = ConstantSize->getZExtValue();
SDValue InFlag;
EVT AVT;
SDValue Count;
ConstantSDNode *ValC = dyn_cast<ConstantSDNode>(Val);
unsigned BytesLeft = 0;
if (ValC) {
unsigned ValReg;
uint64_t Val = ValC->getZExtValue() & 255;
// If the value is a constant, then we can potentially use larger sets.
switch (Align & 3) {
case 2: // WORD aligned
AVT = MVT::i16;
ValReg = X86::AX;
Val = (Val << 8) | Val;
break;
case 0: // DWORD aligned
AVT = MVT::i32;
ValReg = X86::EAX;
Val = (Val << 8) | Val;
Val = (Val << 16) | Val;
if (Subtarget.is64Bit() && ((Align & 0x7) == 0)) { // QWORD aligned
AVT = MVT::i64;
ValReg = X86::RAX;
Val = (Val << 32) | Val;
}
break;
default: // Byte aligned
AVT = MVT::i8;
ValReg = X86::AL;
Count = DAG.getIntPtrConstant(SizeVal, dl);
break;
}
if (AVT.bitsGT(MVT::i8)) {
unsigned UBytes = AVT.getSizeInBits() / 8;
Count = DAG.getIntPtrConstant(SizeVal / UBytes, dl);
BytesLeft = SizeVal % UBytes;
}
Chain = DAG.getCopyToReg(Chain, dl, ValReg, DAG.getConstant(Val, dl, AVT),
InFlag);
InFlag = Chain.getValue(1);
} else {
AVT = MVT::i8;
Count = DAG.getIntPtrConstant(SizeVal, dl);
Chain = DAG.getCopyToReg(Chain, dl, X86::AL, Val, InFlag);
InFlag = Chain.getValue(1);
}
bool Use64BitRegs = Subtarget.isTarget64BitLP64();
Chain = DAG.getCopyToReg(Chain, dl, Use64BitRegs ? X86::RCX : X86::ECX,
Count, InFlag);
InFlag = Chain.getValue(1);
Chain = DAG.getCopyToReg(Chain, dl, Use64BitRegs ? X86::RDI : X86::EDI,
Dst, InFlag);
InFlag = Chain.getValue(1);
SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
SDValue Ops[] = { Chain, DAG.getValueType(AVT), InFlag };
Chain = DAG.getNode(X86ISD::REP_STOS, dl, Tys, Ops);
if (BytesLeft) {
// Handle the last 1 - 7 bytes.
unsigned Offset = SizeVal - BytesLeft;
EVT AddrVT = Dst.getValueType();
EVT SizeVT = Size.getValueType();
Chain = DAG.getMemset(Chain, dl,
DAG.getNode(ISD::ADD, dl, AddrVT, Dst,
DAG.getConstant(Offset, dl, AddrVT)),
Val,
DAG.getConstant(BytesLeft, dl, SizeVT),
Align, isVolatile, false,
DstPtrInfo.getWithOffset(Offset));
}
// TODO: Use a Tokenfactor, as in memcpy, instead of a single chain.
return Chain;
}
void Mips16TargetLowering::
getOpndList(SmallVectorImpl<SDValue> &Ops,
std::deque< std::pair<unsigned, SDValue> > &RegsToPass,
bool IsPICCall, bool GlobalOrExternal, bool InternalLinkage,
CallLoweringInfo &CLI, SDValue Callee, SDValue Chain) const {
SelectionDAG &DAG = CLI.DAG;
MachineFunction &MF = DAG.getMachineFunction();
MipsFunctionInfo *FuncInfo = MF.getInfo<MipsFunctionInfo>();
const char* Mips16HelperFunction = 0;
bool NeedMips16Helper = false;
if (getTargetMachine().Options.UseSoftFloat &&
Subtarget->inMips16HardFloat()) {
//
// currently we don't have symbols tagged with the mips16 or mips32
// qualifier so we will assume that we don't know what kind it is.
// and generate the helper
//
bool LookupHelper = true;
if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(CLI.Callee)) {
Mips16Libcall Find = { RTLIB::UNKNOWN_LIBCALL, S->getSymbol() };
if (std::binary_search(HardFloatLibCalls, array_endof(HardFloatLibCalls),
Find))
LookupHelper = false;
else {
Mips16IntrinsicHelperType IntrinsicFind = {S->getSymbol(), ""};
// one more look at list of intrinsics
if (std::binary_search(Mips16IntrinsicHelper,
array_endof(Mips16IntrinsicHelper),
IntrinsicFind)) {
const Mips16IntrinsicHelperType *h =(std::find(Mips16IntrinsicHelper,
array_endof(Mips16IntrinsicHelper),
IntrinsicFind));
Mips16HelperFunction = h->Helper;
NeedMips16Helper = true;
LookupHelper = false;
}
}
} else if (GlobalAddressSDNode *G =
dyn_cast<GlobalAddressSDNode>(CLI.Callee)) {
Mips16Libcall Find = { RTLIB::UNKNOWN_LIBCALL,
G->getGlobal()->getName().data() };
if (std::binary_search(HardFloatLibCalls, array_endof(HardFloatLibCalls),
Find))
LookupHelper = false;
}
if (LookupHelper) Mips16HelperFunction =
getMips16HelperFunction(CLI.RetTy, CLI.Args, NeedMips16Helper);
}
SDValue JumpTarget = Callee;
// T9 should contain the address of the callee function if
// -reloction-model=pic or it is an indirect call.
if (IsPICCall || !GlobalOrExternal) {
unsigned V0Reg = Mips::V0;
if (NeedMips16Helper) {
RegsToPass.push_front(std::make_pair(V0Reg, Callee));
JumpTarget = DAG.getExternalSymbol(Mips16HelperFunction, getPointerTy());
ExternalSymbolSDNode *S = cast<ExternalSymbolSDNode>(JumpTarget);
JumpTarget = getAddrGlobal(S, JumpTarget.getValueType(), DAG,
MipsII::MO_GOT, Chain,
FuncInfo->callPtrInfo(S->getSymbol()));
} else
RegsToPass.push_front(std::make_pair((unsigned)Mips::T9, Callee));
}
Ops.push_back(JumpTarget);
MipsTargetLowering::getOpndList(Ops, RegsToPass, IsPICCall, GlobalOrExternal,
InternalLinkage, CLI, Callee, Chain);
}
/// Lower deopt state and gc pointer arguments of the statepoint. The actual
/// lowering is described in lowerIncomingStatepointValue. This function is
/// responsible for lowering everything in the right position and playing some
/// tricks to avoid redundant stack manipulation where possible. On
/// completion, 'Ops' will contain ready to use operands for machine code
/// statepoint. The chain nodes will have already been created and the DAG root
/// will be set to the last value spilled (if any were).
static void
lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
SelectionDAGBuilder::StatepointLoweringInfo &SI,
SelectionDAGBuilder &Builder) {
// Lower the deopt and gc arguments for this statepoint. Layout will be:
// deopt argument length, deopt arguments.., gc arguments...
#ifndef NDEBUG
if (auto *GFI = Builder.GFI) {
// Check that each of the gc pointer and bases we've gotten out of the
// safepoint is something the strategy thinks might be a pointer (or vector
// of pointers) into the GC heap. This is basically just here to help catch
// errors during statepoint insertion. TODO: This should actually be in the
// Verifier, but we can't get to the GCStrategy from there (yet).
GCStrategy &S = GFI->getStrategy();
for (const Value *V : SI.Bases) {
auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
if (Opt.hasValue()) {
assert(Opt.getValue() &&
"non gc managed base pointer found in statepoint");
}
}
for (const Value *V : SI.Ptrs) {
auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
if (Opt.hasValue()) {
assert(Opt.getValue() &&
"non gc managed derived pointer found in statepoint");
}
}
assert(SI.Bases.size() == SI.Ptrs.size() && "Pointer without base!");
} else {
assert(SI.Bases.empty() && "No gc specified, so cannot relocate pointers!");
assert(SI.Ptrs.empty() && "No gc specified, so cannot relocate pointers!");
}
#endif
// Figure out what lowering strategy we're going to use for each part
// Note: Is is conservatively correct to lower both "live-in" and "live-out"
// as "live-through". A "live-through" variable is one which is "live-in",
// "live-out", and live throughout the lifetime of the call (i.e. we can find
// it from any PC within the transitive callee of the statepoint). In
// particular, if the callee spills callee preserved registers we may not
// be able to find a value placed in that register during the call. This is
// fine for live-out, but not for live-through. If we were willing to make
// assumptions about the code generator producing the callee, we could
// potentially allow live-through values in callee saved registers.
const bool LiveInDeopt =
SI.StatepointFlags & (uint64_t)StatepointFlags::DeoptLiveIn;
auto isGCValue =[&](const Value *V) {
return is_contained(SI.Ptrs, V) || is_contained(SI.Bases, V);
};
// Before we actually start lowering (and allocating spill slots for values),
// reserve any stack slots which we judge to be profitable to reuse for a
// particular value. This is purely an optimization over the code below and
// doesn't change semantics at all. It is important for performance that we
// reserve slots for both deopt and gc values before lowering either.
for (const Value *V : SI.DeoptState) {
if (!LiveInDeopt || isGCValue(V))
reservePreviousStackSlotForValue(V, Builder);
}
for (unsigned i = 0; i < SI.Bases.size(); ++i) {
reservePreviousStackSlotForValue(SI.Bases[i], Builder);
reservePreviousStackSlotForValue(SI.Ptrs[i], Builder);
}
// First, prefix the list with the number of unique values to be
// lowered. Note that this is the number of *Values* not the
// number of SDValues required to lower them.
const int NumVMSArgs = SI.DeoptState.size();
pushStackMapConstant(Ops, Builder, NumVMSArgs);
// The vm state arguments are lowered in an opaque manner. We do not know
// what type of values are contained within.
for (const Value *V : SI.DeoptState) {
SDValue Incoming = Builder.getValue(V);
const bool LiveInValue = LiveInDeopt && !isGCValue(V);
lowerIncomingStatepointValue(Incoming, LiveInValue, Ops, Builder);
}
// Finally, go ahead and lower all the gc arguments. There's no prefixed
// length for this one. After lowering, we'll have the base and pointer
// arrays interwoven with each (lowered) base pointer immediately followed by
// it's (lowered) derived pointer. i.e
// (base[0], ptr[0], base[1], ptr[1], ...)
for (unsigned i = 0; i < SI.Bases.size(); ++i) {
const Value *Base = SI.Bases[i];
lowerIncomingStatepointValue(Builder.getValue(Base), /*LiveInOnly*/ false,
Ops, Builder);
const Value *Ptr = SI.Ptrs[i];
lowerIncomingStatepointValue(Builder.getValue(Ptr), /*LiveInOnly*/ false,
Ops, Builder);
}
// If there are any explicit spill slots passed to the statepoint, record
// them, but otherwise do not do anything special. These are user provided
// allocas and give control over placement to the consumer. In this case,
// it is the contents of the slot which may get updated, not the pointer to
// the alloca
for (Value *V : SI.GCArgs) {
SDValue Incoming = Builder.getValue(V);
if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
// This handles allocas as arguments to the statepoint
assert(Incoming.getValueType() == Builder.getFrameIndexTy() &&
"Incoming value is a frame index!");
Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
Builder.getFrameIndexTy()));
}
}
// Record computed locations for all lowered values.
// This can not be embedded in lowering loops as we need to record *all*
// values, while previous loops account only values with unique SDValues.
const Instruction *StatepointInstr = SI.StatepointInstr;
auto &SpillMap = Builder.FuncInfo.StatepointSpillMaps[StatepointInstr];
for (const GCRelocateInst *Relocate : SI.GCRelocates) {
const Value *V = Relocate->getDerivedPtr();
SDValue SDV = Builder.getValue(V);
SDValue Loc = Builder.StatepointLowering.getLocation(SDV);
if (Loc.getNode()) {
SpillMap.SlotMap[V] = cast<FrameIndexSDNode>(Loc)->getIndex();
} else {
// Record value as visited, but not spilled. This is case for allocas
// and constants. For this values we can avoid emitting spill load while
// visiting corresponding gc_relocate.
// Actually we do not need to record them in this map at all.
// We do this only to check that we are not relocating any unvisited
// value.
SpillMap.SlotMap[V] = None;
// Default llvm mechanisms for exporting values which are used in
// different basic blocks does not work for gc relocates.
// Note that it would be incorrect to teach llvm that all relocates are
// uses of the corresponding values so that it would automatically
// export them. Relocates of the spilled values does not use original
// value.
if (Relocate->getParent() != StatepointInstr->getParent())
Builder.ExportFromCurrentBlock(V);
}
}
}
SDValue
SparcTargetLowering::LowerCall(SDValue Chain, SDValue Callee,
CallingConv::ID CallConv, bool isVarArg,
bool isTailCall,
const SmallVectorImpl<ISD::OutputArg> &Outs,
const SmallVectorImpl<ISD::InputArg> &Ins,
DebugLoc dl, SelectionDAG &DAG,
SmallVectorImpl<SDValue> &InVals) {
#if 0
// Analyze operands of the call, assigning locations to each operand.
SmallVector<CCValAssign, 16> ArgLocs;
CCState CCInfo(CallConv, isVarArg, DAG.getTarget(), ArgLocs);
CCInfo.AnalyzeCallOperands(Outs, CC_Sparc32);
// Get the size of the outgoing arguments stack space requirement.
unsigned ArgsSize = CCInfo.getNextStackOffset();
// FIXME: We can't use this until f64 is known to take two GPRs.
#else
(void)CC_Sparc32;
// Count the size of the outgoing arguments.
unsigned ArgsSize = 0;
for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
switch (Outs[i].Val.getValueType().getSimpleVT().SimpleTy) {
default: llvm_unreachable("Unknown value type!");
case MVT::i1:
case MVT::i8:
case MVT::i16:
case MVT::i32:
case MVT::f32:
ArgsSize += 4;
break;
case MVT::i64:
case MVT::f64:
ArgsSize += 8;
break;
}
}
if (ArgsSize > 4*6)
ArgsSize -= 4*6; // Space for first 6 arguments is prereserved.
else
ArgsSize = 0;
#endif
// Keep stack frames 8-byte aligned.
ArgsSize = (ArgsSize+7) & ~7;
Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(ArgsSize, true));
SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
SmallVector<SDValue, 8> MemOpChains;
#if 0
// Walk the register/memloc assignments, inserting copies/loads.
for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
CCValAssign &VA = ArgLocs[i];
SDValue Arg = Outs[i].Val;
// Promote the value if needed.
switch (VA.getLocInfo()) {
default: llvm_unreachable("Unknown loc info!");
case CCValAssign::Full: break;
case CCValAssign::SExt:
Arg = DAG.getNode(ISD::SIGN_EXTEND, VA.getLocVT(), Arg);
break;
case CCValAssign::ZExt:
Arg = DAG.getNode(ISD::ZERO_EXTEND, VA.getLocVT(), Arg);
break;
case CCValAssign::AExt:
Arg = DAG.getNode(ISD::ANY_EXTEND, VA.getLocVT(), Arg);
break;
}
// Arguments that can be passed on register must be kept at
// RegsToPass vector
if (VA.isRegLoc()) {
RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
continue;
}
assert(VA.isMemLoc());
// Create a store off the stack pointer for this argument.
SDValue StackPtr = DAG.getRegister(SP::O6, MVT::i32);
// FIXME: VERIFY THAT 68 IS RIGHT.
SDValue PtrOff = DAG.getIntPtrConstant(VA.getLocMemOffset()+68);
PtrOff = DAG.getNode(ISD::ADD, MVT::i32, StackPtr, PtrOff);
MemOpChains.push_back(DAG.getStore(Chain, Arg, PtrOff, NULL, 0));
}
#else
static const unsigned ArgRegs[] = {
SP::I0, SP::I1, SP::I2, SP::I3, SP::I4, SP::I5
};
unsigned ArgOffset = 68;
for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
SDValue Val = Outs[i].Val;
EVT ObjectVT = Val.getValueType();
SDValue ValToStore(0, 0);
unsigned ObjSize;
switch (ObjectVT.getSimpleVT().SimpleTy) {
default: llvm_unreachable("Unhandled argument type!");
case MVT::i32:
ObjSize = 4;
if (RegsToPass.size() >= 6) {
ValToStore = Val;
} else {
RegsToPass.push_back(std::make_pair(ArgRegs[RegsToPass.size()], Val));
}
break;
case MVT::f32:
ObjSize = 4;
if (RegsToPass.size() >= 6) {
ValToStore = Val;
} else {
// Convert this to a FP value in an int reg.
Val = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::i32, Val);
RegsToPass.push_back(std::make_pair(ArgRegs[RegsToPass.size()], Val));
}
break;
case MVT::f64: {
ObjSize = 8;
if (RegsToPass.size() >= 6) {
ValToStore = Val; // Whole thing is passed in memory.
break;
}
// Break into top and bottom parts by storing to the stack and loading
// out the parts as integers. Top part goes in a reg.
SDValue StackPtr = DAG.CreateStackTemporary(MVT::f64, MVT::i32);
SDValue Store = DAG.getStore(DAG.getEntryNode(), dl,
Val, StackPtr, NULL, 0);
// Sparc is big-endian, so the high part comes first.
SDValue Hi = DAG.getLoad(MVT::i32, dl, Store, StackPtr, NULL, 0, 0);
// Increment the pointer to the other half.
StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
DAG.getIntPtrConstant(4));
// Load the low part.
SDValue Lo = DAG.getLoad(MVT::i32, dl, Store, StackPtr, NULL, 0, 0);
RegsToPass.push_back(std::make_pair(ArgRegs[RegsToPass.size()], Hi));
if (RegsToPass.size() >= 6) {
ValToStore = Lo;
ArgOffset += 4;
ObjSize = 4;
} else {
RegsToPass.push_back(std::make_pair(ArgRegs[RegsToPass.size()], Lo));
}
break;
}
case MVT::i64: {
ObjSize = 8;
if (RegsToPass.size() >= 6) {
ValToStore = Val; // Whole thing is passed in memory.
break;
}
// Split the value into top and bottom part. Top part goes in a reg.
SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Val,
DAG.getConstant(1, MVT::i32));
SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Val,
DAG.getConstant(0, MVT::i32));
RegsToPass.push_back(std::make_pair(ArgRegs[RegsToPass.size()], Hi));
if (RegsToPass.size() >= 6) {
ValToStore = Lo;
ArgOffset += 4;
ObjSize = 4;
} else {
RegsToPass.push_back(std::make_pair(ArgRegs[RegsToPass.size()], Lo));
}
break;
}
}
if (ValToStore.getNode()) {
SDValue StackPtr = DAG.getRegister(SP::O6, MVT::i32);
SDValue PtrOff = DAG.getConstant(ArgOffset, MVT::i32);
PtrOff = DAG.getNode(ISD::ADD, dl, MVT::i32, StackPtr, PtrOff);
MemOpChains.push_back(DAG.getStore(Chain, dl, ValToStore,
PtrOff, NULL, 0));
}
ArgOffset += ObjSize;
}
#endif
// Emit all stores, make sure the occur before any copies into physregs.
if (!MemOpChains.empty())
Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
&MemOpChains[0], MemOpChains.size());
// Build a sequence of copy-to-reg nodes chained together with token
// chain and flag operands which copy the outgoing args into registers.
// The InFlag in necessary since all emited instructions must be
// stuck together.
SDValue InFlag;
for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
unsigned Reg = RegsToPass[i].first;
// Remap I0->I7 -> O0->O7.
if (Reg >= SP::I0 && Reg <= SP::I7)
Reg = Reg-SP::I0+SP::O0;
Chain = DAG.getCopyToReg(Chain, dl, Reg, RegsToPass[i].second, InFlag);
InFlag = Chain.getValue(1);
}
// If the callee is a GlobalAddress node (quite common, every direct call is)
// turn it into a TargetGlobalAddress node so that legalize doesn't hack it.
// Likewise ExternalSymbol -> TargetExternalSymbol.
if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee))
Callee = DAG.getTargetGlobalAddress(G->getGlobal(), MVT::i32);
else if (ExternalSymbolSDNode *E = dyn_cast<ExternalSymbolSDNode>(Callee))
Callee = DAG.getTargetExternalSymbol(E->getSymbol(), MVT::i32);
std::vector<EVT> NodeTys;
NodeTys.push_back(MVT::Other); // Returns a chain
NodeTys.push_back(MVT::Flag); // Returns a flag for retval copy to use.
SDValue Ops[] = { Chain, Callee, InFlag };
Chain = DAG.getNode(SPISD::CALL, dl, NodeTys, Ops, InFlag.getNode() ? 3 : 2);
InFlag = Chain.getValue(1);
Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(ArgsSize, true),
DAG.getIntPtrConstant(0, true), InFlag);
InFlag = Chain.getValue(1);
// Assign locations to each value returned by this call.
SmallVector<CCValAssign, 16> RVLocs;
CCState RVInfo(CallConv, isVarArg, DAG.getTarget(),
RVLocs, *DAG.getContext());
RVInfo.AnalyzeCallResult(Ins, RetCC_Sparc32);
// Copy all of the result registers out of their specified physreg.
for (unsigned i = 0; i != RVLocs.size(); ++i) {
unsigned Reg = RVLocs[i].getLocReg();
// Remap I0->I7 -> O0->O7.
if (Reg >= SP::I0 && Reg <= SP::I7)
Reg = Reg-SP::I0+SP::O0;
Chain = DAG.getCopyFromReg(Chain, dl, Reg,
RVLocs[i].getValVT(), InFlag).getValue(1);
InFlag = Chain.getValue(2);
InVals.push_back(Chain.getValue(0));
}
return Chain;
}
void DAGTypeLegalizer::ExpandRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi) {
EVT OutVT = N->getValueType(0);
EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
SDValue InOp = N->getOperand(0);
EVT InVT = InOp.getValueType();
DebugLoc dl = N->getDebugLoc();
// Handle some special cases efficiently.
switch (getTypeAction(InVT)) {
default:
assert(false && "Unknown type action!");
case TargetLowering::TypeLegal:
case TargetLowering::TypePromoteInteger:
break;
case TargetLowering::TypeSoftenFloat:
// Convert the integer operand instead.
SplitInteger(GetSoftenedFloat(InOp), Lo, Hi);
Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
return;
case TargetLowering::TypeExpandInteger:
case TargetLowering::TypeExpandFloat:
// Convert the expanded pieces of the input.
GetExpandedOp(InOp, Lo, Hi);
Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
return;
case TargetLowering::TypeSplitVector:
GetSplitVector(InOp, Lo, Hi);
if (TLI.isBigEndian())
std::swap(Lo, Hi);
Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
return;
case TargetLowering::TypeScalarizeVector:
// Convert the element instead.
SplitInteger(BitConvertToInteger(GetScalarizedVector(InOp)), Lo, Hi);
Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
return;
case TargetLowering::TypeWidenVector: {
assert(!(InVT.getVectorNumElements() & 1) && "Unsupported BITCAST");
InOp = GetWidenedVector(InOp);
EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(),
InVT.getVectorNumElements()/2);
Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp,
DAG.getIntPtrConstant(0));
Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp,
DAG.getIntPtrConstant(InNVT.getVectorNumElements()));
if (TLI.isBigEndian())
std::swap(Lo, Hi);
Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
return;
}
}
if (InVT.isVector() && OutVT.isInteger()) {
// Handle cases like i64 = BITCAST v1i64 on x86, where the operand
// is legal but the result is not.
EVT NVT = EVT::getVectorVT(*DAG.getContext(), NOutVT, 2);
if (isTypeLegal(NVT)) {
SDValue CastInOp = DAG.getNode(ISD::BITCAST, dl, NVT, InOp);
Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NOutVT, CastInOp,
DAG.getIntPtrConstant(0));
Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NOutVT, CastInOp,
DAG.getIntPtrConstant(1));
if (TLI.isBigEndian())
std::swap(Lo, Hi);
return;
}
}
// Lower the bit-convert to a store/load from the stack.
assert(NOutVT.isByteSized() && "Expanded type not byte sized!");
// Create the stack frame object. Make sure it is aligned for both
// the source and expanded destination types.
unsigned Alignment =
TLI.getTargetData()->getPrefTypeAlignment(NOutVT.
getTypeForEVT(*DAG.getContext()));
SDValue StackPtr = DAG.CreateStackTemporary(InVT, Alignment);
int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(SPFI);
// Emit a store to the stack slot.
SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, InOp, StackPtr, PtrInfo,
false, false, 0);
// Load the first half from the stack slot.
Lo = DAG.getLoad(NOutVT, dl, Store, StackPtr, PtrInfo, false, false, 0);
// Increment the pointer to the other half.
unsigned IncrementSize = NOutVT.getSizeInBits() / 8;
StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
DAG.getIntPtrConstant(IncrementSize));
// Load the second half from the stack slot.
Hi = DAG.getLoad(NOutVT, dl, Store, StackPtr,
PtrInfo.getWithOffset(IncrementSize), false,
false, MinAlign(Alignment, IncrementSize));
// Handle endianness of the load.
if (TLI.isBigEndian())
std::swap(Lo, Hi);
}
/// MatchAddress - Add the specified node to the specified addressing mode,
/// returning true if it cannot be done. This just pattern matches for the
/// addressing mode.
bool SystemZDAGToDAGISel::MatchAddress(SDValue N, SystemZRRIAddressMode &AM,
bool is12Bit, unsigned Depth) {
DebugLoc dl = N.getDebugLoc();
DEBUG(errs() << "MatchAddress: "; AM.dump());
// Limit recursion.
if (Depth > 5)
return MatchAddressBase(N, AM);
// FIXME: We can perform better here. If we have something like
// (shift (add A, imm), N), we can try to reassociate stuff and fold shift of
// imm into addressing mode.
switch (N.getOpcode()) {
default: break;
case ISD::Constant: {
int64_t Val = cast<ConstantSDNode>(N)->getSExtValue();
int64_t Imm = 0;
bool Match = (is12Bit ?
isImmZExt12(AM.Disp + Val, Imm) :
isImmSExt20(AM.Disp + Val, Imm));
if (Match) {
AM.Disp = Imm;
return false;
}
break;
}
case ISD::FrameIndex:
if (AM.BaseType == SystemZRRIAddressMode::RegBase &&
AM.Base.Reg.getNode() == 0) {
AM.BaseType = SystemZRRIAddressMode::FrameIndexBase;
AM.Base.FrameIndex = cast<FrameIndexSDNode>(N)->getIndex();
return false;
}
break;
case ISD::SUB: {
// Given A-B, if A can be completely folded into the address and
// the index field with the index field unused, use -B as the index.
// This is a win if a has multiple parts that can be folded into
// the address. Also, this saves a mov if the base register has
// other uses, since it avoids a two-address sub instruction, however
// it costs an additional mov if the index register has other uses.
// Test if the LHS of the sub can be folded.
SystemZRRIAddressMode Backup = AM;
if (MatchAddress(N.getNode()->getOperand(0), AM, is12Bit, Depth+1)) {
AM = Backup;
break;
}
// Test if the index field is free for use.
if (AM.IndexReg.getNode() || AM.isRI) {
AM = Backup;
break;
}
// If the base is a register with multiple uses, this transformation may
// save a mov. Otherwise it's probably better not to do it.
if (AM.BaseType == SystemZRRIAddressMode::RegBase &&
(!AM.Base.Reg.getNode() || AM.Base.Reg.getNode()->hasOneUse())) {
AM = Backup;
break;
}
// Ok, the transformation is legal and appears profitable. Go for it.
SDValue RHS = N.getNode()->getOperand(1);
SDValue Zero = CurDAG->getConstant(0, N.getValueType());
SDValue Neg = CurDAG->getNode(ISD::SUB, dl, N.getValueType(), Zero, RHS);
AM.IndexReg = Neg;
// Insert the new nodes into the topological ordering.
if (Zero.getNode()->getNodeId() == -1 ||
Zero.getNode()->getNodeId() > N.getNode()->getNodeId()) {
CurDAG->RepositionNode(N.getNode(), Zero.getNode());
Zero.getNode()->setNodeId(N.getNode()->getNodeId());
}
if (Neg.getNode()->getNodeId() == -1 ||
Neg.getNode()->getNodeId() > N.getNode()->getNodeId()) {
CurDAG->RepositionNode(N.getNode(), Neg.getNode());
Neg.getNode()->setNodeId(N.getNode()->getNodeId());
}
return false;
}
case ISD::ADD: {
SystemZRRIAddressMode Backup = AM;
if (!MatchAddress(N.getNode()->getOperand(0), AM, is12Bit, Depth+1) &&
!MatchAddress(N.getNode()->getOperand(1), AM, is12Bit, Depth+1))
return false;
AM = Backup;
if (!MatchAddress(N.getNode()->getOperand(1), AM, is12Bit, Depth+1) &&
!MatchAddress(N.getNode()->getOperand(0), AM, is12Bit, Depth+1))
return false;
AM = Backup;
// If we couldn't fold both operands into the address at the same time,
// see if we can just put each operand into a register and fold at least
// the add.
if (!AM.isRI &&
AM.BaseType == SystemZRRIAddressMode::RegBase &&
!AM.Base.Reg.getNode() && !AM.IndexReg.getNode()) {
AM.Base.Reg = N.getNode()->getOperand(0);
AM.IndexReg = N.getNode()->getOperand(1);
return false;
}
break;
}
case ISD::OR:
// Handle "X | C" as "X + C" iff X is known to have C bits clear.
if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
SystemZRRIAddressMode Backup = AM;
int64_t Offset = CN->getSExtValue();
int64_t Imm = 0;
bool MatchOffset = (is12Bit ?
isImmZExt12(AM.Disp + Offset, Imm) :
isImmSExt20(AM.Disp + Offset, Imm));
// The resultant disp must fit in 12 or 20-bits.
if (MatchOffset &&
// LHS should be an addr mode.
!MatchAddress(N.getOperand(0), AM, is12Bit, Depth+1) &&
// Check to see if the LHS & C is zero.
CurDAG->MaskedValueIsZero(N.getOperand(0), CN->getAPIntValue())) {
AM.Disp = Imm;
return false;
}
AM = Backup;
}
break;
}
return MatchAddressBase(N, AM);
}
std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForStrlen(
SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src,
MachinePointerInfo SrcPtrInfo) const {
EVT PtrVT = Src.getValueType();
return getBoundedStrlen(DAG, DL, Chain, Src, DAG.getConstant(0, DL, PtrVT));
}
void DAGTypeLegalizer::ExpandRes_BIT_CONVERT(SDNode *N, SDValue &Lo,
SDValue &Hi) {
MVT OutVT = N->getValueType(0);
MVT NOutVT = TLI.getTypeToTransformTo(OutVT);
SDValue InOp = N->getOperand(0);
MVT InVT = InOp.getValueType();
DebugLoc dl = N->getDebugLoc();
// Handle some special cases efficiently.
switch (getTypeAction(InVT)) {
default:
assert(false && "Unknown type action!");
case Legal:
case PromoteInteger:
break;
case SoftenFloat:
// Convert the integer operand instead.
SplitInteger(GetSoftenedFloat(InOp), Lo, Hi);
Lo = DAG.getNode(ISD::BIT_CONVERT, dl, NOutVT, Lo);
Hi = DAG.getNode(ISD::BIT_CONVERT, dl, NOutVT, Hi);
return;
case ExpandInteger:
case ExpandFloat:
// Convert the expanded pieces of the input.
GetExpandedOp(InOp, Lo, Hi);
Lo = DAG.getNode(ISD::BIT_CONVERT, dl, NOutVT, Lo);
Hi = DAG.getNode(ISD::BIT_CONVERT, dl, NOutVT, Hi);
return;
case SplitVector:
// Convert the split parts of the input if it was split in two.
GetSplitVector(InOp, Lo, Hi);
if (Lo.getValueType() == Hi.getValueType()) {
if (TLI.isBigEndian())
std::swap(Lo, Hi);
Lo = DAG.getNode(ISD::BIT_CONVERT, dl, NOutVT, Lo);
Hi = DAG.getNode(ISD::BIT_CONVERT, dl, NOutVT, Hi);
return;
}
break;
case ScalarizeVector:
// Convert the element instead.
SplitInteger(BitConvertToInteger(GetScalarizedVector(InOp)), Lo, Hi);
Lo = DAG.getNode(ISD::BIT_CONVERT, dl, NOutVT, Lo);
Hi = DAG.getNode(ISD::BIT_CONVERT, dl, NOutVT, Hi);
return;
case WidenVector: {
assert(!(InVT.getVectorNumElements() & 1) && "Unsupported BIT_CONVERT");
InOp = GetWidenedVector(InOp);
MVT InNVT = MVT::getVectorVT(InVT.getVectorElementType(),
InVT.getVectorNumElements()/2);
Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp,
DAG.getIntPtrConstant(0));
Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp,
DAG.getIntPtrConstant(InNVT.getVectorNumElements()));
if (TLI.isBigEndian())
std::swap(Lo, Hi);
Lo = DAG.getNode(ISD::BIT_CONVERT, dl, NOutVT, Lo);
Hi = DAG.getNode(ISD::BIT_CONVERT, dl, NOutVT, Hi);
return;
}
}
// Lower the bit-convert to a store/load from the stack.
assert(NOutVT.isByteSized() && "Expanded type not byte sized!");
// Create the stack frame object. Make sure it is aligned for both
// the source and expanded destination types.
unsigned Alignment =
TLI.getTargetData()->getPrefTypeAlignment(NOutVT.getTypeForMVT());
SDValue StackPtr = DAG.CreateStackTemporary(InVT, Alignment);
int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
const Value *SV = PseudoSourceValue::getFixedStack(SPFI);
// Emit a store to the stack slot.
SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, InOp, StackPtr, SV, 0);
// Load the first half from the stack slot.
Lo = DAG.getLoad(NOutVT, dl, Store, StackPtr, SV, 0);
// Increment the pointer to the other half.
unsigned IncrementSize = NOutVT.getSizeInBits() / 8;
StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
DAG.getIntPtrConstant(IncrementSize));
// Load the second half from the stack slot.
Hi = DAG.getLoad(NOutVT, dl, Store, StackPtr, SV, IncrementSize, false,
MinAlign(Alignment, IncrementSize));
// Handle endianness of the load.
if (TLI.isBigEndian())
std::swap(Lo, Hi);
}
SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemset(
SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst,
SDValue Byte, SDValue Size, unsigned Align, bool IsVolatile,
MachinePointerInfo DstPtrInfo) const {
EVT PtrVT = Dst.getValueType();
if (IsVolatile)
return SDValue();
if (auto *CSize = dyn_cast<ConstantSDNode>(Size)) {
uint64_t Bytes = CSize->getZExtValue();
if (Bytes == 0)
return SDValue();
if (auto *CByte = dyn_cast<ConstantSDNode>(Byte)) {
// Handle cases that can be done using at most two of
// MVI, MVHI, MVHHI and MVGHI. The latter two can only be
// used if ByteVal is all zeros or all ones; in other casees,
// we can move at most 2 halfwords.
uint64_t ByteVal = CByte->getZExtValue();
if (ByteVal == 0 || ByteVal == 255 ?
Bytes <= 16 && countPopulation(Bytes) <= 2 :
Bytes <= 4) {
unsigned Size1 = Bytes == 16 ? 8 : 1 << findLastSet(Bytes);
unsigned Size2 = Bytes - Size1;
SDValue Chain1 = memsetStore(DAG, DL, Chain, Dst, ByteVal, Size1,
Align, DstPtrInfo);
if (Size2 == 0)
return Chain1;
Dst = DAG.getNode(ISD::ADD, DL, PtrVT, Dst,
DAG.getConstant(Size1, DL, PtrVT));
DstPtrInfo = DstPtrInfo.getWithOffset(Size1);
SDValue Chain2 = memsetStore(DAG, DL, Chain, Dst, ByteVal, Size2,
std::min(Align, Size1), DstPtrInfo);
return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chain1, Chain2);
}
} else {
// Handle one and two bytes using STC.
if (Bytes <= 2) {
SDValue Chain1 = DAG.getStore(Chain, DL, Byte, Dst, DstPtrInfo, Align);
if (Bytes == 1)
return Chain1;
SDValue Dst2 = DAG.getNode(ISD::ADD, DL, PtrVT, Dst,
DAG.getConstant(1, DL, PtrVT));
SDValue Chain2 =
DAG.getStore(Chain, DL, Byte, Dst2, DstPtrInfo.getWithOffset(1),
/* Alignment = */ 1);
return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chain1, Chain2);
}
}
assert(Bytes >= 2 && "Should have dealt with 0- and 1-byte cases already");
// Handle the special case of a memset of 0, which can use XC.
auto *CByte = dyn_cast<ConstantSDNode>(Byte);
if (CByte && CByte->getZExtValue() == 0)
return emitMemMem(DAG, DL, SystemZISD::XC, SystemZISD::XC_LOOP,
Chain, Dst, Dst, Bytes);
// Copy the byte to the first location and then use MVC to copy
// it to the rest.
Chain = DAG.getStore(Chain, DL, Byte, Dst, DstPtrInfo, Align);
SDValue DstPlus1 = DAG.getNode(ISD::ADD, DL, PtrVT, Dst,
DAG.getConstant(1, DL, PtrVT));
return emitMemMem(DAG, DL, SystemZISD::MVC, SystemZISD::MVC_LOOP,
Chain, DstPlus1, Dst, Bytes - 1);
}
return SDValue();
}
std::pair<SDValue, SDValue>
IA64TargetLowering::LowerCallTo(SDValue Chain, const Type *RetTy,
bool RetSExt, bool RetZExt, bool isVarArg,
bool isInreg, unsigned CallingConv,
bool isTailCall, SDValue Callee,
ArgListTy &Args, SelectionDAG &DAG,
DebugLoc dl) {
MachineFunction &MF = DAG.getMachineFunction();
unsigned NumBytes = 16;
unsigned outRegsUsed = 0;
if (Args.size() > 8) {
NumBytes += (Args.size() - 8) * 8;
outRegsUsed = 8;
} else {
outRegsUsed = Args.size();
}
// FIXME? this WILL fail if we ever try to pass around an arg that
// consumes more than a single output slot (a 'real' double, int128
// some sort of aggregate etc.), as we'll underestimate how many 'outX'
// registers we use. Hopefully, the assembler will notice.
MF.getInfo<IA64FunctionInfo>()->outRegsUsed=
std::max(outRegsUsed, MF.getInfo<IA64FunctionInfo>()->outRegsUsed);
// keep stack frame 16-byte aligned
// assert(NumBytes==((NumBytes+15) & ~15) &&
// "stack frame not 16-byte aligned!");
NumBytes = (NumBytes+15) & ~15;
Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes, true));
SDValue StackPtr;
std::vector<SDValue> Stores;
std::vector<SDValue> Converts;
std::vector<SDValue> RegValuesToPass;
unsigned ArgOffset = 16;
for (unsigned i = 0, e = Args.size(); i != e; ++i)
{
SDValue Val = Args[i].Node;
MVT ObjectVT = Val.getValueType();
SDValue ValToStore(0, 0), ValToConvert(0, 0);
unsigned ObjSize=8;
switch (ObjectVT.getSimpleVT()) {
default: assert(0 && "unexpected argument type!");
case MVT::i1:
case MVT::i8:
case MVT::i16:
case MVT::i32: {
//promote to 64-bits, sign/zero extending based on type
//of the argument
ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
if (Args[i].isSExt)
ExtendKind = ISD::SIGN_EXTEND;
else if (Args[i].isZExt)
ExtendKind = ISD::ZERO_EXTEND;
Val = DAG.getNode(ExtendKind, dl, MVT::i64, Val);
// XXX: fall through
}
case MVT::i64:
//ObjSize = 8;
if(RegValuesToPass.size() >= 8) {
ValToStore = Val;
} else {
RegValuesToPass.push_back(Val);
}
break;
case MVT::f32:
//promote to 64-bits
Val = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Val);
// XXX: fall through
case MVT::f64:
if(RegValuesToPass.size() >= 8) {
ValToStore = Val;
} else {
RegValuesToPass.push_back(Val);
if(1 /* TODO: if(calling external or varadic function)*/ ) {
ValToConvert = Val; // additionally pass this FP value as an int
}
}
break;
}
if(ValToStore.getNode()) {
if(!StackPtr.getNode()) {
StackPtr = DAG.getRegister(IA64::r12, MVT::i64);
}
SDValue PtrOff = DAG.getConstant(ArgOffset, getPointerTy());
PtrOff = DAG.getNode(ISD::ADD, dl, MVT::i64, StackPtr, PtrOff);
Stores.push_back(DAG.getStore(Chain, dl, ValToStore, PtrOff, NULL, 0));
ArgOffset += ObjSize;
}
if(ValToConvert.getNode()) {
Converts.push_back(DAG.getNode(IA64ISD::GETFD, dl,
MVT::i64, ValToConvert));
}
}
// Emit all stores, make sure they occur before any copies into physregs.
if (!Stores.empty())
Chain = DAG.getNode(ISD::TokenFactor, dl,
MVT::Other, &Stores[0],Stores.size());
static const unsigned IntArgRegs[] = {
IA64::out0, IA64::out1, IA64::out2, IA64::out3,
IA64::out4, IA64::out5, IA64::out6, IA64::out7
};
static const unsigned FPArgRegs[] = {
IA64::F8, IA64::F9, IA64::F10, IA64::F11,
IA64::F12, IA64::F13, IA64::F14, IA64::F15
};
SDValue InFlag;
// save the current GP, SP and RP : FIXME: do we need to do all 3 always?
SDValue GPBeforeCall = DAG.getCopyFromReg(Chain, IA64::r1, MVT::i64, InFlag);
Chain = GPBeforeCall.getValue(1);
InFlag = Chain.getValue(2);
SDValue SPBeforeCall = DAG.getCopyFromReg(Chain, IA64::r12, MVT::i64, InFlag);
Chain = SPBeforeCall.getValue(1);
InFlag = Chain.getValue(2);
SDValue RPBeforeCall = DAG.getCopyFromReg(Chain, IA64::rp, MVT::i64, InFlag);
Chain = RPBeforeCall.getValue(1);
InFlag = Chain.getValue(2);
// Build a sequence of copy-to-reg nodes chained together with token chain
// and flag operands which copy the outgoing integer args into regs out[0-7]
// mapped 1:1 and the FP args into regs F8-F15 "lazily"
// TODO: for performance, we should only copy FP args into int regs when we
// know this is required (i.e. for varardic or external (unknown) functions)
// first to the FP->(integer representation) conversions, these are
// flagged for now, but shouldn't have to be (TODO)
unsigned seenConverts = 0;
for (unsigned i = 0, e = RegValuesToPass.size(); i != e; ++i) {
if(RegValuesToPass[i].getValueType().isFloatingPoint()) {
Chain = DAG.getCopyToReg(Chain, IntArgRegs[i], Converts[seenConverts++],
InFlag);
InFlag = Chain.getValue(1);
}
}
// next copy args into the usual places, these are flagged
unsigned usedFPArgs = 0;
for (unsigned i = 0, e = RegValuesToPass.size(); i != e; ++i) {
Chain = DAG.getCopyToReg(Chain,
RegValuesToPass[i].getValueType().isInteger() ?
IntArgRegs[i] : FPArgRegs[usedFPArgs++], RegValuesToPass[i], InFlag);
InFlag = Chain.getValue(1);
}
// If the callee is a GlobalAddress node (quite common, every direct call is)
// turn it into a TargetGlobalAddress node so that legalize doesn't hack it.
/*
if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
Callee = DAG.getTargetGlobalAddress(G->getGlobal(), MVT::i64);
}
*/
std::vector<MVT> NodeTys;
std::vector<SDValue> CallOperands;
NodeTys.push_back(MVT::Other); // Returns a chain
NodeTys.push_back(MVT::Flag); // Returns a flag for retval copy to use.
CallOperands.push_back(Chain);
CallOperands.push_back(Callee);
// emit the call itself
if (InFlag.getNode())
CallOperands.push_back(InFlag);
else
assert(0 && "this should never happen!\n");
// to make way for a hack:
Chain = DAG.getNode(IA64ISD::BRCALL, dl, NodeTys,
&CallOperands[0], CallOperands.size());
InFlag = Chain.getValue(1);
// restore the GP, SP and RP after the call
Chain = DAG.getCopyToReg(Chain, IA64::r1, GPBeforeCall, InFlag);
InFlag = Chain.getValue(1);
Chain = DAG.getCopyToReg(Chain, IA64::r12, SPBeforeCall, InFlag);
InFlag = Chain.getValue(1);
Chain = DAG.getCopyToReg(Chain, IA64::rp, RPBeforeCall, InFlag);
InFlag = Chain.getValue(1);
std::vector<MVT> RetVals;
RetVals.push_back(MVT::Other);
RetVals.push_back(MVT::Flag);
MVT RetTyVT = getValueType(RetTy);
SDValue RetVal;
if (RetTyVT != MVT::isVoid) {
switch (RetTyVT.getSimpleVT()) {
default: assert(0 && "Unknown value type to return!");
case MVT::i1: { // bools are just like other integers (returned in r8)
// we *could* fall through to the truncate below, but this saves a
// few redundant predicate ops
SDValue boolInR8 = DAG.getCopyFromReg(Chain, IA64::r8, MVT::i64,InFlag);
InFlag = boolInR8.getValue(2);
Chain = boolInR8.getValue(1);
SDValue zeroReg = DAG.getCopyFromReg(Chain, IA64::r0, MVT::i64, InFlag);
InFlag = zeroReg.getValue(2);
Chain = zeroReg.getValue(1);
RetVal = DAG.getSetCC(dl, MVT::i1, boolInR8, zeroReg, ISD::SETNE);
break;
}
case MVT::i8:
case MVT::i16:
case MVT::i32:
RetVal = DAG.getCopyFromReg(Chain, IA64::r8, MVT::i64, InFlag);
Chain = RetVal.getValue(1);
// keep track of whether it is sign or zero extended (todo: bools?)
/* XXX
RetVal = DAG.getNode(RetTy->isSigned() ? ISD::AssertSext :ISD::AssertZext,
dl, MVT::i64, RetVal, DAG.getValueType(RetTyVT));
*/
RetVal = DAG.getNode(ISD::TRUNCATE, dl, RetTyVT, RetVal);
break;
case MVT::i64:
RetVal = DAG.getCopyFromReg(Chain, IA64::r8, MVT::i64, InFlag);
Chain = RetVal.getValue(1);
InFlag = RetVal.getValue(2); // XXX dead
break;
case MVT::f32:
RetVal = DAG.getCopyFromReg(Chain, IA64::F8, MVT::f64, InFlag);
Chain = RetVal.getValue(1);
RetVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, RetVal,
DAG.getIntPtrConstant(0));
break;
case MVT::f64:
RetVal = DAG.getCopyFromReg(Chain, IA64::F8, MVT::f64, InFlag);
Chain = RetVal.getValue(1);
InFlag = RetVal.getValue(2); // XXX dead
break;
}
}
Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, true),
DAG.getIntPtrConstant(0, true), SDValue());
return std::make_pair(RetVal, Chain);
}
/// ComplexPattern used on MipsInstrInfo
/// Used on Mips Load/Store instructions
bool MipsDAGToDAGISel::
SelectAddr(SDValue Addr, SDValue &Base, SDValue &Offset) {
EVT ValTy = Addr.getValueType();
unsigned GPReg = ValTy == MVT::i32 ? Mips::GP : Mips::GP_64;
// if Address is FI, get the TargetFrameIndex.
if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), ValTy);
Offset = CurDAG->getTargetConstant(0, ValTy);
return true;
}
// on PIC code Load GA
if (TM.getRelocationModel() == Reloc::PIC_) {
if (Addr.getOpcode() == MipsISD::WrapperPIC) {
Base = CurDAG->getRegister(GPReg, ValTy);
Offset = Addr.getOperand(0);
return true;
}
} else {
if ((Addr.getOpcode() == ISD::TargetExternalSymbol ||
Addr.getOpcode() == ISD::TargetGlobalAddress))
return false;
else if (Addr.getOpcode() == ISD::TargetGlobalTLSAddress) {
Base = CurDAG->getRegister(GPReg, ValTy);
Offset = Addr;
return true;
}
}
// Addresses of the form FI+const or FI|const
if (CurDAG->isBaseWithConstantOffset(Addr)) {
ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1));
if (isInt<16>(CN->getSExtValue())) {
// If the first operand is a FI, get the TargetFI Node
if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>
(Addr.getOperand(0)))
Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), ValTy);
else
Base = Addr.getOperand(0);
Offset = CurDAG->getTargetConstant(CN->getZExtValue(), ValTy);
return true;
}
}
// Operand is a result from an ADD.
if (Addr.getOpcode() == ISD::ADD) {
// When loading from constant pools, load the lower address part in
// the instruction itself. Example, instead of:
// lui $2, %hi($CPI1_0)
// addiu $2, $2, %lo($CPI1_0)
// lwc1 $f0, 0($2)
// Generate:
// lui $2, %hi($CPI1_0)
// lwc1 $f0, %lo($CPI1_0)($2)
if ((Addr.getOperand(0).getOpcode() == MipsISD::Hi ||
Addr.getOperand(0).getOpcode() == ISD::LOAD) &&
Addr.getOperand(1).getOpcode() == MipsISD::Lo) {
SDValue LoVal = Addr.getOperand(1);
if (isa<ConstantPoolSDNode>(LoVal.getOperand(0)) ||
isa<GlobalAddressSDNode>(LoVal.getOperand(0))) {
Base = Addr.getOperand(0);
Offset = LoVal.getOperand(0);
return true;
}
}
}
Base = Addr;
Offset = CurDAG->getTargetConstant(0, ValTy);
return true;
}
/// AddOperand - Add the specified operand to the specified machine instr. II
/// specifies the instruction information for the node, and IIOpNum is the
/// operand number (in the II) that we are adding.
void InstrEmitter::AddOperand(MachineInstrBuilder &MIB,
SDValue Op,
unsigned IIOpNum,
const MCInstrDesc *II,
DenseMap<SDValue, unsigned> &VRBaseMap,
bool IsDebug, bool IsClone, bool IsCloned) {
if (Op.isMachineOpcode()) {
AddRegisterOperand(MIB, Op, IIOpNum, II, VRBaseMap,
IsDebug, IsClone, IsCloned);
} else if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
MIB.addImm(C->getSExtValue());
} else if (ConstantFPSDNode *F = dyn_cast<ConstantFPSDNode>(Op)) {
MIB.addFPImm(F->getConstantFPValue());
} else if (RegisterSDNode *R = dyn_cast<RegisterSDNode>(Op)) {
// Turn additional physreg operands into implicit uses on non-variadic
// instructions. This is used by call and return instructions passing
// arguments in registers.
bool Imp = II && (IIOpNum >= II->getNumOperands() && !II->isVariadic());
MIB.addReg(R->getReg(), getImplRegState(Imp));
} else if (RegisterMaskSDNode *RM = dyn_cast<RegisterMaskSDNode>(Op)) {
MIB.addRegMask(RM->getRegMask());
} else if (GlobalAddressSDNode *TGA = dyn_cast<GlobalAddressSDNode>(Op)) {
MIB.addGlobalAddress(TGA->getGlobal(), TGA->getOffset(),
TGA->getTargetFlags());
} else if (BasicBlockSDNode *BBNode = dyn_cast<BasicBlockSDNode>(Op)) {
MIB.addMBB(BBNode->getBasicBlock());
} else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op)) {
MIB.addFrameIndex(FI->getIndex());
} else if (JumpTableSDNode *JT = dyn_cast<JumpTableSDNode>(Op)) {
MIB.addJumpTableIndex(JT->getIndex(), JT->getTargetFlags());
} else if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(Op)) {
int Offset = CP->getOffset();
unsigned Align = CP->getAlignment();
Type *Type = CP->getType();
// MachineConstantPool wants an explicit alignment.
if (Align == 0) {
Align = TM->getDataLayout()->getPrefTypeAlignment(Type);
if (Align == 0) {
// Alignment of vector types. FIXME!
Align = TM->getDataLayout()->getTypeAllocSize(Type);
}
}
unsigned Idx;
MachineConstantPool *MCP = MF->getConstantPool();
if (CP->isMachineConstantPoolEntry())
Idx = MCP->getConstantPoolIndex(CP->getMachineCPVal(), Align);
else
Idx = MCP->getConstantPoolIndex(CP->getConstVal(), Align);
MIB.addConstantPoolIndex(Idx, Offset, CP->getTargetFlags());
} else if (ExternalSymbolSDNode *ES = dyn_cast<ExternalSymbolSDNode>(Op)) {
MIB.addExternalSymbol(ES->getSymbol(), ES->getTargetFlags());
} else if (BlockAddressSDNode *BA = dyn_cast<BlockAddressSDNode>(Op)) {
MIB.addBlockAddress(BA->getBlockAddress(),
BA->getOffset(),
BA->getTargetFlags());
} else if (TargetIndexSDNode *TI = dyn_cast<TargetIndexSDNode>(Op)) {
MIB.addTargetIndex(TI->getIndex(), TI->getOffset(), TI->getTargetFlags());
} else {
assert(Op.getValueType() != MVT::Other &&
Op.getValueType() != MVT::Glue &&
"Chain and glue operands should occur at end of operand list!");
AddRegisterOperand(MIB, Op, IIOpNum, II, VRBaseMap,
IsDebug, IsClone, IsCloned);
}
}
bool MipsSEDAGToDAGISel::selectAddrDefault(SDValue Addr, SDValue &Base,
SDValue &Offset) const {
Base = Addr;
Offset = CurDAG->getTargetConstant(0, Addr.getValueType());
return true;
}
bool Mips16DAGToDAGISel::selectAddr(bool SPAllowed, SDValue Addr, SDValue &Base,
SDValue &Offset) {
SDLoc DL(Addr);
EVT ValTy = Addr.getValueType();
// if Address is FI, get the TargetFrameIndex.
if (SPAllowed) {
if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), ValTy);
Offset = CurDAG->getTargetConstant(0, DL, ValTy);
return true;
}
}
// on PIC code Load GA
if (Addr.getOpcode() == MipsISD::Wrapper) {
Base = Addr.getOperand(0);
Offset = Addr.getOperand(1);
return true;
}
if (!TM.isPositionIndependent()) {
if ((Addr.getOpcode() == ISD::TargetExternalSymbol ||
Addr.getOpcode() == ISD::TargetGlobalAddress))
return false;
}
// Addresses of the form FI+const or FI|const
if (CurDAG->isBaseWithConstantOffset(Addr)) {
ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1));
if (isInt<16>(CN->getSExtValue())) {
// If the first operand is a FI, get the TargetFI Node
if (SPAllowed) {
if (FrameIndexSDNode *FIN =
dyn_cast<FrameIndexSDNode>(Addr.getOperand(0))) {
Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), ValTy);
Offset = CurDAG->getTargetConstant(CN->getZExtValue(), DL, ValTy);
return true;
}
}
Base = Addr.getOperand(0);
Offset = CurDAG->getTargetConstant(CN->getZExtValue(), DL, ValTy);
return true;
}
}
// Operand is a result from an ADD.
if (Addr.getOpcode() == ISD::ADD) {
// When loading from constant pools, load the lower address part in
// the instruction itself. Example, instead of:
// lui $2, %hi($CPI1_0)
// addiu $2, $2, %lo($CPI1_0)
// lwc1 $f0, 0($2)
// Generate:
// lui $2, %hi($CPI1_0)
// lwc1 $f0, %lo($CPI1_0)($2)
if (Addr.getOperand(1).getOpcode() == MipsISD::Lo ||
Addr.getOperand(1).getOpcode() == MipsISD::GPRel) {
SDValue Opnd0 = Addr.getOperand(1).getOperand(0);
if (isa<ConstantPoolSDNode>(Opnd0) || isa<GlobalAddressSDNode>(Opnd0) ||
isa<JumpTableSDNode>(Opnd0)) {
Base = Addr.getOperand(0);
Offset = Opnd0;
return true;
}
}
}
Base = Addr;
Offset = CurDAG->getTargetConstant(0, DL, ValTy);
return true;
}
SDValue
X86SelectionDAGInfo::EmitTargetCodeForMemcpy(SelectionDAG &DAG, SDLoc dl,
SDValue Chain, SDValue Dst, SDValue Src,
SDValue Size, unsigned Align,
bool isVolatile, bool AlwaysInline,
MachinePointerInfo DstPtrInfo,
MachinePointerInfo SrcPtrInfo) const {
// This requires the copy size to be a constant, preferably
// within a subtarget-specific limit.
ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
if (!ConstantSize)
return SDValue();
uint64_t SizeVal = ConstantSize->getZExtValue();
if (!AlwaysInline && SizeVal > Subtarget->getMaxInlineSizeThreshold())
return SDValue();
/// If not DWORD aligned, it is more efficient to call the library. However
/// if calling the library is not allowed (AlwaysInline), then soldier on as
/// the code generated here is better than the long load-store sequence we
/// would otherwise get.
if (!AlwaysInline && (Align & 3) != 0)
return SDValue();
// If to a segment-relative address space, use the default lowering.
if (DstPtrInfo.getAddrSpace() >= 256 ||
SrcPtrInfo.getAddrSpace() >= 256)
return SDValue();
// ESI might be used as a base pointer, in that case we can't simply overwrite
// the register. Fall back to generic code.
const X86RegisterInfo *TRI =
static_cast<const X86RegisterInfo *>(DAG.getTarget().getRegisterInfo());
if (TRI->hasBasePointer(DAG.getMachineFunction()) &&
TRI->getBaseRegister() == X86::ESI)
return SDValue();
MVT AVT;
if (Align & 1)
AVT = MVT::i8;
else if (Align & 2)
AVT = MVT::i16;
else if (Align & 4)
// DWORD aligned
AVT = MVT::i32;
else
// QWORD aligned
AVT = Subtarget->is64Bit() ? MVT::i64 : MVT::i32;
unsigned UBytes = AVT.getSizeInBits() / 8;
unsigned CountVal = SizeVal / UBytes;
SDValue Count = DAG.getIntPtrConstant(CountVal);
unsigned BytesLeft = SizeVal % UBytes;
SDValue InFlag(0, 0);
Chain = DAG.getCopyToReg(Chain, dl, Subtarget->is64Bit() ? X86::RCX :
X86::ECX,
Count, InFlag);
InFlag = Chain.getValue(1);
Chain = DAG.getCopyToReg(Chain, dl, Subtarget->is64Bit() ? X86::RDI :
X86::EDI,
Dst, InFlag);
InFlag = Chain.getValue(1);
Chain = DAG.getCopyToReg(Chain, dl, Subtarget->is64Bit() ? X86::RSI :
X86::ESI,
Src, InFlag);
InFlag = Chain.getValue(1);
SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
SDValue Ops[] = { Chain, DAG.getValueType(AVT), InFlag };
SDValue RepMovs = DAG.getNode(X86ISD::REP_MOVS, dl, Tys, Ops,
array_lengthof(Ops));
SmallVector<SDValue, 4> Results;
Results.push_back(RepMovs);
if (BytesLeft) {
// Handle the last 1 - 7 bytes.
unsigned Offset = SizeVal - BytesLeft;
EVT DstVT = Dst.getValueType();
EVT SrcVT = Src.getValueType();
EVT SizeVT = Size.getValueType();
Results.push_back(DAG.getMemcpy(Chain, dl,
DAG.getNode(ISD::ADD, dl, DstVT, Dst,
DAG.getConstant(Offset, DstVT)),
DAG.getNode(ISD::ADD, dl, SrcVT, Src,
DAG.getConstant(Offset, SrcVT)),
DAG.getConstant(BytesLeft, SizeVT),
Align, isVolatile, AlwaysInline,
DstPtrInfo.getWithOffset(Offset),
SrcPtrInfo.getWithOffset(Offset)));
}
return DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
&Results[0], Results.size());
}
