/// Lower deopt state and gc pointer arguments of the statepoint. The actual
/// lowering is described in lowerIncomingStatepointValue. This function is
/// responsible for lowering everything in the right position and playing some
/// tricks to avoid redundant stack manipulation where possible. On
/// completion, 'Ops' will contain ready to use operands for machine code
/// statepoint. The chain nodes will have already been created and the DAG root
/// will be set to the last value spilled (if any were).
static void lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
ImmutableStatepoint StatepointSite,
SelectionDAGBuilder &Builder) {
// Lower the deopt and gc arguments for this statepoint. Layout will
// be: deopt argument length, deopt arguments.., gc arguments...
SmallVector<const Value *, 64> Bases, Ptrs, Relocations;
getIncomingStatepointGCValues(Bases, Ptrs, Relocations, StatepointSite,
Builder);
#ifndef NDEBUG
// Check that each of the gc pointer and bases we've gotten out of the
// safepoint is something the strategy thinks might be a pointer into the GC
// heap. This is basically just here to help catch errors during statepoint
// insertion. TODO: This should actually be in the Verifier, but we can't get
// to the GCStrategy from there (yet).
GCStrategy &S = Builder.GFI->getStrategy();
for (const Value *V : Bases) {
auto Opt = S.isGCManagedPointer(V);
if (Opt.hasValue()) {
assert(Opt.getValue() &&
"non gc managed base pointer found in statepoint");
}
}
for (const Value *V : Ptrs) {
auto Opt = S.isGCManagedPointer(V);
if (Opt.hasValue()) {
assert(Opt.getValue() &&
"non gc managed derived pointer found in statepoint");
}
}
for (const Value *V : Relocations) {
auto Opt = S.isGCManagedPointer(V);
if (Opt.hasValue()) {
assert(Opt.getValue() && "non gc managed pointer relocated");
}
}
#endif
// Before we actually start lowering (and allocating spill slots for values),
// reserve any stack slots which we judge to be profitable to reuse for a
// particular value. This is purely an optimization over the code below and
// doesn't change semantics at all. It is important for performance that we
// reserve slots for both deopt and gc values before lowering either.
for (const Value *V : StatepointSite.vm_state_args()) {
reservePreviousStackSlotForValue(V, Builder);
}
for (unsigned i = 0; i < Bases.size(); ++i) {
reservePreviousStackSlotForValue(Bases[i], Builder);
reservePreviousStackSlotForValue(Ptrs[i], Builder);
}
// First, prefix the list with the number of unique values to be
// lowered. Note that this is the number of *Values* not the
// number of SDValues required to lower them.
const int NumVMSArgs = StatepointSite.getNumTotalVMSArgs();
pushStackMapConstant(Ops, Builder, NumVMSArgs);
assert(NumVMSArgs == std::distance(StatepointSite.vm_state_begin(),
StatepointSite.vm_state_end()));
// The vm state arguments are lowered in an opaque manner. We do
// not know what type of values are contained within. We skip the
// first one since that happens to be the total number we lowered
// explicitly just above. We could have left it in the loop and
// not done it explicitly, but it's far easier to understand this
// way.
for (const Value *V : StatepointSite.vm_state_args()) {
SDValue Incoming = Builder.getValue(V);
lowerIncomingStatepointValue(Incoming, Ops, Builder);
}
// Finally, go ahead and lower all the gc arguments. There's no prefixed
// length for this one. After lowering, we'll have the base and pointer
// arrays interwoven with each (lowered) base pointer immediately followed by
// it's (lowered) derived pointer. i.e
// (base[0], ptr[0], base[1], ptr[1], ...)
for (unsigned i = 0; i < Bases.size(); ++i) {
const Value *Base = Bases[i];
lowerIncomingStatepointValue(Builder.getValue(Base), Ops, Builder);
const Value *Ptr = Ptrs[i];
lowerIncomingStatepointValue(Builder.getValue(Ptr), Ops, Builder);
}
// If there are any explicit spill slots passed to the statepoint, record
// them, but otherwise do not do anything special. These are user provided
// allocas and give control over placement to the consumer. In this case,
// it is the contents of the slot which may get updated, not the pointer to
// the alloca
for (Value *V : StatepointSite.gc_args()) {
SDValue Incoming = Builder.getValue(V);
if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
// This handles allocas as arguments to the statepoint
Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
Incoming.getValueType()));
}
}
// Record computed locations for all lowered values.
// This can not be embedded in lowering loops as we need to record *all*
// values, while previous loops account only values with unique SDValues.
const Instruction *StatepointInstr =
StatepointSite.getCallSite().getInstruction();
FunctionLoweringInfo::StatepointSpilledValueMapTy &SpillMap =
Builder.FuncInfo.StatepointRelocatedValues[StatepointInstr];
for (GCRelocateOperands RelocateOpers : StatepointSite.getRelocates()) {
const Value *V = RelocateOpers.getDerivedPtr();
SDValue SDV = Builder.getValue(V);
SDValue Loc = Builder.StatepointLowering.getLocation(SDV);
if (Loc.getNode()) {
SpillMap[V] = cast<FrameIndexSDNode>(Loc)->getIndex();
} else {
// Record value as visited, but not spilled. This is case for allocas
// and constants. For this values we can avoid emiting spill load while
// visiting corresponding gc_relocate.
// Actually we do not need to record them in this map at all.
// We do this only to check that we are not relocating any unvisited value.
SpillMap[V] = None;
// Default llvm mechanisms for exporting values which are used in
// different basic blocks does not work for gc relocates.
// Note that it would be incorrect to teach llvm that all relocates are
// uses of the corresponging values so that it would automatically
// export them. Relocates of the spilled values does not use original
// value.
if (StatepointSite.getCallSite().isInvoke())
Builder.ExportFromCurrentBlock(V);
}
}
}
/// Lower deopt state and gc pointer arguments of the statepoint. The actual
/// lowering is described in lowerIncomingStatepointValue. This function is
/// responsible for lowering everything in the right position and playing some
/// tricks to avoid redundant stack manipulation where possible. On
/// completion, 'Ops' will contain ready to use operands for machine code
/// statepoint. The chain nodes will have already been created and the DAG root
/// will be set to the last value spilled (if any were).
static void
lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
SelectionDAGBuilder::StatepointLoweringInfo &SI,
SelectionDAGBuilder &Builder) {
// Lower the deopt and gc arguments for this statepoint. Layout will be:
// deopt argument length, deopt arguments.., gc arguments...
#ifndef NDEBUG
if (auto *GFI = Builder.GFI) {
// Check that each of the gc pointer and bases we've gotten out of the
// safepoint is something the strategy thinks might be a pointer (or vector
// of pointers) into the GC heap. This is basically just here to help catch
// errors during statepoint insertion. TODO: This should actually be in the
// Verifier, but we can't get to the GCStrategy from there (yet).
GCStrategy &S = GFI->getStrategy();
for (const Value *V : SI.Bases) {
auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
if (Opt.hasValue()) {
assert(Opt.getValue() &&
"non gc managed base pointer found in statepoint");
}
}
for (const Value *V : SI.Ptrs) {
auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
if (Opt.hasValue()) {
assert(Opt.getValue() &&
"non gc managed derived pointer found in statepoint");
}
}
assert(SI.Bases.size() == SI.Ptrs.size() && "Pointer without base!");
} else {
assert(SI.Bases.empty() && "No gc specified, so cannot relocate pointers!");
assert(SI.Ptrs.empty() && "No gc specified, so cannot relocate pointers!");
}
#endif
// Figure out what lowering strategy we're going to use for each part
// Note: Is is conservatively correct to lower both "live-in" and "live-out"
// as "live-through". A "live-through" variable is one which is "live-in",
// "live-out", and live throughout the lifetime of the call (i.e. we can find
// it from any PC within the transitive callee of the statepoint). In
// particular, if the callee spills callee preserved registers we may not
// be able to find a value placed in that register during the call. This is
// fine for live-out, but not for live-through. If we were willing to make
// assumptions about the code generator producing the callee, we could
// potentially allow live-through values in callee saved registers.
const bool LiveInDeopt =
SI.StatepointFlags & (uint64_t)StatepointFlags::DeoptLiveIn;
auto isGCValue =[&](const Value *V) {
return is_contained(SI.Ptrs, V) || is_contained(SI.Bases, V);
};
// Before we actually start lowering (and allocating spill slots for values),
// reserve any stack slots which we judge to be profitable to reuse for a
// particular value. This is purely an optimization over the code below and
// doesn't change semantics at all. It is important for performance that we
// reserve slots for both deopt and gc values before lowering either.
for (const Value *V : SI.DeoptState) {
if (!LiveInDeopt || isGCValue(V))
reservePreviousStackSlotForValue(V, Builder);
}
for (unsigned i = 0; i < SI.Bases.size(); ++i) {
reservePreviousStackSlotForValue(SI.Bases[i], Builder);
reservePreviousStackSlotForValue(SI.Ptrs[i], Builder);
}
// First, prefix the list with the number of unique values to be
// lowered. Note that this is the number of *Values* not the
// number of SDValues required to lower them.
const int NumVMSArgs = SI.DeoptState.size();
pushStackMapConstant(Ops, Builder, NumVMSArgs);
// The vm state arguments are lowered in an opaque manner. We do not know
// what type of values are contained within.
for (const Value *V : SI.DeoptState) {
SDValue Incoming = Builder.getValue(V);
const bool LiveInValue = LiveInDeopt && !isGCValue(V);
lowerIncomingStatepointValue(Incoming, LiveInValue, Ops, Builder);
}
// Finally, go ahead and lower all the gc arguments. There's no prefixed
// length for this one. After lowering, we'll have the base and pointer
// arrays interwoven with each (lowered) base pointer immediately followed by
// it's (lowered) derived pointer. i.e
// (base[0], ptr[0], base[1], ptr[1], ...)
for (unsigned i = 0; i < SI.Bases.size(); ++i) {
const Value *Base = SI.Bases[i];
lowerIncomingStatepointValue(Builder.getValue(Base), /*LiveInOnly*/ false,
Ops, Builder);
const Value *Ptr = SI.Ptrs[i];
lowerIncomingStatepointValue(Builder.getValue(Ptr), /*LiveInOnly*/ false,
Ops, Builder);
}
// If there are any explicit spill slots passed to the statepoint, record
// them, but otherwise do not do anything special. These are user provided
// allocas and give control over placement to the consumer. In this case,
// it is the contents of the slot which may get updated, not the pointer to
// the alloca
for (Value *V : SI.GCArgs) {
SDValue Incoming = Builder.getValue(V);
if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
// This handles allocas as arguments to the statepoint
assert(Incoming.getValueType() == Builder.getFrameIndexTy() &&
"Incoming value is a frame index!");
Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
Builder.getFrameIndexTy()));
}
}
// Record computed locations for all lowered values.
// This can not be embedded in lowering loops as we need to record *all*
// values, while previous loops account only values with unique SDValues.
const Instruction *StatepointInstr = SI.StatepointInstr;
auto &SpillMap = Builder.FuncInfo.StatepointSpillMaps[StatepointInstr];
for (const GCRelocateInst *Relocate : SI.GCRelocates) {
const Value *V = Relocate->getDerivedPtr();
SDValue SDV = Builder.getValue(V);
SDValue Loc = Builder.StatepointLowering.getLocation(SDV);
if (Loc.getNode()) {
SpillMap.SlotMap[V] = cast<FrameIndexSDNode>(Loc)->getIndex();
} else {
// Record value as visited, but not spilled. This is case for allocas
// and constants. For this values we can avoid emitting spill load while
// visiting corresponding gc_relocate.
// Actually we do not need to record them in this map at all.
// We do this only to check that we are not relocating any unvisited
// value.
SpillMap.SlotMap[V] = None;
// Default llvm mechanisms for exporting values which are used in
// different basic blocks does not work for gc relocates.
// Note that it would be incorrect to teach llvm that all relocates are
// uses of the corresponding values so that it would automatically
// export them. Relocates of the spilled values does not use original
// value.
if (Relocate->getParent() != StatepointInstr->getParent())
Builder.ExportFromCurrentBlock(V);
}
}
}
