/// Lower deopt state and gc pointer arguments of the statepoint. The actual /// lowering is described in lowerIncomingStatepointValue. This function is /// responsible for lowering everything in the right position and playing some /// tricks to avoid redundant stack manipulation where possible. On /// completion, 'Ops' will contain ready to use operands for machine code /// statepoint. The chain nodes will have already been created and the DAG root /// will be set to the last value spilled (if any were). static void lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops, ImmutableStatepoint StatepointSite, SelectionDAGBuilder &Builder) { // Lower the deopt and gc arguments for this statepoint. Layout will // be: deopt argument length, deopt arguments.., gc arguments... SmallVector<const Value *, 64> Bases, Ptrs, Relocations; getIncomingStatepointGCValues(Bases, Ptrs, Relocations, StatepointSite, Builder); #ifndef NDEBUG // Check that each of the gc pointer and bases we've gotten out of the // safepoint is something the strategy thinks might be a pointer into the GC // heap. This is basically just here to help catch errors during statepoint // insertion. TODO: This should actually be in the Verifier, but we can't get // to the GCStrategy from there (yet). GCStrategy &S = Builder.GFI->getStrategy(); for (const Value *V : Bases) { auto Opt = S.isGCManagedPointer(V); if (Opt.hasValue()) { assert(Opt.getValue() && "non gc managed base pointer found in statepoint"); } } for (const Value *V : Ptrs) { auto Opt = S.isGCManagedPointer(V); if (Opt.hasValue()) { assert(Opt.getValue() && "non gc managed derived pointer found in statepoint"); } } for (const Value *V : Relocations) { auto Opt = S.isGCManagedPointer(V); if (Opt.hasValue()) { assert(Opt.getValue() && "non gc managed pointer relocated"); } } #endif // Before we actually start lowering (and allocating spill slots for values), // reserve any stack slots which we judge to be profitable to reuse for a // particular value. This is purely an optimization over the code below and // doesn't change semantics at all. It is important for performance that we // reserve slots for both deopt and gc values before lowering either. for (const Value *V : StatepointSite.vm_state_args()) { reservePreviousStackSlotForValue(V, Builder); } for (unsigned i = 0; i < Bases.size(); ++i) { reservePreviousStackSlotForValue(Bases[i], Builder); reservePreviousStackSlotForValue(Ptrs[i], Builder); } // First, prefix the list with the number of unique values to be // lowered. Note that this is the number of *Values* not the // number of SDValues required to lower them. const int NumVMSArgs = StatepointSite.getNumTotalVMSArgs(); pushStackMapConstant(Ops, Builder, NumVMSArgs); assert(NumVMSArgs == std::distance(StatepointSite.vm_state_begin(), StatepointSite.vm_state_end())); // The vm state arguments are lowered in an opaque manner. We do // not know what type of values are contained within. We skip the // first one since that happens to be the total number we lowered // explicitly just above. We could have left it in the loop and // not done it explicitly, but it's far easier to understand this // way. for (const Value *V : StatepointSite.vm_state_args()) { SDValue Incoming = Builder.getValue(V); lowerIncomingStatepointValue(Incoming, Ops, Builder); } // Finally, go ahead and lower all the gc arguments. There's no prefixed // length for this one. After lowering, we'll have the base and pointer // arrays interwoven with each (lowered) base pointer immediately followed by // it's (lowered) derived pointer. i.e // (base[0], ptr[0], base[1], ptr[1], ...) for (unsigned i = 0; i < Bases.size(); ++i) { const Value *Base = Bases[i]; lowerIncomingStatepointValue(Builder.getValue(Base), Ops, Builder); const Value *Ptr = Ptrs[i]; lowerIncomingStatepointValue(Builder.getValue(Ptr), Ops, Builder); } // If there are any explicit spill slots passed to the statepoint, record // them, but otherwise do not do anything special. These are user provided // allocas and give control over placement to the consumer. In this case, // it is the contents of the slot which may get updated, not the pointer to // the alloca for (Value *V : StatepointSite.gc_args()) { SDValue Incoming = Builder.getValue(V); if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) { // This handles allocas as arguments to the statepoint Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(), Incoming.getValueType())); } } // Record computed locations for all lowered values. // This can not be embedded in lowering loops as we need to record *all* // values, while previous loops account only values with unique SDValues. const Instruction *StatepointInstr = StatepointSite.getCallSite().getInstruction(); FunctionLoweringInfo::StatepointSpilledValueMapTy &SpillMap = Builder.FuncInfo.StatepointRelocatedValues[StatepointInstr]; for (GCRelocateOperands RelocateOpers : StatepointSite.getRelocates()) { const Value *V = RelocateOpers.getDerivedPtr(); SDValue SDV = Builder.getValue(V); SDValue Loc = Builder.StatepointLowering.getLocation(SDV); if (Loc.getNode()) { SpillMap[V] = cast<FrameIndexSDNode>(Loc)->getIndex(); } else { // Record value as visited, but not spilled. This is case for allocas // and constants. For this values we can avoid emiting spill load while // visiting corresponding gc_relocate. // Actually we do not need to record them in this map at all. // We do this only to check that we are not relocating any unvisited value. SpillMap[V] = None; // Default llvm mechanisms for exporting values which are used in // different basic blocks does not work for gc relocates. // Note that it would be incorrect to teach llvm that all relocates are // uses of the corresponging values so that it would automatically // export them. Relocates of the spilled values does not use original // value. if (StatepointSite.getCallSite().isInvoke()) Builder.ExportFromCurrentBlock(V); } } }
/// Lower deopt state and gc pointer arguments of the statepoint. The actual /// lowering is described in lowerIncomingStatepointValue. This function is /// responsible for lowering everything in the right position and playing some /// tricks to avoid redundant stack manipulation where possible. On /// completion, 'Ops' will contain ready to use operands for machine code /// statepoint. The chain nodes will have already been created and the DAG root /// will be set to the last value spilled (if any were). static void lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops, SelectionDAGBuilder::StatepointLoweringInfo &SI, SelectionDAGBuilder &Builder) { // Lower the deopt and gc arguments for this statepoint. Layout will be: // deopt argument length, deopt arguments.., gc arguments... #ifndef NDEBUG if (auto *GFI = Builder.GFI) { // Check that each of the gc pointer and bases we've gotten out of the // safepoint is something the strategy thinks might be a pointer (or vector // of pointers) into the GC heap. This is basically just here to help catch // errors during statepoint insertion. TODO: This should actually be in the // Verifier, but we can't get to the GCStrategy from there (yet). GCStrategy &S = GFI->getStrategy(); for (const Value *V : SI.Bases) { auto Opt = S.isGCManagedPointer(V->getType()->getScalarType()); if (Opt.hasValue()) { assert(Opt.getValue() && "non gc managed base pointer found in statepoint"); } } for (const Value *V : SI.Ptrs) { auto Opt = S.isGCManagedPointer(V->getType()->getScalarType()); if (Opt.hasValue()) { assert(Opt.getValue() && "non gc managed derived pointer found in statepoint"); } } assert(SI.Bases.size() == SI.Ptrs.size() && "Pointer without base!"); } else { assert(SI.Bases.empty() && "No gc specified, so cannot relocate pointers!"); assert(SI.Ptrs.empty() && "No gc specified, so cannot relocate pointers!"); } #endif // Figure out what lowering strategy we're going to use for each part // Note: Is is conservatively correct to lower both "live-in" and "live-out" // as "live-through". A "live-through" variable is one which is "live-in", // "live-out", and live throughout the lifetime of the call (i.e. we can find // it from any PC within the transitive callee of the statepoint). In // particular, if the callee spills callee preserved registers we may not // be able to find a value placed in that register during the call. This is // fine for live-out, but not for live-through. If we were willing to make // assumptions about the code generator producing the callee, we could // potentially allow live-through values in callee saved registers. const bool LiveInDeopt = SI.StatepointFlags & (uint64_t)StatepointFlags::DeoptLiveIn; auto isGCValue =[&](const Value *V) { return is_contained(SI.Ptrs, V) || is_contained(SI.Bases, V); }; // Before we actually start lowering (and allocating spill slots for values), // reserve any stack slots which we judge to be profitable to reuse for a // particular value. This is purely an optimization over the code below and // doesn't change semantics at all. It is important for performance that we // reserve slots for both deopt and gc values before lowering either. for (const Value *V : SI.DeoptState) { if (!LiveInDeopt || isGCValue(V)) reservePreviousStackSlotForValue(V, Builder); } for (unsigned i = 0; i < SI.Bases.size(); ++i) { reservePreviousStackSlotForValue(SI.Bases[i], Builder); reservePreviousStackSlotForValue(SI.Ptrs[i], Builder); } // First, prefix the list with the number of unique values to be // lowered. Note that this is the number of *Values* not the // number of SDValues required to lower them. const int NumVMSArgs = SI.DeoptState.size(); pushStackMapConstant(Ops, Builder, NumVMSArgs); // The vm state arguments are lowered in an opaque manner. We do not know // what type of values are contained within. for (const Value *V : SI.DeoptState) { SDValue Incoming = Builder.getValue(V); const bool LiveInValue = LiveInDeopt && !isGCValue(V); lowerIncomingStatepointValue(Incoming, LiveInValue, Ops, Builder); } // Finally, go ahead and lower all the gc arguments. There's no prefixed // length for this one. After lowering, we'll have the base and pointer // arrays interwoven with each (lowered) base pointer immediately followed by // it's (lowered) derived pointer. i.e // (base[0], ptr[0], base[1], ptr[1], ...) for (unsigned i = 0; i < SI.Bases.size(); ++i) { const Value *Base = SI.Bases[i]; lowerIncomingStatepointValue(Builder.getValue(Base), /*LiveInOnly*/ false, Ops, Builder); const Value *Ptr = SI.Ptrs[i]; lowerIncomingStatepointValue(Builder.getValue(Ptr), /*LiveInOnly*/ false, Ops, Builder); } // If there are any explicit spill slots passed to the statepoint, record // them, but otherwise do not do anything special. These are user provided // allocas and give control over placement to the consumer. In this case, // it is the contents of the slot which may get updated, not the pointer to // the alloca for (Value *V : SI.GCArgs) { SDValue Incoming = Builder.getValue(V); if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) { // This handles allocas as arguments to the statepoint assert(Incoming.getValueType() == Builder.getFrameIndexTy() && "Incoming value is a frame index!"); Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(), Builder.getFrameIndexTy())); } } // Record computed locations for all lowered values. // This can not be embedded in lowering loops as we need to record *all* // values, while previous loops account only values with unique SDValues. const Instruction *StatepointInstr = SI.StatepointInstr; auto &SpillMap = Builder.FuncInfo.StatepointSpillMaps[StatepointInstr]; for (const GCRelocateInst *Relocate : SI.GCRelocates) { const Value *V = Relocate->getDerivedPtr(); SDValue SDV = Builder.getValue(V); SDValue Loc = Builder.StatepointLowering.getLocation(SDV); if (Loc.getNode()) { SpillMap.SlotMap[V] = cast<FrameIndexSDNode>(Loc)->getIndex(); } else { // Record value as visited, but not spilled. This is case for allocas // and constants. For this values we can avoid emitting spill load while // visiting corresponding gc_relocate. // Actually we do not need to record them in this map at all. // We do this only to check that we are not relocating any unvisited // value. SpillMap.SlotMap[V] = None; // Default llvm mechanisms for exporting values which are used in // different basic blocks does not work for gc relocates. // Note that it would be incorrect to teach llvm that all relocates are // uses of the corresponding values so that it would automatically // export them. Relocates of the spilled values does not use original // value. if (Relocate->getParent() != StatepointInstr->getParent()) Builder.ExportFromCurrentBlock(V); } } }