/// Copy an r-value into memory as part of storing to an atomic type.
/// This needs to create a bit-pattern suitable for atomic operations.
void AtomicInfo::emitCopyIntoMemory(RValue rvalue, LValue dest) const {
// If we have an r-value, the rvalue should be of the atomic type,
// which means that the caller is responsible for having zeroed
// any padding. Just do an aggregate copy of that type.
if (rvalue.isAggregate()) {
CGF.EmitAggregateCopy(dest.getAddress(),
rvalue.getAggregateAddr(),
getAtomicType(),
(rvalue.isVolatileQualified()
|| dest.isVolatileQualified()),
dest.getAlignment());
return;
}
// Okay, otherwise we're copying stuff.
// Zero out the buffer if necessary.
emitMemSetZeroIfNecessary(dest);
// Drill past the padding if present.
dest = projectValue(dest);
// Okay, store the rvalue in.
if (rvalue.isScalar()) {
CGF.EmitStoreOfScalar(rvalue.getScalarVal(), dest, /*init*/ true);
} else {
CGF.EmitStoreOfComplex(rvalue.getComplexVal(), dest, /*init*/ true);
}
}
DominatingValue<RValue>::saved_type
DominatingValue<RValue>::saved_type::save(CodeGenFunction &CGF, RValue rv) {
if (rv.isScalar()) {
llvm::Value *V = rv.getScalarVal();
// These automatically dominate and don't need to be saved.
if (!DominatingLLVMValue::needsSaving(V))
return saved_type(V, ScalarLiteral);
// Everything else needs an alloca.
llvm::Value *addr = CGF.CreateTempAlloca(V->getType(), "saved-rvalue");
CGF.Builder.CreateStore(V, addr);
return saved_type(addr, ScalarAddress);
}
if (rv.isComplex()) {
CodeGenFunction::ComplexPairTy V = rv.getComplexVal();
llvm::Type *ComplexTy =
llvm::StructType::get(V.first->getType(), V.second->getType(),
(void*) nullptr);
llvm::Value *addr = CGF.CreateTempAlloca(ComplexTy, "saved-complex");
CGF.Builder.CreateStore(V.first, CGF.Builder.CreateStructGEP(addr, 0));
CGF.Builder.CreateStore(V.second, CGF.Builder.CreateStructGEP(addr, 1));
return saved_type(addr, ComplexAddress);
}
assert(rv.isAggregate());
llvm::Value *V = rv.getAggregateAddr(); // TODO: volatile?
if (!DominatingLLVMValue::needsSaving(V))
return saved_type(V, AggregateLiteral);
llvm::Value *addr = CGF.CreateTempAlloca(V->getType(), "saved-rvalue");
CGF.Builder.CreateStore(V, addr);
return saved_type(addr, AggregateAddress);
}
// Compound assignments.
ComplexPairTy ComplexExprEmitter::
EmitCompoundAssign(const CompoundAssignOperator *E,
ComplexPairTy (ComplexExprEmitter::*Func)(const BinOpInfo&)){
RValue Val;
LValue LV = EmitCompoundAssignLValue(E, Func, Val);
// The result of an assignment in C is the assigned r-value.
if (!CGF.getLangOpts().CPlusPlus)
return Val.getComplexVal();
// If the lvalue is non-volatile, return the computed value of the assignment.
if (!LV.isVolatileQualified())
return Val.getComplexVal();
return EmitLoadOfLValue(LV, E->getExprLoc());
}
/// \brief Emit a libcall for a binary operation on complex types.
ComplexPairTy ComplexExprEmitter::EmitComplexBinOpLibCall(StringRef LibCallName,
const BinOpInfo &Op) {
CallArgList Args;
Args.add(RValue::get(Op.LHS.first),
Op.Ty->castAs<ComplexType>()->getElementType());
Args.add(RValue::get(Op.LHS.second),
Op.Ty->castAs<ComplexType>()->getElementType());
Args.add(RValue::get(Op.RHS.first),
Op.Ty->castAs<ComplexType>()->getElementType());
Args.add(RValue::get(Op.RHS.second),
Op.Ty->castAs<ComplexType>()->getElementType());
// We *must* use the full CG function call building logic here because the
// complex type has special ABI handling. We also should not forget about
// special calling convention which may be used for compiler builtins.
const CGFunctionInfo &FuncInfo =
CGF.CGM.getTypes().arrangeFreeFunctionCall(
Op.Ty, Args, FunctionType::ExtInfo(/* No CC here - will be added later */),
RequiredArgs::All);
llvm::FunctionType *FTy = CGF.CGM.getTypes().GetFunctionType(FuncInfo);
llvm::Constant *Func = CGF.CGM.CreateBuiltinFunction(FTy, LibCallName);
llvm::Instruction *Call;
RValue Res = CGF.EmitCall(FuncInfo, Func, ReturnValueSlot(), Args,
nullptr, &Call);
cast<llvm::CallInst>(Call)->setCallingConv(CGF.CGM.getBuiltinCC());
cast<llvm::CallInst>(Call)->setDoesNotThrow();
return Res.getComplexVal();
}
void CodeGenFunction::EmitReturnOfRValue(RValue RV, QualType Ty) {
if (RV.isScalar()) {
Builder.CreateStore(RV.getScalarVal(), ReturnValue);
} else if (RV.isAggregate()) {
EmitAggregateCopy(ReturnValue, RV.getAggregateAddr(), Ty);
} else {
StoreComplexToAddr(RV.getComplexVal(), ReturnValue, false);
}
EmitBranchThroughCleanup(ReturnBlock);
}
static CodeGenFunction::ComplexPairTy
convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType,
QualType DestType) {
assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
"DestType must have complex evaluation kind.");
CodeGenFunction::ComplexPairTy ComplexVal;
if (Val.isScalar()) {
// Convert the input element to the element type of the complex.
auto DestElementType = DestType->castAs<ComplexType>()->getElementType();
auto ScalarVal =
CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, DestElementType);
ComplexVal = CodeGenFunction::ComplexPairTy(
ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
} else {
assert(Val.isComplex() && "Must be a scalar or complex.");
auto SrcElementType = SrcType->castAs<ComplexType>()->getElementType();
auto DestElementType = DestType->castAs<ComplexType>()->getElementType();
ComplexVal.first = CGF.EmitScalarConversion(
Val.getComplexVal().first, SrcElementType, DestElementType);
ComplexVal.second = CGF.EmitScalarConversion(
Val.getComplexVal().second, SrcElementType, DestElementType);
}
return ComplexVal;
}
DominatingValue<RValue>::saved_type
DominatingValue<RValue>::saved_type::save(CodeGenFunction &CGF, RValue rv) {
if (rv.isScalar()) {
llvm::Value *V = rv.getScalarVal();
// These automatically dominate and don't need to be saved.
if (!DominatingLLVMValue::needsSaving(V))
return saved_type(V, ScalarLiteral);
// Everything else needs an alloca.
Address addr =
CGF.CreateDefaultAlignTempAlloca(V->getType(), "saved-rvalue");
CGF.Builder.CreateStore(V, addr);
return saved_type(addr.getPointer(), ScalarAddress);
}
if (rv.isComplex()) {
CodeGenFunction::ComplexPairTy V = rv.getComplexVal();
llvm::Type *ComplexTy =
llvm::StructType::get(V.first->getType(), V.second->getType(),
(void*) nullptr);
Address addr = CGF.CreateDefaultAlignTempAlloca(ComplexTy, "saved-complex");
CGF.Builder.CreateStore(V.first,
CGF.Builder.CreateStructGEP(addr, 0, CharUnits()));
CharUnits offset = CharUnits::fromQuantity(
CGF.CGM.getDataLayout().getTypeAllocSize(V.first->getType()));
CGF.Builder.CreateStore(V.second,
CGF.Builder.CreateStructGEP(addr, 1, offset));
return saved_type(addr.getPointer(), ComplexAddress);
}
assert(rv.isAggregate());
Address V = rv.getAggregateAddress(); // TODO: volatile?
if (!DominatingLLVMValue::needsSaving(V.getPointer()))
return saved_type(V.getPointer(), AggregateLiteral,
V.getAlignment().getQuantity());
Address addr =
CGF.CreateTempAlloca(V.getType(), CGF.getPointerAlign(), "saved-rvalue");
CGF.Builder.CreateStore(V.getPointer(), addr);
return saved_type(addr.getPointer(), AggregateAddress,
V.getAlignment().getQuantity());
}
/// \brief Emit a libcall for a binary operation on complex types.
ComplexPairTy ComplexExprEmitter::EmitComplexBinOpLibCall(StringRef LibCallName,
const BinOpInfo &Op) {
CallArgList Args;
Args.add(RValue::get(Op.LHS.first),
Op.Ty->castAs<ComplexType>()->getElementType());
Args.add(RValue::get(Op.LHS.second),
Op.Ty->castAs<ComplexType>()->getElementType());
Args.add(RValue::get(Op.RHS.first),
Op.Ty->castAs<ComplexType>()->getElementType());
Args.add(RValue::get(Op.RHS.second),
Op.Ty->castAs<ComplexType>()->getElementType());
// We *must* use the full CG function call building logic here because the
// complex type has special ABI handling. We also should not forget about
// special calling convention which may be used for compiler builtins.
// We create a function qualified type to state that this call does not have
// any exceptions.
FunctionProtoType::ExtProtoInfo EPI;
EPI = EPI.withExceptionSpec(
FunctionProtoType::ExceptionSpecInfo(EST_BasicNoexcept));
SmallVector<QualType, 4> ArgsQTys(
4, Op.Ty->castAs<ComplexType>()->getElementType());
QualType FQTy = CGF.getContext().getFunctionType(Op.Ty, ArgsQTys, EPI);
const CGFunctionInfo &FuncInfo = CGF.CGM.getTypes().arrangeFreeFunctionCall(
Args, cast<FunctionType>(FQTy.getTypePtr()), false);
llvm::FunctionType *FTy = CGF.CGM.getTypes().GetFunctionType(FuncInfo);
llvm::Constant *Func = CGF.CGM.CreateBuiltinFunction(FTy, LibCallName);
CGCallee Callee = CGCallee::forDirect(Func, FQTy->getAs<FunctionProtoType>());
llvm::Instruction *Call;
RValue Res = CGF.EmitCall(FuncInfo, Callee, ReturnValueSlot(), Args, &Call);
cast<llvm::CallInst>(Call)->setCallingConv(CGF.CGM.getBuiltinCC());
return Res.getComplexVal();
}
RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
llvm::Value *Callee,
const CallArgList &CallArgs,
const Decl *TargetDecl) {
// FIXME: We no longer need the types from CallArgs; lift up and simplify.
llvm::SmallVector<llvm::Value*, 16> Args;
// Handle struct-return functions by passing a pointer to the
// location that we would like to return into.
QualType RetTy = CallInfo.getReturnType();
const ABIArgInfo &RetAI = CallInfo.getReturnInfo();
// If the call returns a temporary with struct return, create a temporary
// alloca to hold the result.
if (CGM.ReturnTypeUsesSret(CallInfo))
Args.push_back(CreateTempAlloca(ConvertTypeForMem(RetTy)));
assert(CallInfo.arg_size() == CallArgs.size() &&
"Mismatch between function signature & arguments.");
CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin();
for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();
I != E; ++I, ++info_it) {
const ABIArgInfo &ArgInfo = info_it->info;
RValue RV = I->first;
switch (ArgInfo.getKind()) {
case ABIArgInfo::Indirect:
if (RV.isScalar() || RV.isComplex()) {
// Make a temporary alloca to pass the argument.
Args.push_back(CreateTempAlloca(ConvertTypeForMem(I->second)));
if (RV.isScalar())
EmitStoreOfScalar(RV.getScalarVal(), Args.back(), false, I->second);
else
StoreComplexToAddr(RV.getComplexVal(), Args.back(), false);
} else {
Args.push_back(RV.getAggregateAddr());
}
break;
case ABIArgInfo::Extend:
case ABIArgInfo::Direct:
if (RV.isScalar()) {
Args.push_back(RV.getScalarVal());
} else if (RV.isComplex()) {
llvm::Value *Tmp = llvm::UndefValue::get(ConvertType(I->second));
Tmp = Builder.CreateInsertValue(Tmp, RV.getComplexVal().first, 0);
Tmp = Builder.CreateInsertValue(Tmp, RV.getComplexVal().second, 1);
Args.push_back(Tmp);
} else {
Args.push_back(Builder.CreateLoad(RV.getAggregateAddr()));
}
break;
case ABIArgInfo::Ignore:
break;
case ABIArgInfo::Coerce: {
// FIXME: Avoid the conversion through memory if possible.
llvm::Value *SrcPtr;
if (RV.isScalar()) {
SrcPtr = CreateTempAlloca(ConvertTypeForMem(I->second), "coerce");
EmitStoreOfScalar(RV.getScalarVal(), SrcPtr, false, I->second);
} else if (RV.isComplex()) {
SrcPtr = CreateTempAlloca(ConvertTypeForMem(I->second), "coerce");
StoreComplexToAddr(RV.getComplexVal(), SrcPtr, false);
} else
SrcPtr = RV.getAggregateAddr();
Args.push_back(CreateCoercedLoad(SrcPtr, ArgInfo.getCoerceToType(),
*this));
break;
}
case ABIArgInfo::Expand:
ExpandTypeToArgs(I->second, RV, Args);
break;
}
}
// If the callee is a bitcast of a function to a varargs pointer to function
// type, check to see if we can remove the bitcast. This handles some cases
// with unprototyped functions.
if (llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Callee))
if (llvm::Function *CalleeF = dyn_cast<llvm::Function>(CE->getOperand(0))) {
const llvm::PointerType *CurPT=cast<llvm::PointerType>(Callee->getType());
const llvm::FunctionType *CurFT =
cast<llvm::FunctionType>(CurPT->getElementType());
const llvm::FunctionType *ActualFT = CalleeF->getFunctionType();
if (CE->getOpcode() == llvm::Instruction::BitCast &&
ActualFT->getReturnType() == CurFT->getReturnType() &&
ActualFT->getNumParams() == CurFT->getNumParams() &&
ActualFT->getNumParams() == Args.size()) {
bool ArgsMatch = true;
for (unsigned i = 0, e = ActualFT->getNumParams(); i != e; ++i)
if (ActualFT->getParamType(i) != CurFT->getParamType(i)) {
ArgsMatch = false;
break;
}
// Strip the cast if we can get away with it. This is a nice cleanup,
// but also allows us to inline the function at -O0 if it is marked
// always_inline.
if (ArgsMatch)
Callee = CalleeF;
}
}
llvm::BasicBlock *InvokeDest = getInvokeDest();
unsigned CallingConv;
CodeGen::AttributeListType AttributeList;
CGM.ConstructAttributeList(CallInfo, TargetDecl, AttributeList, CallingConv);
llvm::AttrListPtr Attrs = llvm::AttrListPtr::get(AttributeList.begin(),
AttributeList.end());
llvm::CallSite CS;
if (!InvokeDest || (Attrs.getFnAttributes() & llvm::Attribute::NoUnwind)) {
CS = Builder.CreateCall(Callee, Args.data(), Args.data()+Args.size());
} else {
llvm::BasicBlock *Cont = createBasicBlock("invoke.cont");
CS = Builder.CreateInvoke(Callee, Cont, InvokeDest,
Args.data(), Args.data()+Args.size());
EmitBlock(Cont);
}
CS.setAttributes(Attrs);
CS.setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
// If the call doesn't return, finish the basic block and clear the
// insertion point; this allows the rest of IRgen to discard
// unreachable code.
if (CS.doesNotReturn()) {
Builder.CreateUnreachable();
Builder.ClearInsertionPoint();
// FIXME: For now, emit a dummy basic block because expr emitters in
// generally are not ready to handle emitting expressions at unreachable
// points.
EnsureInsertPoint();
// Return a reasonable RValue.
return GetUndefRValue(RetTy);
}
llvm::Instruction *CI = CS.getInstruction();
if (Builder.isNamePreserving() && !CI->getType()->isVoidTy())
CI->setName("call");
switch (RetAI.getKind()) {
case ABIArgInfo::Indirect:
if (RetTy->isAnyComplexType())
return RValue::getComplex(LoadComplexFromAddr(Args[0], false));
if (CodeGenFunction::hasAggregateLLVMType(RetTy))
return RValue::getAggregate(Args[0]);
return RValue::get(EmitLoadOfScalar(Args[0], false, RetTy));
case ABIArgInfo::Extend:
case ABIArgInfo::Direct:
if (RetTy->isAnyComplexType()) {
llvm::Value *Real = Builder.CreateExtractValue(CI, 0);
llvm::Value *Imag = Builder.CreateExtractValue(CI, 1);
return RValue::getComplex(std::make_pair(Real, Imag));
}
if (CodeGenFunction::hasAggregateLLVMType(RetTy)) {
llvm::Value *V = CreateTempAlloca(ConvertTypeForMem(RetTy), "agg.tmp");
Builder.CreateStore(CI, V);
return RValue::getAggregate(V);
}
return RValue::get(CI);
case ABIArgInfo::Ignore:
// If we are ignoring an argument that had a result, make sure to
// construct the appropriate return value for our caller.
return GetUndefRValue(RetTy);
case ABIArgInfo::Coerce: {
// FIXME: Avoid the conversion through memory if possible.
llvm::Value *V = CreateTempAlloca(ConvertTypeForMem(RetTy), "coerce");
CreateCoercedStore(CI, V, *this);
if (RetTy->isAnyComplexType())
return RValue::getComplex(LoadComplexFromAddr(V, false));
if (CodeGenFunction::hasAggregateLLVMType(RetTy))
return RValue::getAggregate(V);
return RValue::get(EmitLoadOfScalar(V, false, RetTy));
}
case ABIArgInfo::Expand:
assert(0 && "Invalid ABI kind for return argument");
}
assert(0 && "Unhandled ABIArgInfo::Kind");
return RValue::get(0);
}
