/// Copy an r-value into memory as part of storing to an atomic type.
/// This needs to create a bit-pattern suitable for atomic operations.
void AtomicInfo::emitCopyIntoMemory(RValue rvalue, LValue dest) const {
// If we have an r-value, the rvalue should be of the atomic type,
// which means that the caller is responsible for having zeroed
// any padding. Just do an aggregate copy of that type.
if (rvalue.isAggregate()) {
CGF.EmitAggregateCopy(dest.getAddress(),
rvalue.getAggregateAddr(),
getAtomicType(),
(rvalue.isVolatileQualified()
|| dest.isVolatileQualified()),
dest.getAlignment());
return;
}
// Okay, otherwise we're copying stuff.
// Zero out the buffer if necessary.
emitMemSetZeroIfNecessary(dest);
// Drill past the padding if present.
dest = projectValue(dest);
// Okay, store the rvalue in.
if (rvalue.isScalar()) {
CGF.EmitStoreOfScalar(rvalue.getScalarVal(), dest, /*init*/ true);
} else {
CGF.EmitStoreOfComplex(rvalue.getComplexVal(), dest, /*init*/ true);
}
}
/// EmitLoadOfLValue - Given an RValue reference for a complex, emit code to
/// load the real and imaginary pieces, returning them as Real/Imag.
ComplexPairTy ComplexExprEmitter::EmitLoadOfLValue(LValue lvalue,
SourceLocation loc) {
assert(lvalue.isSimple() && "non-simple complex l-value?");
if (lvalue.getType()->isAtomicType())
return CGF.EmitAtomicLoad(lvalue, loc).getComplexVal();
llvm::Value *SrcPtr = lvalue.getAddress();
bool isVolatile = lvalue.isVolatileQualified();
unsigned AlignR = lvalue.getAlignment().getQuantity();
ASTContext &C = CGF.getContext();
QualType ComplexTy = lvalue.getType();
unsigned ComplexAlign = C.getTypeAlignInChars(ComplexTy).getQuantity();
unsigned AlignI = std::min(AlignR, ComplexAlign);
llvm::Value *Real=nullptr, *Imag=nullptr;
if (!IgnoreReal || isVolatile) {
llvm::Value *RealP = Builder.CreateStructGEP(SrcPtr, 0,
SrcPtr->getName() + ".realp");
Real = Builder.CreateAlignedLoad(RealP, AlignR, isVolatile,
SrcPtr->getName() + ".real");
}
if (!IgnoreImag || isVolatile) {
llvm::Value *ImagP = Builder.CreateStructGEP(SrcPtr, 1,
SrcPtr->getName() + ".imagp");
Imag = Builder.CreateAlignedLoad(ImagP, AlignI, isVolatile,
SrcPtr->getName() + ".imag");
}
return ComplexPairTy(Real, Imag);
}
void AtomicInfo::emitMemSetZeroIfNecessary(LValue dest) const {
llvm::Value *addr = dest.getAddress();
if (!requiresMemSetZero(addr->getType()->getPointerElementType()))
return;
CGF.Builder.CreateMemSet(addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
AtomicSizeInBits / 8,
dest.getAlignment().getQuantity());
}
/// Emit a store to an l-value of atomic type.
///
/// Note that the r-value is expected to be an r-value *of the atomic
/// type*; this means that for aggregate r-values, it should include
/// storage for any padding that was necessary.
void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, bool isInit) {
// If this is an aggregate r-value, it should agree in type except
// maybe for address-space qualification.
assert(!rvalue.isAggregate() ||
rvalue.getAggregateAddr()->getType()->getPointerElementType()
== dest.getAddress()->getType()->getPointerElementType());
AtomicInfo atomics(*this, dest);
// If this is an initialization, just put the value there normally.
if (isInit) {
atomics.emitCopyIntoMemory(rvalue, dest);
return;
}
// Check whether we should use a library call.
if (atomics.shouldUseLibcall()) {
// Produce a source address.
llvm::Value *srcAddr = atomics.materializeRValue(rvalue);
// void __atomic_store(size_t size, void *mem, void *val, int order)
CallArgList args;
args.add(RValue::get(atomics.getAtomicSizeValue()),
getContext().getSizeType());
args.add(RValue::get(EmitCastToVoidPtr(dest.getAddress())),
getContext().VoidPtrTy);
args.add(RValue::get(EmitCastToVoidPtr(srcAddr)),
getContext().VoidPtrTy);
args.add(RValue::get(llvm::ConstantInt::get(
IntTy, AtomicExpr::AO_ABI_memory_order_seq_cst)),
getContext().IntTy);
emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
return;
}
// Okay, we're doing this natively.
llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
// Do the atomic store.
llvm::Value *addr = atomics.emitCastToAtomicIntPointer(dest.getAddress());
llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
// Initializations don't need to be atomic.
if (!isInit) store->setAtomic(llvm::SequentiallyConsistent);
// Other decoration.
store->setAlignment(dest.getAlignment().getQuantity());
if (dest.isVolatileQualified())
store->setVolatile(true);
if (dest.getTBAAInfo())
CGM.DecorateInstruction(store, dest.getTBAAInfo());
}
/// Emit a load from an l-value of atomic type. Note that the r-value
/// we produce is an r-value of the atomic *value* type.
RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
AggValueSlot resultSlot) {
AtomicInfo atomics(*this, src);
// Check whether we should use a library call.
if (atomics.shouldUseLibcall()) {
llvm::Value *tempAddr;
if (!resultSlot.isIgnored()) {
assert(atomics.getEvaluationKind() == TEK_Aggregate);
tempAddr = resultSlot.getAddr();
} else {
tempAddr = CreateMemTemp(atomics.getAtomicType(), "atomic-load-temp");
}
// void __atomic_load(size_t size, void *mem, void *return, int order);
CallArgList args;
args.add(RValue::get(atomics.getAtomicSizeValue()),
getContext().getSizeType());
args.add(RValue::get(EmitCastToVoidPtr(src.getAddress())),
getContext().VoidPtrTy);
args.add(RValue::get(EmitCastToVoidPtr(tempAddr)),
getContext().VoidPtrTy);
args.add(RValue::get(llvm::ConstantInt::get(
IntTy, AtomicExpr::AO_ABI_memory_order_seq_cst)),
getContext().IntTy);
emitAtomicLibcall(*this, "__atomic_load", getContext().VoidTy, args);
// Produce the r-value.
return atomics.convertTempToRValue(tempAddr, resultSlot, loc);
}
// Okay, we're doing this natively.
llvm::Value *addr = atomics.emitCastToAtomicIntPointer(src.getAddress());
llvm::LoadInst *load = Builder.CreateLoad(addr, "atomic-load");
load->setAtomic(llvm::SequentiallyConsistent);
// Other decoration.
load->setAlignment(src.getAlignment().getQuantity());
if (src.isVolatileQualified())
load->setVolatile(true);
if (src.getTBAAInfo())
CGM.DecorateInstruction(load, src.getTBAAInfo());
// If we're ignoring an aggregate return, don't do anything.
if (atomics.getEvaluationKind() == TEK_Aggregate && resultSlot.isIgnored())
return RValue::getAggregate(nullptr, false);
// Okay, turn that back into the original value type.
return atomics.convertIntToValue(load, resultSlot, loc);
}
/// EmitStoreOfComplex - Store the specified real/imag parts into the
/// specified value pointer.
void ComplexExprEmitter::EmitStoreOfComplex(ComplexPairTy Val, LValue lvalue,
bool isInit) {
if (lvalue.getType()->isAtomicType())
return CGF.EmitAtomicStore(RValue::getComplex(Val), lvalue, isInit);
llvm::Value *Ptr = lvalue.getAddress();
llvm::Value *RealPtr = Builder.CreateStructGEP(Ptr, 0, "real");
llvm::Value *ImagPtr = Builder.CreateStructGEP(Ptr, 1, "imag");
unsigned AlignR = lvalue.getAlignment().getQuantity();
ASTContext &C = CGF.getContext();
QualType ComplexTy = lvalue.getType();
unsigned ComplexAlign = C.getTypeAlignInChars(ComplexTy).getQuantity();
unsigned AlignI = std::min(AlignR, ComplexAlign);
Builder.CreateAlignedStore(Val.first, RealPtr, AlignR,
lvalue.isVolatileQualified());
Builder.CreateAlignedStore(Val.second, ImagPtr, AlignI,
lvalue.isVolatileQualified());
}
/// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
void AggExprEmitter::EmitFinalDestCopy(const Expr *E, LValue Src, bool Ignore) {
assert(Src.isSimple() && "Can't have aggregate bitfield, vector, etc");
CharUnits Alignment = std::min(Src.getAlignment(), Dest.getAlignment());
EmitFinalDestCopy(E, Src.asAggregateRValue(), Ignore, Alignment.getQuantity());
}
ComplexPairTy ComplexExprEmitter::EmitCast(CastExpr::CastKind CK, Expr *Op,
QualType DestTy) {
switch (CK) {
case CK_Dependent: llvm_unreachable("dependent cast kind in IR gen!");
// Atomic to non-atomic casts may be more than a no-op for some platforms and
// for some types.
case CK_AtomicToNonAtomic:
case CK_NonAtomicToAtomic:
case CK_NoOp:
case CK_LValueToRValue:
case CK_UserDefinedConversion:
return Visit(Op);
case CK_LValueBitCast: {
LValue origLV = CGF.EmitLValue(Op);
llvm::Value *V = origLV.getAddress();
V = Builder.CreateBitCast(V,
CGF.ConvertType(CGF.getContext().getPointerType(DestTy)));
return EmitLoadOfLValue(CGF.MakeAddrLValue(V, DestTy,
origLV.getAlignment()));
}
case CK_BitCast:
case CK_BaseToDerived:
case CK_DerivedToBase:
case CK_UncheckedDerivedToBase:
case CK_Dynamic:
case CK_ToUnion:
case CK_ArrayToPointerDecay:
case CK_FunctionToPointerDecay:
case CK_NullToPointer:
case CK_NullToMemberPointer:
case CK_BaseToDerivedMemberPointer:
case CK_DerivedToBaseMemberPointer:
case CK_MemberPointerToBoolean:
case CK_ReinterpretMemberPointer:
case CK_ConstructorConversion:
case CK_IntegralToPointer:
case CK_PointerToIntegral:
case CK_PointerToBoolean:
case CK_ToVoid:
case CK_VectorSplat:
case CK_IntegralCast:
case CK_IntegralToBoolean:
case CK_IntegralToFloating:
case CK_FloatingToIntegral:
case CK_FloatingToBoolean:
case CK_FloatingCast:
case CK_CPointerToObjCPointerCast:
case CK_BlockPointerToObjCPointerCast:
case CK_AnyPointerToBlockPointerCast:
case CK_ObjCObjectLValueCast:
case CK_FloatingComplexToReal:
case CK_FloatingComplexToBoolean:
case CK_IntegralComplexToReal:
case CK_IntegralComplexToBoolean:
case CK_ARCProduceObject:
case CK_ARCConsumeObject:
case CK_ARCReclaimReturnedObject:
case CK_ARCExtendBlockObject:
case CK_CopyAndAutoreleaseBlockObject:
case CK_BuiltinFnToFnPtr:
case CK_ZeroToOCLEvent:
llvm_unreachable("invalid cast kind for complex value");
case CK_FloatingRealToComplex:
case CK_IntegralRealToComplex: {
llvm::Value *Elt = CGF.EmitScalarExpr(Op);
// Convert the input element to the element type of the complex.
DestTy = DestTy->castAs<ComplexType>()->getElementType();
Elt = CGF.EmitScalarConversion(Elt, Op->getType(), DestTy);
// Return (realval, 0).
return ComplexPairTy(Elt, llvm::Constant::getNullValue(Elt->getType()));
}
case CK_FloatingComplexCast:
case CK_FloatingComplexToIntegralComplex:
case CK_IntegralComplexCast:
case CK_IntegralComplexToFloatingComplex:
return EmitComplexToComplexCast(Visit(Op), Op->getType(), DestTy);
}
llvm_unreachable("unknown cast resulting in complex value");
}
/// Emit a store to an l-value of atomic type.
///
/// Note that the r-value is expected to be an r-value *of the atomic
/// type*; this means that for aggregate r-values, it should include
/// storage for any padding that was necessary.
void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, bool isInit) {
// If this is an aggregate r-value, it should agree in type except
// maybe for address-space qualification.
assert(!rvalue.isAggregate() ||
rvalue.getAggregateAddr()->getType()->getPointerElementType()
== dest.getAddress()->getType()->getPointerElementType());
AtomicInfo atomics(*this, dest);
// If this is an initialization, just put the value there normally.
if (isInit) {
atomics.emitCopyIntoMemory(rvalue, dest);
return;
}
// Check whether we should use a library call.
if (atomics.shouldUseLibcall()) {
// Produce a source address.
llvm::Value *srcAddr = atomics.materializeRValue(rvalue);
// void __atomic_store(size_t size, void *mem, void *val, int order)
CallArgList args;
args.add(RValue::get(atomics.getAtomicSizeValue()),
getContext().getSizeType());
args.add(RValue::get(EmitCastToVoidPtr(dest.getAddress())),
getContext().VoidPtrTy);
args.add(RValue::get(EmitCastToVoidPtr(srcAddr)),
getContext().VoidPtrTy);
args.add(RValue::get(llvm::ConstantInt::get(
IntTy, AtomicExpr::AO_ABI_memory_order_seq_cst)),
getContext().IntTy);
emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
return;
}
// Okay, we're doing this natively.
llvm::Value *intValue;
// If we've got a scalar value of the right size, try to avoid going
// through memory.
if (rvalue.isScalar() && !atomics.hasPadding()) {
llvm::Value *value = rvalue.getScalarVal();
if (isa<llvm::IntegerType>(value->getType())) {
intValue = value;
} else {
llvm::IntegerType *inputIntTy =
llvm::IntegerType::get(getLLVMContext(), atomics.getValueSizeInBits());
if (isa<llvm::PointerType>(value->getType())) {
intValue = Builder.CreatePtrToInt(value, inputIntTy);
} else {
intValue = Builder.CreateBitCast(value, inputIntTy);
}
}
// Otherwise, we need to go through memory.
} else {
// Put the r-value in memory.
llvm::Value *addr = atomics.materializeRValue(rvalue);
// Cast the temporary to the atomic int type and pull a value out.
addr = atomics.emitCastToAtomicIntPointer(addr);
intValue = Builder.CreateAlignedLoad(addr,
atomics.getAtomicAlignment().getQuantity());
}
// Do the atomic store.
llvm::Value *addr = atomics.emitCastToAtomicIntPointer(dest.getAddress());
llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
// Initializations don't need to be atomic.
if (!isInit) store->setAtomic(llvm::SequentiallyConsistent);
// Other decoration.
store->setAlignment(dest.getAlignment().getQuantity());
if (dest.isVolatileQualified())
store->setVolatile(true);
if (dest.getTBAAInfo())
CGM.DecorateInstruction(store, dest.getTBAAInfo());
}
/// Emit a load from an l-value of atomic type. Note that the r-value
/// we produce is an r-value of the atomic *value* type.
RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
AggValueSlot resultSlot) {
AtomicInfo atomics(*this, src);
// Check whether we should use a library call.
if (atomics.shouldUseLibcall()) {
llvm::Value *tempAddr;
if (!resultSlot.isIgnored()) {
assert(atomics.getEvaluationKind() == TEK_Aggregate);
tempAddr = resultSlot.getAddr();
} else {
tempAddr = CreateMemTemp(atomics.getAtomicType(), "atomic-load-temp");
}
// void __atomic_load(size_t size, void *mem, void *return, int order);
CallArgList args;
args.add(RValue::get(atomics.getAtomicSizeValue()),
getContext().getSizeType());
args.add(RValue::get(EmitCastToVoidPtr(src.getAddress())),
getContext().VoidPtrTy);
args.add(RValue::get(EmitCastToVoidPtr(tempAddr)),
getContext().VoidPtrTy);
args.add(RValue::get(llvm::ConstantInt::get(
IntTy, AtomicExpr::AO_ABI_memory_order_seq_cst)),
getContext().IntTy);
emitAtomicLibcall(*this, "__atomic_load", getContext().VoidTy, args);
// Produce the r-value.
return atomics.convertTempToRValue(tempAddr, resultSlot, loc);
}
// Okay, we're doing this natively.
llvm::Value *addr = atomics.emitCastToAtomicIntPointer(src.getAddress());
llvm::LoadInst *load = Builder.CreateLoad(addr, "atomic-load");
load->setAtomic(llvm::SequentiallyConsistent);
// Other decoration.
load->setAlignment(src.getAlignment().getQuantity());
if (src.isVolatileQualified())
load->setVolatile(true);
if (src.getTBAAInfo())
CGM.DecorateInstruction(load, src.getTBAAInfo());
// Okay, turn that back into the original value type.
QualType valueType = atomics.getValueType();
llvm::Value *result = load;
// If we're ignoring an aggregate return, don't do anything.
if (atomics.getEvaluationKind() == TEK_Aggregate && resultSlot.isIgnored())
return RValue::getAggregate(0, false);
// The easiest way to do this this is to go through memory, but we
// try not to in some easy cases.
if (atomics.getEvaluationKind() == TEK_Scalar && !atomics.hasPadding()) {
llvm::Type *resultTy = CGM.getTypes().ConvertTypeForMem(valueType);
if (isa<llvm::IntegerType>(resultTy)) {
assert(result->getType() == resultTy);
result = EmitFromMemory(result, valueType);
} else if (isa<llvm::PointerType>(resultTy)) {
result = Builder.CreateIntToPtr(result, resultTy);
} else {
result = Builder.CreateBitCast(result, resultTy);
}
return RValue::get(result);
}
// Create a temporary. This needs to be big enough to hold the
// atomic integer.
llvm::Value *temp;
bool tempIsVolatile = false;
CharUnits tempAlignment;
if (atomics.getEvaluationKind() == TEK_Aggregate) {
assert(!resultSlot.isIgnored());
temp = resultSlot.getAddr();
tempAlignment = atomics.getValueAlignment();
tempIsVolatile = resultSlot.isVolatile();
} else {
temp = CreateMemTemp(atomics.getAtomicType(), "atomic-load-temp");
tempAlignment = atomics.getAtomicAlignment();
}
// Slam the integer into the temporary.
llvm::Value *castTemp = atomics.emitCastToAtomicIntPointer(temp);
Builder.CreateAlignedStore(result, castTemp, tempAlignment.getQuantity())
->setVolatile(tempIsVolatile);
return atomics.convertTempToRValue(temp, resultSlot, loc);
}
