RValue AtomicInfo::convertTempToRValue(llvm::Value *addr,
AggValueSlot resultSlot) const {
if (EvaluationKind == TEK_Aggregate) {
// Nothing to do if the result is ignored.
if (resultSlot.isIgnored()) return resultSlot.asRValue();
assert(resultSlot.getAddr() == addr || hasPadding());
// In these cases, we should have emitted directly into the result slot.
if (!hasPadding() || resultSlot.isValueOfAtomic())
return resultSlot.asRValue();
// Otherwise, fall into the common path.
}
// Drill into the padding structure if we have one.
if (hasPadding())
addr = CGF.Builder.CreateStructGEP(addr, 0);
// If we're emitting to an aggregate, copy into the result slot.
if (EvaluationKind == TEK_Aggregate) {
CGF.EmitAggregateCopy(resultSlot.getAddr(), addr, getValueType(),
resultSlot.isVolatile());
return resultSlot.asRValue();
}
// Otherwise, just convert the temporary to an r-value using the
// normal conversion routine.
return CGF.convertTempToRValue(addr, getValueType());
}
/// Emit a load from an l-value of atomic type. Note that the r-value
/// we produce is an r-value of the atomic *value* type.
RValue CodeGenFunction::EmitAtomicLoad(LValue src, AggValueSlot resultSlot) {
AtomicInfo atomics(*this, src);
// Check whether we should use a library call.
if (atomics.shouldUseLibcall()) {
llvm::Value *tempAddr;
if (resultSlot.isValueOfAtomic()) {
assert(atomics.getEvaluationKind() == TEK_Aggregate);
tempAddr = resultSlot.getPaddedAtomicAddr();
} else if (!resultSlot.isIgnored() && !atomics.hasPadding()) {
assert(atomics.getEvaluationKind() == TEK_Aggregate);
tempAddr = resultSlot.getAddr();
} else {
tempAddr = CreateMemTemp(atomics.getAtomicType(), "atomic-load-temp");
}
// void __atomic_load(size_t size, void *mem, void *return, int order);
CallArgList args;
args.add(RValue::get(atomics.getAtomicSizeValue()),
getContext().getSizeType());
args.add(RValue::get(EmitCastToVoidPtr(src.getAddress())),
getContext().VoidPtrTy);
args.add(RValue::get(EmitCastToVoidPtr(tempAddr)),
getContext().VoidPtrTy);
args.add(RValue::get(llvm::ConstantInt::get(IntTy,
AO_ABI_memory_order_seq_cst)),
getContext().IntTy);
emitAtomicLibcall(*this, "__atomic_load", getContext().VoidTy, args);
// Produce the r-value.
return atomics.convertTempToRValue(tempAddr, resultSlot);
}
// Okay, we're doing this natively.
llvm::Value *addr = atomics.emitCastToAtomicIntPointer(src.getAddress());
llvm::LoadInst *load = Builder.CreateLoad(addr, "atomic-load");
load->setAtomic(llvm::SequentiallyConsistent);
// Other decoration.
load->setAlignment(src.getAlignment().getQuantity());
if (src.isVolatileQualified())
load->setVolatile(true);
if (src.getTBAAInfo())
CGM.DecorateInstruction(load, src.getTBAAInfo());
// Okay, turn that back into the original value type.
QualType valueType = atomics.getValueType();
llvm::Value *result = load;
// If we're ignoring an aggregate return, don't do anything.
if (atomics.getEvaluationKind() == TEK_Aggregate && resultSlot.isIgnored())
return RValue::getAggregate(0, false);
// The easiest way to do this this is to go through memory, but we
// try not to in some easy cases.
if (atomics.getEvaluationKind() == TEK_Scalar && !atomics.hasPadding()) {
llvm::Type *resultTy = CGM.getTypes().ConvertTypeForMem(valueType);
if (isa<llvm::IntegerType>(resultTy)) {
assert(result->getType() == resultTy);
result = EmitFromMemory(result, valueType);
} else if (isa<llvm::PointerType>(resultTy)) {
result = Builder.CreateIntToPtr(result, resultTy);
} else {
result = Builder.CreateBitCast(result, resultTy);
}
return RValue::get(result);
}
// Create a temporary. This needs to be big enough to hold the
// atomic integer.
llvm::Value *temp;
bool tempIsVolatile = false;
CharUnits tempAlignment;
if (atomics.getEvaluationKind() == TEK_Aggregate &&
(!atomics.hasPadding() || resultSlot.isValueOfAtomic())) {
assert(!resultSlot.isIgnored());
if (resultSlot.isValueOfAtomic()) {
temp = resultSlot.getPaddedAtomicAddr();
tempAlignment = atomics.getAtomicAlignment();
} else {
temp = resultSlot.getAddr();
tempAlignment = atomics.getValueAlignment();
}
tempIsVolatile = resultSlot.isVolatile();
} else {
temp = CreateMemTemp(atomics.getAtomicType(), "atomic-load-temp");
tempAlignment = atomics.getAtomicAlignment();
}
// Slam the integer into the temporary.
llvm::Value *castTemp = atomics.emitCastToAtomicIntPointer(temp);
Builder.CreateAlignedStore(result, castTemp, tempAlignment.getQuantity())
->setVolatile(tempIsVolatile);
return atomics.convertTempToRValue(temp, resultSlot);
}
