void CodeGenVTables::createVTableInitializer(ConstantStructBuilder &builder,
const VTableLayout &layout,
llvm::Constant *rtti) {
unsigned nextVTableThunkIndex = 0;
for (unsigned i = 0, e = layout.getNumVTables(); i != e; ++i) {
auto vtableElem = builder.beginArray(CGM.Int8PtrTy);
size_t thisIndex = layout.getVTableOffset(i);
size_t nextIndex = thisIndex + layout.getVTableSize(i);
for (unsigned i = thisIndex; i != nextIndex; ++i) {
addVTableComponent(vtableElem, layout, i, rtti, nextVTableThunkIndex);
}
vtableElem.finishAndAddTo(builder);
}
}
void CodeGenModule::EmitVTableTypeMetadata(llvm::GlobalVariable *VTable,
const VTableLayout &VTLayout) {
if (!getCodeGenOpts().LTOUnit)
return;
CharUnits PointerWidth =
Context.toCharUnitsFromBits(Context.getTargetInfo().getPointerWidth(0));
typedef std::pair<const CXXRecordDecl *, unsigned> BSEntry;
std::vector<BSEntry> BitsetEntries;
// Create a bit set entry for each address point.
for (auto &&AP : VTLayout.getAddressPoints())
BitsetEntries.push_back(
std::make_pair(AP.first.getBase(),
VTLayout.getVTableOffset(AP.second.VTableIndex) +
AP.second.AddressPointIndex));
// Sort the bit set entries for determinism.
std::sort(BitsetEntries.begin(), BitsetEntries.end(),
[this](const BSEntry &E1, const BSEntry &E2) {
if (&E1 == &E2)
return false;
std::string S1;
llvm::raw_string_ostream O1(S1);
getCXXABI().getMangleContext().mangleTypeName(
QualType(E1.first->getTypeForDecl(), 0), O1);
O1.flush();
std::string S2;
llvm::raw_string_ostream O2(S2);
getCXXABI().getMangleContext().mangleTypeName(
QualType(E2.first->getTypeForDecl(), 0), O2);
O2.flush();
if (S1 < S2)
return true;
if (S1 != S2)
return false;
return E1.second < E2.second;
});
for (auto BitsetEntry : BitsetEntries)
AddVTableTypeMetadata(VTable, PointerWidth * BitsetEntry.second,
BitsetEntry.first);
}
void CodeGenModule::EmitVTableBitSetEntries(llvm::GlobalVariable *VTable,
const VTableLayout &VTLayout) {
if (!LangOpts.Sanitize.has(SanitizerKind::CFIVCall) &&
!LangOpts.Sanitize.has(SanitizerKind::CFINVCall) &&
!LangOpts.Sanitize.has(SanitizerKind::CFIDerivedCast) &&
!LangOpts.Sanitize.has(SanitizerKind::CFIUnrelatedCast))
return;
CharUnits PointerWidth =
Context.toCharUnitsFromBits(Context.getTargetInfo().getPointerWidth(0));
typedef std::pair<const CXXRecordDecl *, unsigned> BSEntry;
std::vector<BSEntry> BitsetEntries;
// Create a bit set entry for each address point.
for (auto &&AP : VTLayout.getAddressPoints()) {
if (IsCFIBlacklistedRecord(AP.first.getBase()))
continue;
BitsetEntries.push_back(std::make_pair(AP.first.getBase(), AP.second));
}
// Sort the bit set entries for determinism.
std::sort(BitsetEntries.begin(), BitsetEntries.end(),
[this](const BSEntry &E1, const BSEntry &E2) {
if (&E1 == &E2)
return false;
std::string S1;
llvm::raw_string_ostream O1(S1);
getCXXABI().getMangleContext().mangleTypeName(
QualType(E1.first->getTypeForDecl(), 0), O1);
O1.flush();
std::string S2;
llvm::raw_string_ostream O2(S2);
getCXXABI().getMangleContext().mangleTypeName(
QualType(E2.first->getTypeForDecl(), 0), O2);
O2.flush();
if (S1 < S2)
return true;
if (S1 != S2)
return false;
return E1.second < E2.second;
});
llvm::NamedMDNode *BitsetsMD =
getModule().getOrInsertNamedMetadata("llvm.bitsets");
for (auto BitsetEntry : BitsetEntries)
CreateVTableBitSetEntry(BitsetsMD, VTable,
PointerWidth * BitsetEntry.second,
BitsetEntry.first);
}
void CodeGenVTables::addVTableComponent(
ConstantArrayBuilder &builder, const VTableLayout &layout,
unsigned idx, llvm::Constant *rtti, unsigned &nextVTableThunkIndex) {
auto &component = layout.vtable_components()[idx];
auto addOffsetConstant = [&](CharUnits offset) {
builder.add(llvm::ConstantExpr::getIntToPtr(
llvm::ConstantInt::get(CGM.PtrDiffTy, offset.getQuantity()),
CGM.Int8PtrTy));
};
switch (component.getKind()) {
case VTableComponent::CK_VCallOffset:
return addOffsetConstant(component.getVCallOffset());
case VTableComponent::CK_VBaseOffset:
return addOffsetConstant(component.getVBaseOffset());
case VTableComponent::CK_OffsetToTop:
return addOffsetConstant(component.getOffsetToTop());
case VTableComponent::CK_RTTI:
return builder.add(llvm::ConstantExpr::getBitCast(rtti, CGM.Int8PtrTy));
case VTableComponent::CK_FunctionPointer:
case VTableComponent::CK_CompleteDtorPointer:
case VTableComponent::CK_DeletingDtorPointer: {
GlobalDecl GD;
// Get the right global decl.
switch (component.getKind()) {
default:
llvm_unreachable("Unexpected vtable component kind");
case VTableComponent::CK_FunctionPointer:
GD = component.getFunctionDecl();
break;
case VTableComponent::CK_CompleteDtorPointer:
GD = GlobalDecl(component.getDestructorDecl(), Dtor_Complete);
break;
case VTableComponent::CK_DeletingDtorPointer:
GD = GlobalDecl(component.getDestructorDecl(), Dtor_Deleting);
break;
}
if (CGM.getLangOpts().CUDA) {
// Emit NULL for methods we can't codegen on this
// side. Otherwise we'd end up with vtable with unresolved
// references.
const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
// OK on device side: functions w/ __device__ attribute
// OK on host side: anything except __device__-only functions.
bool CanEmitMethod =
CGM.getLangOpts().CUDAIsDevice
? MD->hasAttr<CUDADeviceAttr>()
: (MD->hasAttr<CUDAHostAttr>() || !MD->hasAttr<CUDADeviceAttr>());
if (!CanEmitMethod)
return builder.addNullPointer(CGM.Int8PtrTy);
// Method is acceptable, continue processing as usual.
}
auto getSpecialVirtualFn = [&](StringRef name) {
llvm::FunctionType *fnTy =
llvm::FunctionType::get(CGM.VoidTy, /*isVarArg=*/false);
llvm::Constant *fn = CGM.CreateRuntimeFunction(fnTy, name);
if (auto f = dyn_cast<llvm::Function>(fn))
f->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
return llvm::ConstantExpr::getBitCast(fn, CGM.Int8PtrTy);
};
llvm::Constant *fnPtr;
// Pure virtual member functions.
if (cast<CXXMethodDecl>(GD.getDecl())->isPure()) {
if (!PureVirtualFn)
PureVirtualFn =
getSpecialVirtualFn(CGM.getCXXABI().GetPureVirtualCallName());
fnPtr = PureVirtualFn;
// Deleted virtual member functions.
} else if (cast<CXXMethodDecl>(GD.getDecl())->isDeleted()) {
if (!DeletedVirtualFn)
DeletedVirtualFn =
getSpecialVirtualFn(CGM.getCXXABI().GetDeletedVirtualCallName());
fnPtr = DeletedVirtualFn;
// Thunks.
} else if (nextVTableThunkIndex < layout.vtable_thunks().size() &&
layout.vtable_thunks()[nextVTableThunkIndex].first == idx) {
auto &thunkInfo = layout.vtable_thunks()[nextVTableThunkIndex].second;
maybeEmitThunkForVTable(GD, thunkInfo);
nextVTableThunkIndex++;
fnPtr = CGM.GetAddrOfThunk(GD, thunkInfo);
// Otherwise we can use the method definition directly.
} else {
llvm::Type *fnTy = CGM.getTypes().GetFunctionTypeForVTable(GD);
fnPtr = CGM.GetAddrOfFunction(GD, fnTy, /*ForVTable=*/true);
}
fnPtr = llvm::ConstantExpr::getBitCast(fnPtr, CGM.Int8PtrTy);
builder.add(fnPtr);
return;
}
case VTableComponent::CK_UnusedFunctionPointer:
return builder.addNullPointer(CGM.Int8PtrTy);
}
llvm_unreachable("Unexpected vtable component kind");
}
void CodeGenModule::EmitVTableTypeMetadata(llvm::GlobalVariable *VTable,
const VTableLayout &VTLayout) {
if (!getCodeGenOpts().LTOUnit)
return;
CharUnits PointerWidth =
Context.toCharUnitsFromBits(Context.getTargetInfo().getPointerWidth(0));
typedef std::pair<const CXXRecordDecl *, unsigned> AddressPoint;
std::vector<AddressPoint> AddressPoints;
for (auto &&AP : VTLayout.getAddressPoints())
AddressPoints.push_back(std::make_pair(
AP.first.getBase(), VTLayout.getVTableOffset(AP.second.VTableIndex) +
AP.second.AddressPointIndex));
// Sort the address points for determinism.
llvm::sort(AddressPoints, [this](const AddressPoint &AP1,
const AddressPoint &AP2) {
if (&AP1 == &AP2)
return false;
std::string S1;
llvm::raw_string_ostream O1(S1);
getCXXABI().getMangleContext().mangleTypeName(
QualType(AP1.first->getTypeForDecl(), 0), O1);
O1.flush();
std::string S2;
llvm::raw_string_ostream O2(S2);
getCXXABI().getMangleContext().mangleTypeName(
QualType(AP2.first->getTypeForDecl(), 0), O2);
O2.flush();
if (S1 < S2)
return true;
if (S1 != S2)
return false;
return AP1.second < AP2.second;
});
ArrayRef<VTableComponent> Comps = VTLayout.vtable_components();
for (auto AP : AddressPoints) {
// Create type metadata for the address point.
AddVTableTypeMetadata(VTable, PointerWidth * AP.second, AP.first);
// The class associated with each address point could also potentially be
// used for indirect calls via a member function pointer, so we need to
// annotate the address of each function pointer with the appropriate member
// function pointer type.
for (unsigned I = 0; I != Comps.size(); ++I) {
if (Comps[I].getKind() != VTableComponent::CK_FunctionPointer)
continue;
llvm::Metadata *MD = CreateMetadataIdentifierForVirtualMemPtrType(
Context.getMemberPointerType(
Comps[I].getFunctionDecl()->getType(),
Context.getRecordType(AP.first).getTypePtr()));
VTable->addTypeMetadata((PointerWidth * I).getQuantity(), MD);
}
}
}
void CodeGenModule::EmitVTableBitSetEntries(llvm::GlobalVariable *VTable,
const VTableLayout &VTLayout) {
if (!LangOpts.Sanitize.has(SanitizerKind::CFIVCall) &&
!LangOpts.Sanitize.has(SanitizerKind::CFINVCall) &&
!LangOpts.Sanitize.has(SanitizerKind::CFIDerivedCast) &&
!LangOpts.Sanitize.has(SanitizerKind::CFIUnrelatedCast))
return;
llvm::Metadata *VTableMD = llvm::ConstantAsMetadata::get(VTable);
std::vector<llvm::MDTuple *> BitsetEntries;
// Create a bit set entry for each address point.
for (auto &&AP : VTLayout.getAddressPoints()) {
// FIXME: Add blacklisting scheme.
if (AP.first.getBase()->isInStdNamespace())
continue;
std::string OutName;
llvm::raw_string_ostream Out(OutName);
getCXXABI().getMangleContext().mangleCXXVTableBitSet(AP.first.getBase(),
Out);
CharUnits PointerWidth =
Context.toCharUnitsFromBits(Context.getTargetInfo().getPointerWidth(0));
uint64_t AddrPointOffset = AP.second * PointerWidth.getQuantity();
llvm::Metadata *BitsetOps[] = {
llvm::MDString::get(getLLVMContext(), Out.str()),
VTableMD,
llvm::ConstantAsMetadata::get(
llvm::ConstantInt::get(Int64Ty, AddrPointOffset))};
llvm::MDTuple *BitsetEntry =
llvm::MDTuple::get(getLLVMContext(), BitsetOps);
BitsetEntries.push_back(BitsetEntry);
}
// Sort the bit set entries for determinism.
std::sort(BitsetEntries.begin(), BitsetEntries.end(), [](llvm::MDTuple *T1,
llvm::MDTuple *T2) {
if (T1 == T2)
return false;
StringRef S1 = cast<llvm::MDString>(T1->getOperand(0))->getString();
StringRef S2 = cast<llvm::MDString>(T2->getOperand(0))->getString();
if (S1 < S2)
return true;
if (S1 != S2)
return false;
uint64_t Offset1 = cast<llvm::ConstantInt>(
cast<llvm::ConstantAsMetadata>(T1->getOperand(2))
->getValue())->getZExtValue();
uint64_t Offset2 = cast<llvm::ConstantInt>(
cast<llvm::ConstantAsMetadata>(T2->getOperand(2))
->getValue())->getZExtValue();
assert(Offset1 != Offset2);
return Offset1 < Offset2;
});
llvm::NamedMDNode *BitsetsMD =
getModule().getOrInsertNamedMetadata("llvm.bitsets");
for (auto BitsetEntry : BitsetEntries)
BitsetsMD->addOperand(BitsetEntry);
}
