static LLVMBool LLVMTargetMachineEmit(LLVMTargetMachineRef T, LLVMModuleRef M,
raw_pwrite_stream &OS,
LLVMCodeGenFileType codegen,
char **ErrorMessage) {
TargetMachine* TM = unwrap(T);
Module* Mod = unwrap(M);
legacy::PassManager pass;
std::string error;
Mod->setDataLayout(TM->createDataLayout());
TargetMachine::CodeGenFileType ft;
switch (codegen) {
case LLVMAssemblyFile:
ft = TargetMachine::CGFT_AssemblyFile;
break;
default:
ft = TargetMachine::CGFT_ObjectFile;
break;
}
if (TM->addPassesToEmitFile(pass, OS, ft)) {
error = "TargetMachine can't emit a file of this type";
*ErrorMessage = strdup(error.c_str());
return true;
}
pass.run(*Mod);
OS.flush();
return false;
}
JuliaOJIT::JuliaOJIT(TargetMachine &TM)
: TM(TM),
DL(TM.createDataLayout()),
ObjStream(ObjBufferSV),
MemMgr(createRTDyldMemoryManager()),
registrar(*this),
ObjectLayer(
#if JL_LLVM_VERSION >= 50000
[&] { return MemMgr; },
#endif
std::ref(registrar)
),
CompileLayer(
ObjectLayer,
CompilerT(this)
)
{
addTargetPasses(&PM, &TM);
addOptimizationPasses(&PM, jl_generating_output() ? 0 : jl_options.opt_level);
if (TM.addPassesToEmitMC(PM, Ctx, ObjStream))
llvm_unreachable("Target does not support MC emission.");
// Make sure SectionMemoryManager::getSymbolAddressInProcess can resolve
// symbols in the program as well. The nullptr argument to the function
// tells DynamicLibrary to load the program, not a library.
std::string *ErrorStr = nullptr;
if (sys::DynamicLibrary::LoadLibraryPermanently(nullptr, ErrorStr))
report_fatal_error("FATAL: unable to dlopen self\n" + *ErrorStr);
}
JuliaOJIT(TargetMachine &TM)
: TM(TM),
DL(TM.createDataLayout()),
ObjStream(ObjBufferSV),
MemMgr(CUSTOM_MEMORY_MANAGER()),
ObjectLayer(DebugObjectRegistrar(*this)),
CompileLayer(
ObjectLayer,
[this](Module &M) {
PM.run(M);
std::unique_ptr<MemoryBuffer> ObjBuffer(
new ObjectMemoryBuffer(std::move(ObjBufferSV)));
auto Obj = object::ObjectFile::createObjectFile(ObjBuffer->getMemBufferRef());
if (!Obj) {
M.dump();
llvm::report_fatal_error("FATAL: Unable to compile LLVM Module.\n"
"The module's content was printed above. Please file a bug report");
}
return OwningObj(std::move(*Obj), std::move(ObjBuffer));
}
)
{
addOptimizationPasses(&PM);
if (TM.addPassesToEmitMC(PM, Ctx, ObjStream))
llvm_unreachable("Target does not support MC emission.");
// Make sure SectionMemoryManager::getSymbolAddressInProcess can resolve
// symbols in the program as well. The nullptr argument to the function
// tells DynamicLibrary to load the program, not a library.
std::string *ErrorStr = nullptr;
if (sys::DynamicLibrary::LoadLibraryPermanently(nullptr, ErrorStr))
report_fatal_error("FATAL: unable to dlopen self\n" + *ErrorStr);
}
JuliaOJIT::JuliaOJIT(TargetMachine &TM)
: TM(TM),
DL(TM.createDataLayout()),
ObjStream(ObjBufferSV),
MemMgr(createRTDyldMemoryManager()),
ObjectLayer(DebugObjectRegistrar(*this)),
CompileLayer(
ObjectLayer,
[this](Module &M) {
JL_TIMING(LLVM_OPT);
PM.run(M);
std::unique_ptr<MemoryBuffer> ObjBuffer(
new ObjectMemoryBuffer(std::move(ObjBufferSV)));
auto Obj = object::ObjectFile::createObjectFile(ObjBuffer->getMemBufferRef());
if (!Obj) {
#if JL_LLVM_VERSION >= 50000
M.print(llvm::dbgs(), nullptr, false, true);
#else
M.dump();
#endif
#if JL_LLVM_VERSION >= 30900
std::string Buf;
raw_string_ostream OS(Buf);
logAllUnhandledErrors(Obj.takeError(), OS, "");
OS.flush();
llvm::report_fatal_error("FATAL: Unable to compile LLVM Module: '" + Buf + "'\n"
"The module's content was printed above. Please file a bug report");
#else
llvm::report_fatal_error("FATAL: Unable to compile LLVM Module.\n"
"The module's content was printed above. Please file a bug report");
#endif
}
return OwningObj(std::move(*Obj), std::move(ObjBuffer));
}
)
{
if (!jl_generating_output()) {
addOptimizationPasses(&PM);
}
else {
PM.add(createLowerGCFramePass());
PM.add(createLowerPTLSPass(imaging_mode));
}
if (TM.addPassesToEmitMC(PM, Ctx, ObjStream))
llvm_unreachable("Target does not support MC emission.");
// Make sure SectionMemoryManager::getSymbolAddressInProcess can resolve
// symbols in the program as well. The nullptr argument to the function
// tells DynamicLibrary to load the program, not a library.
std::string *ErrorStr = nullptr;
if (sys::DynamicLibrary::LoadLibraryPermanently(nullptr, ErrorStr))
report_fatal_error("FATAL: unable to dlopen self\n" + *ErrorStr);
}
extern "C" void
LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module,
LLVMTargetMachineRef TMR) {
TargetMachine *Target = unwrap(TMR);
#if LLVM_VERSION_MINOR >= 7
unwrap(Module)->setDataLayout(Target->createDataLayout());
#elif LLVM_VERSION_MINOR >= 6
if (const DataLayout *DL = Target->getSubtargetImpl()->getDataLayout())
unwrap(Module)->setDataLayout(DL);
#else
if (const DataLayout *DL = Target->getDataLayout())
unwrap(Module)->setDataLayout(DL);
#endif
}
ErrorOr<std::unique_ptr<LTOModule>>
LTOModule::makeLTOModule(MemoryBufferRef Buffer, const TargetOptions &options,
LLVMContext &Context, bool ShouldBeLazy) {
ErrorOr<std::unique_ptr<Module>> MOrErr =
parseBitcodeFileImpl(Buffer, Context, ShouldBeLazy);
if (std::error_code EC = MOrErr.getError())
return EC;
std::unique_ptr<Module> &M = *MOrErr;
std::string TripleStr = M->getTargetTriple();
if (TripleStr.empty())
TripleStr = sys::getDefaultTargetTriple();
llvm::Triple Triple(TripleStr);
// find machine architecture for this module
std::string errMsg;
const Target *march = TargetRegistry::lookupTarget(TripleStr, errMsg);
if (!march)
return std::unique_ptr<LTOModule>(nullptr);
// construct LTOModule, hand over ownership of module and target
SubtargetFeatures Features;
Features.getDefaultSubtargetFeatures(Triple);
std::string FeatureStr = Features.getString();
// Set a default CPU for Darwin triples.
std::string CPU;
if (Triple.isOSDarwin()) {
if (Triple.getArch() == llvm::Triple::x86_64)
CPU = "core2";
else if (Triple.getArch() == llvm::Triple::x86)
CPU = "yonah";
else if (Triple.getArch() == llvm::Triple::aarch64)
CPU = "cyclone";
}
TargetMachine *target =
march->createTargetMachine(TripleStr, CPU, FeatureStr, options, None);
M->setDataLayout(target->createDataLayout());
std::unique_ptr<LTOModule> Ret(new LTOModule(std::move(M), Buffer, target));
Ret->parseSymbols();
Ret->parseMetadata();
return std::move(Ret);
}
static void runLTOPasses(Module &M, TargetMachine &TM) {
M.setDataLayout(TM.createDataLayout());
legacy::PassManager passes;
passes.add(createTargetTransformInfoWrapperPass(TM.getTargetIRAnalysis()));
PassManagerBuilder PMB;
PMB.LibraryInfo = new TargetLibraryInfoImpl(Triple(TM.getTargetTriple()));
PMB.Inliner = createFunctionInliningPass();
// Unconditionally verify input since it is not verified before this
// point and has unknown origin.
PMB.VerifyInput = true;
PMB.VerifyOutput = !options::DisableVerify;
PMB.LoopVectorize = true;
PMB.SLPVectorize = true;
PMB.OptLevel = options::OptLevel;
PMB.populateLTOPassManager(passes);
passes.run(M);
}
std::unique_ptr<llvm::SmallVectorImpl<char>>
getBPFObjectFromModule(llvm::Module *Module)
{
using namespace llvm;
std::string TargetTriple("bpf-pc-linux");
std::string Error;
const Target* Target = TargetRegistry::lookupTarget(TargetTriple, Error);
if (!Target) {
llvm::errs() << Error;
return std::unique_ptr<llvm::SmallVectorImpl<char>>(nullptr);
}
llvm::TargetOptions Opt;
TargetMachine *TargetMachine =
Target->createTargetMachine(TargetTriple,
"generic", "",
Opt, Reloc::Static);
Module->setDataLayout(TargetMachine->createDataLayout());
Module->setTargetTriple(TargetTriple);
std::unique_ptr<SmallVectorImpl<char>> Buffer(new SmallVector<char, 0>());
raw_svector_ostream ostream(*Buffer);
legacy::PassManager PM;
bool NotAdded;
#if CLANG_VERSION_MAJOR < 7
NotAdded = TargetMachine->addPassesToEmitFile(PM, ostream,
TargetMachine::CGFT_ObjectFile);
#else
NotAdded = TargetMachine->addPassesToEmitFile(PM, ostream, nullptr,
TargetMachine::CGFT_ObjectFile);
#endif
if (NotAdded) {
llvm::errs() << "TargetMachine can't emit a file of this type\n";
return std::unique_ptr<llvm::SmallVectorImpl<char>>(nullptr);
}
PM.run(*Module);
return Buffer;
}
void llvm::computeSignatureVTs(const FunctionType *Ty, const Function &F,
const TargetMachine &TM,
SmallVectorImpl<MVT> &Params,
SmallVectorImpl<MVT> &Results) {
computeLegalValueVTs(F, TM, Ty->getReturnType(), Results);
MVT PtrVT = MVT::getIntegerVT(TM.createDataLayout().getPointerSizeInBits());
if (Results.size() > 1) {
// WebAssembly currently can't lower returns of multiple values without
// demoting to sret (see WebAssemblyTargetLowering::CanLowerReturn). So
// replace multiple return values with a pointer parameter.
Results.clear();
Params.push_back(PtrVT);
}
for (auto *Param : Ty->params())
computeLegalValueVTs(F, TM, Param, Params);
if (Ty->isVarArg())
Params.push_back(PtrVT);
}
// This is the method invoked by the EE to Jit code.
CorJitResult LLILCJit::compileMethod(ICorJitInfo *JitInfo,
CORINFO_METHOD_INFO *MethodInfo,
UINT Flags, BYTE **NativeEntry,
ULONG *NativeSizeOfCode) {
// Bail if input is malformed
if (nullptr == JitInfo || nullptr == MethodInfo || nullptr == NativeEntry ||
nullptr == NativeSizeOfCode) {
return CORJIT_INTERNALERROR;
}
// Prep main outputs
*NativeEntry = nullptr;
*NativeSizeOfCode = 0;
// Set up state for this thread (if necessary)
LLILCJitPerThreadState *PerThreadState = State.get();
if (PerThreadState == nullptr) {
PerThreadState = new LLILCJitPerThreadState();
State.set(PerThreadState);
}
// Set up context for this Jit request
LLILCJitContext Context(PerThreadState);
// Fill in context information from the CLR
Context.JitInfo = JitInfo;
Context.MethodInfo = MethodInfo;
Context.Flags = Flags;
JitInfo->getEEInfo(&Context.EEInfo);
// Fill in context information from LLVM
Context.LLVMContext = &PerThreadState->LLVMContext;
std::unique_ptr<Module> M = Context.getModuleForMethod(MethodInfo);
Context.CurrentModule = M.get();
Context.CurrentModule->setTargetTriple(LLILC_TARGET_TRIPLE);
Context.CurrentModule->addModuleFlag(Module::Warning, "Debug Info Version",
DEBUG_METADATA_VERSION);
Context.MethodName = Context.CurrentModule->getModuleIdentifier();
Context.TheABIInfo = ABIInfo::get(*Context.CurrentModule);
Context.GcInfo = new GcInfo();
// Initialize per invocation JIT options. This should be done after the
// rest of the Context is filled out as it has dependencies on JitInfo,
// Flags and MethodInfo.
JitOptions JitOptions(Context);
if (JitOptions.IsBreakMethod) {
dbgs() << "INFO: Breaking for method " << Context.MethodName << "\n";
}
if (JitOptions.IsMSILDumpMethod) {
dbgs() << "INFO: Dumping MSIL for method " << Context.MethodName << "\n";
ReaderBase::printMSIL(MethodInfo->ILCode, 0, MethodInfo->ILCodeSize);
}
CorJitResult Result = CORJIT_INTERNALERROR;
if (JitOptions.IsAltJit && !JitOptions.IsExcludeMethod) {
Context.Options = &JitOptions;
// Construct the TargetMachine that we will emit code for
std::string ErrStr;
const llvm::Target *TheTarget =
TargetRegistry::lookupTarget(LLILC_TARGET_TRIPLE, ErrStr);
if (!TheTarget) {
errs() << "Could not create Target: " << ErrStr << "\n";
return CORJIT_INTERNALERROR;
}
TargetOptions Options;
CodeGenOpt::Level OptLevel;
bool IsNgen = Context.Flags & CORJIT_FLG_PREJIT;
bool IsReadyToRun = Context.Flags & CORJIT_FLG_READYTORUN;
// Optimal code for ReadyToRun should have all calls in call [rel32] form to
// enable crossgen to use shared delay-load thunks. We can't guarantee that
// LLVM will always generate this form so we currently don't take advantage
// of that and use non-shared delay-load thunks. The plan is to have as many
// calls as possible in that form and use shared delay-load thunks when
// possible. Setting OptLevel to Default increases the chances of calls via
// memory and setting CodeModel to Default enables rel32 relocations.
if ((Context.Options->EnableOptimization) || IsNgen || IsReadyToRun) {
OptLevel = CodeGenOpt::Level::Default;
} else {
OptLevel = CodeGenOpt::Level::None;
// Options.NoFramePointerElim = true;
}
llvm::CodeModel::Model CodeModel =
(IsNgen || IsReadyToRun) ? CodeModel::Default : CodeModel::JITDefault;
TargetMachine *TM =
TheTarget->createTargetMachine(LLILC_TARGET_TRIPLE, "", "", Options,
Reloc::Default, CodeModel, OptLevel);
Context.TM = TM;
// Set target machine datalayout on the method module.
Context.CurrentModule->setDataLayout(TM->createDataLayout());
// Construct the jitting layers.
EEMemoryManager MM(&Context);
ObjectLoadListener Listener(&Context);
orc::EEObjectLinkingLayer<decltype(Listener)> Loader(Listener);
auto ReserveUnwindSpace = [&MM](std::unique_ptr<object::ObjectFile> Obj) {
MM.reserveUnwindSpace(*Obj);
return std::move(Obj);
};
orc::ObjectTransformLayer<decltype(Loader), decltype(ReserveUnwindSpace)>
UnwindReserver(Loader, ReserveUnwindSpace);
orc::IRCompileLayer<decltype(UnwindReserver)> Compiler(
UnwindReserver, orc::LLILCCompiler(*TM));
// Now jit the method.
if (Context.Options->DumpLevel == DumpLevel::VERBOSE) {
dbgs() << "INFO: jitting method " << Context.MethodName
<< " using LLILCJit\n";
}
bool HasMethod = this->readMethod(&Context);
#ifndef FEATURE_VERIFICATION
bool IsImportOnly = (Context.Flags & CORJIT_FLG_IMPORT_ONLY) != 0;
// If asked to verify, report that it is verifiable.
if (IsImportOnly) {
Result = CORJIT_OK;
CorInfoMethodRuntimeFlags verFlag;
verFlag = CORINFO_FLG_VERIFIABLE;
JitInfo->setMethodAttribs(MethodInfo->ftn, verFlag);
return Result;
}
#endif
if (HasMethod) {
if (JitOptions.IsLLVMDumpMethod) {
dbgs() << "INFO: Dumping LLVM for method " << Context.MethodName
<< "\n";
Context.CurrentModule->dump();
}
if (Context.Options->DoInsertStatepoints) {
// If using Precise GC, run the GC-Safepoint insertion
// and lowering passes before generating code.
legacy::PassManager Passes;
Passes.add(createPlaceSafepointsPass());
Passes.add(createRewriteStatepointsForGCPass());
Passes.run(*M);
}
// Use a custom resolver that will tell the dynamic linker to skip
// relocation processing for external symbols that we create. We will
// report relocations for those symbols via Jit interface's
// recordRelocation method.
EESymbolResolver Resolver(&Context.NameToHandleMap);
auto HandleSet =
Compiler.addModuleSet<ArrayRef<Module *>>(M.get(), &MM, &Resolver);
*NativeEntry =
(BYTE *)Compiler.findSymbol(Context.MethodName, false).getAddress();
// TODO: ColdCodeSize, or separated code, is not enabled or included.
*NativeSizeOfCode = Context.HotCodeSize + Context.ReadOnlyDataSize;
if (JitOptions.IsCodeRangeMethod) {
errs() << "LLILC compiled: "
<< ", Entry = " << *NativeEntry
<< ", End = " << (*NativeEntry + *NativeSizeOfCode)
<< ", size = " << *NativeSizeOfCode
<< " method = " << Context.MethodName << '\n';
}
// The Method Jitted must begin at the start of the allocated
// Code block. The EE's DebugInfoManager relies on this.
// It allocates a CoreHeader block immediately before the
// code block address returned, and expects to find it
// at a fixed offset from *NativeEntry.
assert(*NativeEntry == MM.getHotCodeBlock() &&
"Expect the JITted method at the beginning of the code block");
GcInfoAllocator GcInfoAllocator;
GcInfoEmitter GcInfoEmitter(&Context, MM.getStackMapSection(),
&GcInfoAllocator);
GcInfoEmitter.emitGCInfo();
// Dump out any enabled timing info.
TimerGroup::printAll(errs());
// Give the jit layers a chance to free resources.
Compiler.removeModuleSet(HandleSet);
// Tell the CLR that we've successfully generated code for this method.
Result = CORJIT_OK;
}
// Clean up a bit
delete Context.TM;
Context.TM = nullptr;
} else {
// This method was not selected for jitting by LLILC.
if (JitOptions.DumpLevel == DumpLevel::SUMMARY) {
dbgs() << "INFO: skipping jitting method " << Context.MethodName
<< " using LLILCJit\n";
}
}
// Clean up a bit more
delete Context.TheABIInfo;
delete Context.GcInfo;
Context.TheABIInfo = nullptr;
Context.GcInfo = nullptr;
return Result;
}
extern "C" void
LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module,
LLVMTargetMachineRef TMR) {
TargetMachine *Target = unwrap(TMR);
unwrap(Module)->setDataLayout(Target->createDataLayout());
}
ErrorOr<std::unique_ptr<LTOModule>>
LTOModule::makeLTOModule(MemoryBufferRef Buffer, TargetOptions options,
LLVMContext *Context) {
std::unique_ptr<LLVMContext> OwnedContext;
if (!Context) {
OwnedContext = llvm::make_unique<LLVMContext>();
Context = OwnedContext.get();
}
// If we own a context, we know this is being used only for symbol
// extraction, not linking. Be lazy in that case.
ErrorOr<std::unique_ptr<Module>> MOrErr =
parseBitcodeFileImpl(Buffer, *Context,
/* ShouldBeLazy */ static_cast<bool>(OwnedContext));
if (std::error_code EC = MOrErr.getError())
return EC;
std::unique_ptr<Module> &M = *MOrErr;
std::string TripleStr = M->getTargetTriple();
if (TripleStr.empty())
TripleStr = sys::getDefaultTargetTriple();
llvm::Triple Triple(TripleStr);
// find machine architecture for this module
std::string errMsg;
const Target *march = TargetRegistry::lookupTarget(TripleStr, errMsg);
if (!march)
return std::unique_ptr<LTOModule>(nullptr);
// construct LTOModule, hand over ownership of module and target
SubtargetFeatures Features;
Features.getDefaultSubtargetFeatures(Triple);
std::string FeatureStr = Features.getString();
// Set a default CPU for Darwin triples.
std::string CPU;
if (Triple.isOSDarwin()) {
if (Triple.getArch() == llvm::Triple::x86_64)
CPU = "core2";
else if (Triple.getArch() == llvm::Triple::x86)
CPU = "yonah";
else if (Triple.getArch() == llvm::Triple::aarch64)
CPU = "cyclone";
}
TargetMachine *target = march->createTargetMachine(TripleStr, CPU, FeatureStr,
options);
M->setDataLayout(target->createDataLayout());
std::unique_ptr<object::IRObjectFile> IRObj(
new object::IRObjectFile(Buffer, std::move(M)));
std::unique_ptr<LTOModule> Ret;
if (OwnedContext)
Ret.reset(new LTOModule(std::move(IRObj), target, std::move(OwnedContext)));
else
Ret.reset(new LTOModule(std::move(IRObj), target));
Ret->parseSymbols();
Ret->parseMetadata();
return std::move(Ret);
}
