void AddressSanitizer::instrumentAddress(Instruction *OrigIns,
IRBuilder<> &IRB, Value *Addr,
uint32_t TypeSize, bool IsWrite) {
Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
Type *ShadowTy = IntegerType::get(
*C, std::max(8U, TypeSize >> MappingScale));
Type *ShadowPtrTy = PointerType::get(ShadowTy, 0);
Value *ShadowPtr = memToShadow(AddrLong, IRB);
Value *CmpVal = Constant::getNullValue(ShadowTy);
Value *ShadowValue = IRB.CreateLoad(
IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy));
Value *Cmp = IRB.CreateICmpNE(ShadowValue, CmpVal);
Instruction *CheckTerm = splitBlockAndInsertIfThen(
cast<Instruction>(Cmp)->getNextNode(), Cmp);
IRBuilder<> IRB2(CheckTerm);
size_t Granularity = 1 << MappingScale;
if (TypeSize < 8 * Granularity) {
// Addr & (Granularity - 1)
Value *Lower3Bits = IRB2.CreateAnd(
AddrLong, ConstantInt::get(IntptrTy, Granularity - 1));
// (Addr & (Granularity - 1)) + size - 1
Value *LastAccessedByte = IRB2.CreateAdd(
Lower3Bits, ConstantInt::get(IntptrTy, TypeSize / 8 - 1));
// (uint8_t) ((Addr & (Granularity-1)) + size - 1)
LastAccessedByte = IRB2.CreateIntCast(
LastAccessedByte, IRB.getInt8Ty(), false);
// ((uint8_t) ((Addr & (Granularity-1)) + size - 1)) >= ShadowValue
Value *Cmp2 = IRB2.CreateICmpSGE(LastAccessedByte, ShadowValue);
CheckTerm = splitBlockAndInsertIfThen(CheckTerm, Cmp2);
}
IRBuilder<> IRB1(CheckTerm);
Instruction *Crash = generateCrashCode(IRB1, AddrLong, IsWrite, TypeSize);
Crash->setDebugLoc(OrigIns->getDebugLoc());
ReplaceInstWithInst(CheckTerm, new UnreachableInst(*C));
}
void AddressSanitizer::instrumentAddress(AsanFunctionContext &AFC,
Instruction *OrigIns,
IRBuilder<> &IRB, Value *Addr,
uint32_t TypeSize, bool IsWrite) {
Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
Type *ShadowTy = IntegerType::get(
*C, std::max(8U, TypeSize >> MappingScale));
Type *ShadowPtrTy = PointerType::get(ShadowTy, 0);
Value *ShadowPtr = memToShadow(AddrLong, IRB);
Value *CmpVal = Constant::getNullValue(ShadowTy);
Value *ShadowValue = IRB.CreateLoad(
IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy));
Value *Cmp = IRB.CreateICmpNE(ShadowValue, CmpVal);
size_t AccessSizeIndex = TypeSizeToSizeIndex(TypeSize);
size_t Granularity = 1 << MappingScale;
TerminatorInst *CrashTerm = 0;
if (ClAlwaysSlowPath || (TypeSize < 8 * Granularity)) {
TerminatorInst *CheckTerm = splitBlockAndInsertIfThen(Cmp, false);
assert(dyn_cast<BranchInst>(CheckTerm)->isUnconditional());
BasicBlock *NextBB = CheckTerm->getSuccessor(0);
IRB.SetInsertPoint(CheckTerm);
Value *Cmp2 = createSlowPathCmp(IRB, AddrLong, ShadowValue, TypeSize);
BasicBlock *CrashBlock = BasicBlock::Create(*C, "", &AFC.F, NextBB);
CrashTerm = new UnreachableInst(*C, CrashBlock);
BranchInst *NewTerm = BranchInst::Create(CrashBlock, NextBB, Cmp2);
ReplaceInstWithInst(CheckTerm, NewTerm);
} else {
CrashTerm = splitBlockAndInsertIfThen(Cmp, true);
}
Instruction *Crash =
generateCrashCode(CrashTerm, AddrLong, IsWrite, AccessSizeIndex);
Crash->setDebugLoc(OrigIns->getDebugLoc());
}
codegen_value ast::distribution::createConstructor(Module *module, IRBuilder<> &builder,
const string &ctor_name,
Type *parameter_type,
const vector<type_spec> ¶m_type_list,
Value *eval, Value *sample, Value *pdf, Value *emit,
Function *dtor) {
//create function accepting parameters as arguments
vector<Type*> arg_types;
for (auto it = param_type_list.begin(); it != param_type_list.end(); ++it) arg_types.push_back((*it)->llvm_type());
FunctionType *ft = FunctionType::get(state->types["dfunc"]->llvm_type(), arg_types, false);
Function *f = Function::Create(ft, Function::ExternalLinkage, ctor_name, module);
BasicBlock *bb = BasicBlock::Create(getGlobalContext(), "func_entry", f);
builder.SetInsertPoint(bb);
//setup arguments for the alloc call
Value *gd_scene = module->getNamedGlobal(".__gd_scene");
assert(gd_scene != NULL);
Value *scene_ptr = builder.CreateLoad(gd_scene);
//compute the shader flags
codegen_value flag_val = codegen_all_flags(module, builder);
return errors::codegen_call(flag_val, [&] (Value *&flag_bitmask) -> codegen_value {
//get memory for a new distribution object
Value *dfunc_ptr = state->types["dfunc"]->allocate(module, builder);
//initialize the object and dynamically allocate parameter memory (calling a builtin function)
Type* int_ptr_ty = Type::getInt32Ty(getGlobalContext())->getPointerTo();
vector<Type*> alloc_arg_types({state->types["scene_ptr"]->llvm_type(),
Type::getInt32Ty(getGlobalContext()), state->types["shader_flag"]->llvm_type(),
int_ptr_ty, int_ptr_ty, int_ptr_ty, int_ptr_ty,
dtor->getType(), dfunc_ptr->getType()});
FunctionType *alloc_type = FunctionType::get(Type::getInt32PtrTy(getGlobalContext()), alloc_arg_types, false);
Function *alloc_func = GetExternalFunction(module, "gd_builtin_alloc_dfunc", alloc_type);
int param_data_size = DataLayout(module).getTypeAllocSize(parameter_type);
Constant *param_size_arg = ConstantInt::get(getGlobalContext(), APInt(8*sizeof(int), param_data_size));
vector<Value*> alloc_args({scene_ptr, param_size_arg, flag_bitmask,
builder.CreatePointerCast(eval, int_ptr_ty),
builder.CreatePointerCast(sample, int_ptr_ty),
builder.CreatePointerCast(pdf, int_ptr_ty),
builder.CreatePointerCast(emit, int_ptr_ty),
dtor, dfunc_ptr});
Value *param_ptr = builder.CreatePointerCast(builder.CreateCall(alloc_func, alloc_args),
parameter_type->getPointerTo(), "dfunc_param_ptr");
//set each parameter
auto arg_it = f->arg_begin();
unsigned int field_idx = 0;
for (auto it = param_type_list.begin(); it != param_type_list.end(); ++it, ++arg_it, ++field_idx) {
Value *param_copy = (*it)->copy(arg_it, module, builder);
(*it)->store(param_copy, builder.CreateStructGEP(param_ptr, field_idx), module, builder);
}
//return the object
Value *rt_val = builder.CreateLoad(dfunc_ptr, "dist_ref");
builder.CreateRet(rt_val);
return f;
});
}
JITFunction*
CompilePipeline(Pipeline *pipeline, Thread *thread) {
size_t i = 0;
size_t size = 0;
std::unique_ptr<Module> owner = make_unique<Module>("PipelineFunction", thread->context);
Module *module = owner.get();
std::string fname = std::string("PipelineFunction") + std::to_string(thread->functions++);
size_t input_count = pipeline->inputData->objects.size();
size_t output_count = pipeline->outputData->objects.size();
size_t function_arg_count = 6;
size_t arg_count = input_count + output_count + (function_arg_count - 2);
size_t start_addr = input_count + output_count;
size_t end_addr = start_addr + 1;
size_t result_sizes_addr = end_addr + 1;
size_t thread_nr_addr = result_sizes_addr + 1;
IRBuilder<> *builder = &thread->builder;
auto passmanager = CreatePassManager(module, thread->jit.get());
module->setDataLayout(thread->jit->getTargetMachine().createDataLayout());
Type *int8_tpe = Type::getInt8Ty(thread->context);
Type *int8ptr_tpe = PointerType::get(int8_tpe, 0);
Type *int8ptrptr_tpe = PointerType::get(int8ptr_tpe, 0);
Type *int64_tpe = Type::getInt64Ty(thread->context);
Type *int64ptr_tpe = PointerType::get(int64_tpe, 0);
JITInformation info;
// arguments of the function
// the arguments are (void **result, void** inputs, size_t start, size_t end);
// note that we don't actually use void**, we use int8**, because LLVM does not support void pointers
std::vector<Type*> arguments(function_arg_count);
i = 0;
arguments[i++] = int8ptrptr_tpe; // void** results
arguments[i++] = int8ptrptr_tpe; // void** inputs
arguments[i++] = int64_tpe; // size_t start
arguments[i++] = int64_tpe; // size_t end
arguments[i++] = int64ptr_tpe; // size_t* result_sizes
arguments[i++] = int64_tpe; // size_t thread_nr
assert(i == function_arg_count);
/*for(auto inputs = pipeline->inputData->objects.begin(); inputs != pipeline->inputData->objects.end(); inputs++, i++) {
arguments[i] = PointerType::get(getLLVMType(thread->context, inputs->type), 0);
}
for(auto outputs = pipeline->outputData->objects.begin(); outputs != pipeline->outputData->objects.end(); outputs++, i++) {
arguments[i] = PointerType::get(getLLVMType(thread->context, outputs->type), 0);
}*/
// create the LLVM function
FunctionType *prototype = FunctionType::get(int64_tpe, arguments, false);
Function *function = Function::Create(prototype, GlobalValue::ExternalLinkage, fname, module);
function->setCallingConv(CallingConv::C);
// create the basic blocks
BasicBlock *loop_entry = BasicBlock::Create(thread->context, "entry", function, 0);
BasicBlock *loop_cond = BasicBlock::Create(thread->context, "for.cond", function, 0);
BasicBlock *loop_body = BasicBlock::Create(thread->context, "for.body", function, 0);
BasicBlock *loop_inc = BasicBlock::Create(thread->context, "for.inc", function, 0);
BasicBlock *loop_end = BasicBlock::Create(thread->context, "for.end", function, 0);
info.builder = &thread->builder;
info.context = &thread->context;
info.function = function;
info.loop_entry = loop_entry;
info.loop_cond = loop_cond;
info.loop_body = loop_body;
info.loop_inc = loop_inc;
info.loop_end = loop_end;
info.current = loop_body;
#ifndef _NOTDEBUG
// argument names (for debug purposes only)
std::vector<std::string> argument_names(arg_count);
i = 0;
for(auto inputs = pipeline->inputData->objects.begin(); inputs != pipeline->inputData->objects.end(); inputs++, i++) {
argument_names[i] = std::string("inputs") + std::to_string(i);
}
for(auto outputs = pipeline->outputData->objects.begin(); outputs != pipeline->outputData->objects.end(); outputs++, i++) {
argument_names[i] = std::string("outputs") + std::to_string(i - input_count);
}
argument_names[i++] = "start";
argument_names[i++] = "end";
argument_names[i++] = "result_sizes";
argument_names[i++] = "thread_nr";
#endif
std::vector<AllocaInst*> argument_addresses(arg_count);
builder->SetInsertPoint(loop_entry);
{
// allocate space for the arguments
auto args = function->arg_begin();
i = 0;
for(auto outputs = pipeline->outputData->objects.begin(); outputs != pipeline->outputData->objects.end(); outputs++, i++) {
Type *column_type = PointerType::get(getLLVMType(thread->context, outputs->source->type), 0);
Value *voidptrptr = builder->CreateGEP(int8ptr_tpe, &*args, ConstantInt::get(int64_tpe, i, true));
Value *voidptr = builder->CreateLoad(voidptrptr, "voidptr");
Value *columnptr = builder->CreatePointerCast(voidptr, column_type);
argument_addresses[i] = builder->CreateAlloca(column_type, nullptr, argument_names[i]);
builder->CreateStore(columnptr, argument_addresses[i]);
outputs->alloca_address = (void*) argument_addresses[i];
if (size == 0 || size == 1) {
assert(outputs->source->size >= 0);
size = outputs->source->size;
}
assert(size == outputs->source->size || outputs->source->size == 1);
}
args++;
for(auto inputs = pipeline->inputData->objects.begin(); inputs != pipeline->inputData->objects.end(); inputs++, i++) {
Type *column_type = PointerType::get(getLLVMType(thread->context, inputs->source->type), 0);
Value *voidptrptr = builder->CreateGEP(int8ptr_tpe, &*args, ConstantInt::get(int64_tpe, i - output_count, true));
Value *voidptr = builder->CreateLoad(voidptrptr, "voidptr");
Value *columnptr = builder->CreatePointerCast(voidptr, column_type);
argument_addresses[i] = builder->CreateAlloca(column_type, nullptr, argument_names[i]);
builder->CreateStore(columnptr, argument_addresses[i]);
inputs->alloca_address = (void*) argument_addresses[i];
if (size == 0 || size == 1) {
assert(inputs->source->size >= 0);
size = inputs->source->size;
}
assert(size == inputs->source->size || inputs->source->size == 1);
}
args++;
argument_addresses[i] = builder->CreateAlloca(arguments[2], nullptr, argument_names[i]);
builder->CreateStore(&*args, argument_addresses[i]);
args++; i++;
argument_addresses[i] = builder->CreateAlloca(arguments[3], nullptr, argument_names[i]);
builder->CreateStore(&*args, argument_addresses[i]);
args++; i++;
argument_addresses[i] = builder->CreateAlloca(arguments[4], nullptr, argument_names[i]);
builder->CreateStore(&*args, argument_addresses[i]);
args++; i++;
argument_addresses[i] = builder->CreateAlloca(arguments[5], nullptr, argument_names[i]);
builder->CreateStore(&*args, argument_addresses[i]);
args++; i++;
assert(args == function->arg_end());
assert(i == arg_count);
info.index_addr = argument_addresses[start_addr];
info.thread_addr = argument_addresses[thread_nr_addr];
PerformInitialization(info, pipeline->operation);
builder->CreateBr(loop_cond);
}
// for loop condition: index < end
builder->SetInsertPoint(loop_cond);
{
LoadInst *index = builder->CreateLoad(argument_addresses[start_addr], "index");
LoadInst *end = builder->CreateLoad(argument_addresses[end_addr], "end");
Value *condition = builder->CreateICmpSLT(index, end, "index < end");
builder->CreateCondBr(condition, loop_body, loop_end);
}
// loop body: perform the computation
builder->SetInsertPoint(loop_body);
{
LoadInst *index = builder->CreateLoad(argument_addresses[start_addr], "index");
info.index = index;
info.index_addr = argument_addresses[start_addr];
// perform the computation over the given index
// we don't use the return value because the final assignment has already taken place
Value *v = PerformOperation(info, thread->builder, thread->context, pipeline->operation, pipeline->inputData, pipeline->outputData);
if (v == NULL) {
// failed to perform operation
printf("Failed to compile pipeline %s\n", pipeline->name);
return NULL;
}
builder->CreateBr(loop_inc);
}
// loop increment: index++
builder->SetInsertPoint(loop_inc);
{
LoadInst *index = builder->CreateLoad(argument_addresses[start_addr], "index");
Value *incremented_index = builder->CreateAdd(index, ConstantInt::get(int64_tpe, 1, true), "index++");
builder->CreateStore(incremented_index, argument_addresses[start_addr]);
builder->CreateBr(loop_cond);
}
// loop end: return; (nothing happens here because we have no return value)
builder->SetInsertPoint(loop_end);
{
// return the output size of each of the columns
int i = 0;
Value *result_sizes = builder->CreateLoad(argument_addresses[result_sizes_addr], "result_sizes[]");
for(auto it = pipeline->outputData->objects.begin(); it != pipeline->outputData->objects.end(); it++) {
Value* output_count;
if (it->index_addr) {
output_count = builder->CreateLoad((Value*) it->index_addr, "count");
} else {
output_count = ConstantInt::get(int64_tpe, 1, true);
}
Value *output_addr = builder->CreateGEP(int64_tpe, result_sizes, ConstantInt::get(int64_tpe, i, true));
builder->CreateStore(output_count, output_addr);
i++;
}
builder->CreateRet(ConstantInt::get(int64_tpe, 0, true));
}
#ifndef _NOTDEBUG
verifyFunction(*function);
verifyModule(*module);
#endif
//printf("LLVM for pipeline %s\n", pipeline->name);
module->dump();
passmanager->run(*function);
// dump generated LLVM code
//module->dump();
auto handle = thread->jit->addModule(std::move(owner));
jit_function compiled_function = (jit_function) thread->jit->findSymbol(fname).getAddress();
if (!compiled_function) {
printf("Error creating function.\n");
return NULL;
}
JITFunction *jf = CreateJITFunction(thread, pipeline);
jf->size = size;
jf->function = compiled_function;
jf->jit = thread->jit.get();
jf->handle = handle;
assert(jf->function);
return jf;
}