static LLVMTypeRef llvmTypeForSignature(oThreadContextRef ctx, oSignatureRef sig) {
uword i;
LLVMTypeRef fnType, retType;
oParameterRef* params = (oParameterRef*)oArrayDataPointer(sig->parameters);
LLVMTypeRef* paramTypes = (LLVMTypeRef*)oMalloc(sizeof(LLVMTypeRef) * sig->parameters->num_elements);
if(sig->retType->kind == o_T_OBJECT) {
retType = LLVMPointerType(sig->retType->llvmType, 0);
} else {
retType = sig->retType->llvmType;
}
for(i = 0; i < sig->parameters->num_elements; ++i) {
if(params[i]->type->kind == o_T_OBJECT) {
paramTypes[i] = LLVMPointerType(params[i]->type->llvmType, 0);
} else {
paramTypes[i] = params[i]->type->llvmType;
}
}
fnType = LLVMFunctionType(retType, paramTypes, sig->parameters->num_elements, o_false);
oFree(paramTypes);
return fnType;
}
void gendesc_basetype(compile_t* c, LLVMTypeRef desc_type)
{
LLVMTypeRef params[DESC_LENGTH];
params[DESC_ID] = c->i32;
params[DESC_SIZE] = c->i32;
params[DESC_TRAIT_COUNT] = c->i32;
params[DESC_FIELD_COUNT] = c->i32;
params[DESC_FIELD_OFFSET] = c->i32;
params[DESC_TRACE] = c->trace_fn;
params[DESC_SERIALISE] = c->trace_fn;
params[DESC_DESERIALISE] = c->trace_fn;
params[DESC_DISPATCH] = c->dispatch_fn;
params[DESC_FINALISE] = c->final_fn;
params[DESC_EVENT_NOTIFY] = c->i32;
params[DESC_TRAITS] = LLVMPointerType(LLVMArrayType(c->i32, 0), 0);
params[DESC_FIELDS] = LLVMPointerType(
LLVMArrayType(c->field_descriptor, 0), 0);
params[DESC_VTABLE] = LLVMArrayType(c->void_ptr, 0);
LLVMStructSetBody(desc_type, params, DESC_LENGTH, false);
}
static LLVMValueRef make_field_list(compile_t* c, reach_type_t* t)
{
// The list is an array of field descriptors.
uint32_t count;
if(t->underlying == TK_TUPLETYPE)
count = t->field_count;
else
count = 0;
LLVMTypeRef field_type = LLVMArrayType(c->field_descriptor, count);
// If we aren't a tuple, return a null pointer to a list.
if(count == 0)
return LLVMConstNull(LLVMPointerType(field_type, 0));
// Create a constant array of field descriptors.
size_t buf_size = count * sizeof(LLVMValueRef);
LLVMValueRef* list = (LLVMValueRef*)ponyint_pool_alloc_size(buf_size);
for(uint32_t i = 0; i < count; i++)
{
LLVMValueRef fdesc[2];
fdesc[0] = LLVMConstInt(c->i32,
LLVMOffsetOfElement(c->target_data, t->primitive, i), false);
if(t->fields[i].type->desc != NULL)
{
// We are a concrete type.
fdesc[1] = LLVMConstBitCast(t->fields[i].type->desc,
c->descriptor_ptr);
} else {
// We aren't a concrete type.
fdesc[1] = LLVMConstNull(c->descriptor_ptr);
}
list[i] = LLVMConstStructInContext(c->context, fdesc, 2, false);
}
LLVMValueRef field_array = LLVMConstArray(c->field_descriptor, list, count);
// Create a global to hold the array.
const char* name = genname_fieldlist(t->name);
LLVMValueRef global = LLVMAddGlobal(c->module, field_type, name);
LLVMSetGlobalConstant(global, true);
LLVMSetLinkage(global, LLVMPrivateLinkage);
LLVMSetInitializer(global, field_array);
ponyint_pool_free_size(buf_size, list);
return global;
}
static bool dynamic_tuple_element(compile_t* c, LLVMValueRef ptr,
LLVMValueRef desc, ast_t* pattern, LLVMBasicBlockRef next_block, int elem)
{
// If we have a capture, generate the alloca now.
switch(ast_id(pattern))
{
case TK_MATCH_CAPTURE:
if(gen_localdecl(c, pattern) == NULL)
return false;
break;
default: {}
}
// Get the field offset and field descriptor from the tuple descriptor.
LLVMValueRef field_info = gendesc_fieldinfo(c, desc, elem);
LLVMValueRef field_ptr = gendesc_fieldptr(c, ptr, field_info);
LLVMValueRef field_desc = gendesc_fielddesc(c, field_info);
// If we have a null descriptor, load the object.
LLVMBasicBlockRef null_block = codegen_block(c, "null_desc");
LLVMBasicBlockRef nonnull_block = codegen_block(c, "nonnull_desc");
LLVMBasicBlockRef continue_block = codegen_block(c, "merge_desc");
LLVMValueRef test = LLVMBuildIsNull(c->builder, field_desc, "");
LLVMBuildCondBr(c->builder, test, null_block, nonnull_block);
// Load the object, load its descriptor, and continue from there.
LLVMPositionBuilderAtEnd(c->builder, null_block);
LLVMTypeRef ptr_type = LLVMPointerType(c->object_ptr, 0);
LLVMValueRef object_ptr = LLVMBuildIntToPtr(c->builder, field_ptr, ptr_type,
"");
LLVMValueRef object = LLVMBuildLoad(c->builder, object_ptr, "");
LLVMValueRef object_desc = gendesc_fetch(c, object);
if(!dynamic_match_object(c, object, object_desc, pattern, next_block))
return false;
LLVMBuildBr(c->builder, continue_block);
// Continue with the pointer and descriptor.
LLVMPositionBuilderAtEnd(c->builder, nonnull_block);
if(!dynamic_match_ptr(c, field_ptr, field_desc, pattern, next_block))
return false;
LLVMBuildBr(c->builder, continue_block);
// Merge the two branches.
LLVMPositionBuilderAtEnd(c->builder, continue_block);
return true;
}
/*
* Build LLVM function that exercises the unary operator builder.
*/
static LLVMValueRef
build_unary_test_func(struct gallivm_state *gallivm,
const struct unary_test_t *test)
{
struct lp_type type = lp_type_float_vec(32, lp_native_vector_width);
LLVMContextRef context = gallivm->context;
LLVMModuleRef module = gallivm->module;
LLVMTypeRef vf32t = lp_build_vec_type(gallivm, type);
LLVMTypeRef args[2] = { LLVMPointerType(vf32t, 0), LLVMPointerType(vf32t, 0) };
LLVMValueRef func = LLVMAddFunction(module, test->name,
LLVMFunctionType(LLVMVoidTypeInContext(context),
args, Elements(args), 0));
LLVMValueRef arg0 = LLVMGetParam(func, 0);
LLVMValueRef arg1 = LLVMGetParam(func, 1);
LLVMBuilderRef builder = gallivm->builder;
LLVMBasicBlockRef block = LLVMAppendBasicBlockInContext(context, func, "entry");
LLVMValueRef ret;
struct lp_build_context bld;
lp_build_context_init(&bld, gallivm, type);
LLVMSetFunctionCallConv(func, LLVMCCallConv);
LLVMPositionBuilderAtEnd(builder, block);
arg1 = LLVMBuildLoad(builder, arg1, "");
ret = test->builder(&bld, arg1);
LLVMBuildStore(builder, ret, arg0);
LLVMBuildRetVoid(builder);
gallivm_verify_function(gallivm, func);
return func;
}
static void make_box_type(compile_t* c, gentype_t* g)
{
if(g->primitive == NULL)
return;
if(g->structure == NULL)
{
const char* box_name = genname_box(g->type_name);
g->structure = LLVMGetTypeByName(c->module, box_name);
if(g->structure == NULL)
g->structure = LLVMStructCreateNamed(c->context, box_name);
}
if(LLVMIsOpaqueStruct(g->structure))
{
LLVMTypeRef elements[2];
elements[0] = LLVMPointerType(g->desc_type, 0);
elements[1] = g->primitive;
LLVMStructSetBody(g->structure, elements, 2, false);
}
g->structure_ptr = LLVMPointerType(g->structure, 0);
}
static LLVMValueRef make_trait_list(compile_t* c, gentype_t* g)
{
// The list is an array of integers.
uint32_t count = trait_count(c, g);
// If we have no traits, return a null pointer to a list.
if(count == 0)
return LLVMConstNull(LLVMPointerType(LLVMArrayType(c->i32, 0), 0));
// Sort the trait identifiers.
size_t tid_size = count * sizeof(uint32_t);
uint32_t* tid = (uint32_t*)pool_alloc_size(tid_size);
reachable_type_t* t = reach_type(c->reachable, g->type_name);
assert(t != NULL);
size_t i = HASHMAP_BEGIN;
size_t index = 0;
reachable_type_t* provide;
while((provide = reachable_type_cache_next(&t->subtypes, &i)) != NULL)
tid[index++] = provide->type_id;
qsort(tid, index, sizeof(uint32_t), cmp_uint32);
index = unique_uint32(tid, index);
// Create a constant array of trait identifiers.
size_t list_size = index * sizeof(LLVMValueRef);
LLVMValueRef* list = (LLVMValueRef*)pool_alloc_size(list_size);
for(i = 0; i < index; i++)
list[i] = LLVMConstInt(c->i32, tid[i], false);
count = (uint32_t)index;
LLVMValueRef trait_array = LLVMConstArray(c->i32, list, count);
// Create a global to hold the array.
const char* name = genname_traitlist(g->type_name);
LLVMTypeRef type = LLVMArrayType(c->i32, count);
LLVMValueRef global = LLVMAddGlobal(c->module, type, name);
LLVMSetGlobalConstant(global, true);
LLVMSetLinkage(global, LLVMInternalLinkage);
LLVMSetInitializer(global, trait_array);
pool_free_size(tid_size, tid);
pool_free_size(list_size, list);
return global;
}
static void trace_array_elements(compile_t* c, reach_type_t* t,
LLVMValueRef ctx, LLVMValueRef object, LLVMValueRef pointer)
{
// Get the type argument for the array. This will be used to generate the
// per-element trace call.
ast_t* typeargs = ast_childidx(t->ast, 2);
ast_t* typearg = ast_child(typeargs);
if(!gentrace_needed(typearg))
return;
reach_type_t* t_elem = reach_type(c->reach, typearg);
pointer = LLVMBuildBitCast(c->builder, pointer,
LLVMPointerType(t_elem->use_type, 0), "");
LLVMBasicBlockRef entry_block = LLVMGetInsertBlock(c->builder);
LLVMBasicBlockRef cond_block = codegen_block(c, "cond");
LLVMBasicBlockRef body_block = codegen_block(c, "body");
LLVMBasicBlockRef post_block = codegen_block(c, "post");
// Read the size.
LLVMValueRef size = field_value(c, object, 1);
LLVMBuildBr(c->builder, cond_block);
// While the index is less than the size, trace an element. The initial
// index when coming from the entry block is zero.
LLVMPositionBuilderAtEnd(c->builder, cond_block);
LLVMValueRef phi = LLVMBuildPhi(c->builder, c->intptr, "");
LLVMValueRef zero = LLVMConstInt(c->intptr, 0, false);
LLVMAddIncoming(phi, &zero, &entry_block, 1);
LLVMValueRef test = LLVMBuildICmp(c->builder, LLVMIntULT, phi, size, "");
LLVMBuildCondBr(c->builder, test, body_block, post_block);
// The phi node is the index. Get the element and trace it.
LLVMPositionBuilderAtEnd(c->builder, body_block);
LLVMValueRef elem_ptr = LLVMBuildGEP(c->builder, pointer, &phi, 1, "elem");
LLVMValueRef elem = LLVMBuildLoad(c->builder, elem_ptr, "");
gentrace(c, ctx, elem, typearg);
// Add one to the phi node and branch back to the cond block.
LLVMValueRef one = LLVMConstInt(c->intptr, 1, false);
LLVMValueRef inc = LLVMBuildAdd(c->builder, phi, one, "");
body_block = LLVMGetInsertBlock(c->builder);
LLVMAddIncoming(phi, &inc, &body_block, 1);
LLVMBuildBr(c->builder, cond_block);
LLVMPositionBuilderAtEnd(c->builder, post_block);
}
static LLVMValueRef llvm_load_const_buffer(
struct lp_build_tgsi_context * bld_base,
LLVMValueRef OffsetValue,
unsigned ConstantAddressSpace)
{
LLVMValueRef offset[2] = {
LLVMConstInt(LLVMInt64TypeInContext(bld_base->base.gallivm->context), 0, false),
OffsetValue
};
LLVMTypeRef const_ptr_type = LLVMPointerType(LLVMArrayType(LLVMVectorType(bld_base->base.elem_type, 4), 1024),
ConstantAddressSpace);
LLVMValueRef const_ptr = LLVMBuildIntToPtr(bld_base->base.gallivm->builder, lp_build_const_int32(bld_base->base.gallivm, 0), const_ptr_type, "");
LLVMValueRef ptr = LLVMBuildGEP(bld_base->base.gallivm->builder, const_ptr, offset, 2, "");
return LLVMBuildLoad(bld_base->base.gallivm->builder, ptr, "");
}
static void pointer_delete(compile_t* c, reach_type_t* t, reach_type_t* t_elem)
{
FIND_METHOD("_delete");
LLVMTypeRef params[3];
params[0] = t->use_type;
params[1] = c->intptr;
params[2] = c->intptr;
start_function(c, m, t_elem->use_type, params, 3);
// Set up a constant integer for the allocation size.
size_t size = (size_t)LLVMABISizeOfType(c->target_data, t_elem->use_type);
LLVMValueRef l_size = LLVMConstInt(c->intptr, size, false);
LLVMValueRef ptr = LLVMGetParam(m->func, 0);
LLVMValueRef n = LLVMGetParam(m->func, 1);
LLVMValueRef len = LLVMGetParam(m->func, 2);
LLVMValueRef elem_ptr = LLVMBuildBitCast(c->builder, ptr,
LLVMPointerType(t_elem->use_type, 0), "");
LLVMValueRef result = LLVMBuildLoad(c->builder, elem_ptr, "");
LLVMValueRef dst = LLVMBuildPtrToInt(c->builder, elem_ptr, c->intptr, "");
LLVMValueRef offset = LLVMBuildMul(c->builder, n, l_size, "");
LLVMValueRef src = LLVMBuildAdd(c->builder, dst, offset, "");
LLVMValueRef elen = LLVMBuildMul(c->builder, len, l_size, "");
LLVMValueRef args[5];
args[0] = LLVMBuildIntToPtr(c->builder, dst, c->void_ptr, "");
args[1] = LLVMBuildIntToPtr(c->builder, src, c->void_ptr, "");
args[2] = elen;
args[3] = LLVMConstInt(c->i32, 1, false);
args[4] = LLVMConstInt(c->i1, 0, false);
// llvm.memmove.*(ptr, ptr + (n * sizeof(elem)), len * sizeof(elem))
if(target_is_ilp32(c->opt->triple))
{
gencall_runtime(c, "llvm.memmove.p0i8.p0i8.i32", args, 5, "");
} else {
gencall_runtime(c, "llvm.memmove.p0i8.p0i8.i64", args, 5, "");
}
// Return ptr[0].
LLVMBuildRet(c->builder, result);
codegen_finishfun(c);
}
static LLVMValueRef
translateIdLval(SymbolTable *TyTable, SymbolTable *ValTable, ASTNode *Node) {
Type *IdType = (Type*) symTableFind(ValTable, Node->Value);
LLVMValueRef IdValue;
if (IdType->EscapedLevel > 0) {
LLVMTypeRef LLVMType = getLLVMTypeFromType(TyTable, IdType);
LLVMValueRef EVPtr = getEscapedVar(ValTable, Node->Value, Node->EscapedLevel);
LLVMValueRef EVLoad = LLVMBuildLoad(Builder, EVPtr, "");
IdValue = LLVMBuildBitCast(Builder, EVLoad, LLVMPointerType(LLVMType, 0), "");
} else {
IdValue = resolveAliasId(ValTable, Node->Value, &toValName, &symTableFindLocal);
}
return IdValue;
}
/**
* lp_build_assert.
*
* Build an assertion in LLVM IR by building a function call to the
* lp_assert() function above.
*
* \param condition should be an 'i1' or 'i32' value
* \param msg a string to print if the assertion fails.
*/
LLVMValueRef
lp_build_assert(LLVMBuilderRef builder, LLVMValueRef condition,
const char *msg)
{
LLVMModuleRef module;
LLVMTypeRef arg_types[2];
LLVMValueRef msg_string, assert_func, params[2], r;
module = LLVMGetGlobalParent(LLVMGetBasicBlockParent(
LLVMGetInsertBlock(builder)));
msg_string = lp_build_const_string_variable(module, msg, strlen(msg) + 1);
arg_types[0] = LLVMInt32Type();
arg_types[1] = LLVMPointerType(LLVMInt8Type(), 0);
/* lookup the lp_assert function */
assert_func = LLVMGetNamedFunction(module, "lp_assert");
/* Create the assertion function if not found */
if (!assert_func) {
LLVMTypeRef func_type =
LLVMFunctionType(LLVMVoidType(), arg_types, 2, 0);
assert_func = LLVMAddFunction(module, "lp_assert", func_type);
LLVMSetFunctionCallConv(assert_func, LLVMCCallConv);
LLVMSetLinkage(assert_func, LLVMExternalLinkage);
LLVMAddGlobalMapping(lp_build_engine, assert_func,
func_to_pointer((func_pointer)lp_assert));
}
assert(assert_func);
/* build function call param list */
params[0] = LLVMBuildZExt(builder, condition, arg_types[0], "");
params[1] = LLVMBuildBitCast(builder, msg_string, arg_types[1], "");
/* check arg types */
assert(LLVMTypeOf(params[0]) == arg_types[0]);
assert(LLVMTypeOf(params[1]) == arg_types[1]);
r = LLVMBuildCall(builder, assert_func, params, 2, "");
return r;
}
static void pointer_update(compile_t* c, reach_type_t* t, reach_type_t* t_elem)
{
FIND_METHOD("_update");
LLVMTypeRef params[3];
params[0] = t->use_type;
params[1] = c->intptr;
params[2] = t_elem->use_type;
start_function(c, m, t_elem->use_type, params, 3);
LLVMValueRef ptr = LLVMGetParam(m->func, 0);
LLVMValueRef index = LLVMGetParam(m->func, 1);
LLVMValueRef elem_ptr = LLVMBuildBitCast(c->builder, ptr,
LLVMPointerType(t_elem->use_type, 0), "");
LLVMValueRef loc = LLVMBuildGEP(c->builder, elem_ptr, &index, 1, "");
LLVMValueRef result = LLVMBuildLoad(c->builder, loc, "");
LLVMBuildStore(c->builder, LLVMGetParam(m->func, 2), loc);
LLVMBuildRet(c->builder, result);
codegen_finishfun(c);
}
LLVMValueRef gen_call(struct node *ast)
{
LLVMValueRef func, *arg_list = NULL;
struct node *n;
int arg_count, i;
func = LLVMBuildBitCast(builder,
rvalue_to_lvalue(codegen(ast->one)),
LLVMPointerType(TYPE_FUNC, 0),
"");
arg_count = count_chain(ast->two);
arg_list = calloc(sizeof(LLVMValueRef), arg_count);
if (arg_count > 0 && arg_list == NULL)
generror("out of memory");
for (i = 0, n = ast->two; i < arg_count; i++, n = n->two)
arg_list[arg_count - i - 1] = codegen(n->one);
return LLVMBuildCall(builder, func, arg_list, arg_count, "");
}
static bool dynamic_value_ptr(compile_t* c, LLVMValueRef ptr,
LLVMValueRef desc, ast_t* pattern, LLVMBasicBlockRef next_block)
{
// Get the type of the right-hand side of the pattern's eq() function.
ast_t* param_type = eq_param_type(c, pattern);
// Check the runtime type. We pass a pointer to the fields because we may
// still need to match a tuple type inside a type expression.
if(!check_type(c, ptr, desc, param_type, next_block))
return false;
// We now know that ptr points to something of type pattern_type, and that
// it isn't a boxed primitive, as that would go through the other path, ie
// dynamic_match_object(). We also know it isn't an unboxed tuple. We can
// load from ptr with a type based on the static type of the pattern.
reach_type_t* t = reach_type(c->reach, param_type);
LLVMTypeRef ptr_type = LLVMPointerType(t->use_type, 0);
ptr = LLVMBuildIntToPtr(c->builder, ptr, ptr_type, "");
LLVMValueRef value = LLVMBuildLoad(c->builder, ptr, "");
return check_value(c, pattern, param_type, value, next_block);
}
/**
* Gather elements from scatter positions in memory into a single vector.
*
* @param src_width src element width
* @param dst_width result element width (source will be expanded to fit)
* @param length length of the offsets,
* @param base_ptr base pointer, should be a i8 pointer type.
* @param offsets vector with offsets
*/
LLVMValueRef
lp_build_gather(LLVMBuilderRef builder,
unsigned length,
unsigned src_width,
unsigned dst_width,
LLVMValueRef base_ptr,
LLVMValueRef offsets)
{
LLVMTypeRef src_type = LLVMIntType(src_width);
LLVMTypeRef src_ptr_type = LLVMPointerType(src_type, 0);
LLVMTypeRef dst_elem_type = LLVMIntType(dst_width);
LLVMTypeRef dst_vec_type = LLVMVectorType(dst_elem_type, length);
LLVMValueRef res;
unsigned i;
res = LLVMGetUndef(dst_vec_type);
for(i = 0; i < length; ++i) {
LLVMValueRef index = LLVMConstInt(LLVMInt32Type(), i, 0);
LLVMValueRef elem_offset;
LLVMValueRef elem_ptr;
LLVMValueRef elem;
elem_offset = LLVMBuildExtractElement(builder, offsets, index, "");
elem_ptr = LLVMBuildGEP(builder, base_ptr, &elem_offset, 1, "");
elem_ptr = LLVMBuildBitCast(builder, elem_ptr, src_ptr_type, "");
elem = LLVMBuildLoad(builder, elem_ptr, "");
assert(src_width <= dst_width);
if(src_width > dst_width)
elem = LLVMBuildTrunc(builder, elem, dst_elem_type, "");
if(src_width < dst_width)
elem = LLVMBuildZExt(builder, elem, dst_elem_type, "");
res = LLVMBuildInsertElement(builder, res, elem, index, "");
}
return res;
}
static LLVMValueRef dispatch_function(compile_t* c, ast_t* from, gentype_t* g,
LLVMValueRef l_value, const char* method_name, ast_t* typeargs)
{
LLVMValueRef func;
if(g->use_type == c->object_ptr)
{
// Virtual, get the function by selector colour.
uint32_t index = genfun_vtable_index(c, g, method_name, typeargs);
assert(index != (uint32_t)-1);
// Get the function from the vtable.
func = gendesc_vtable(c, l_value, index);
// Cast to the right function type.
LLVMTypeRef f_type = genfun_sig(c, g, method_name, typeargs);
if(f_type == NULL)
{
ast_error(from, "couldn't create a signature for '%s'", method_name);
return NULL;
}
f_type = LLVMPointerType(f_type, 0);
func = LLVMBuildBitCast(c->builder, func, f_type, "method");
} else {
// Static, get the actual function.
func = genfun_proto(c, g, method_name, typeargs);
if(func == NULL)
{
ast_error(from, "couldn't locate '%s'", method_name);
return NULL;
}
}
return func;
}
static bool dynamic_capture_ptr(compile_t* c, LLVMValueRef ptr,
LLVMValueRef desc, ast_t* pattern, LLVMBasicBlockRef next_block)
{
// Here, ptr is a pointer to a tuple field. It could be a primitive, an
// object, or a nested tuple.
ast_t* pattern_type = ast_type(pattern);
// Check the runtime type. We pass a pointer to the fields because we may
// still need to match a tuple type inside a type expression.
if(!check_type(c, ptr, desc, pattern_type, next_block))
return false;
// We now know that ptr points to something of type pattern_type, and that
// it isn't a boxed primitive or tuple, as that would go through the other
// path, ie dynamic_match_object(). We also know it isn't an unboxed tuple.
// We can load from ptr with a type based on the static type of the pattern.
reach_type_t* t = reach_type(c->reach, pattern_type);
LLVMTypeRef ptr_type = LLVMPointerType(t->use_type, 0);
ptr = LLVMBuildIntToPtr(c->builder, ptr, ptr_type, "");
LLVMValueRef value = LLVMBuildLoad(c->builder, ptr, "");
return gen_assign_value(c, pattern, value, pattern_type) != NULL;
}
static LLVMValueRef make_trait_list(compile_t* c, gentype_t* g)
{
// The list is an array of integers.
uint32_t count = trait_count(c, g);
LLVMTypeRef type = LLVMArrayType(c->i32, count);
// If we have no traits, return a null pointer to a list.
if(count == 0)
return LLVMConstNull(LLVMPointerType(type, 0));
// Create a constant array of trait identifiers.
size_t buf_size = count *sizeof(LLVMValueRef);
LLVMValueRef* list = (LLVMValueRef*)pool_alloc_size(buf_size);
reachable_type_t* t = reach_type(c->reachable, g->type_name);
assert(t != NULL);
size_t i = HASHMAP_BEGIN;
size_t index = 0;
reachable_type_t* provide;
while((provide = reachable_type_cache_next(&t->subtypes, &i)) != NULL)
list[index++] = make_type_id(c, provide->name);
LLVMValueRef trait_array = LLVMConstArray(c->i32, list, count);
// Create a global to hold the array.
const char* name = genname_traitlist(g->type_name);
LLVMValueRef global = LLVMAddGlobal(c->module, type, name);
LLVMSetGlobalConstant(global, true);
LLVMSetLinkage(global, LLVMInternalLinkage);
LLVMSetInitializer(global, trait_array);
pool_free_size(buf_size, list);
return global;
}
/**
* Get the pointer to one element from scatter positions in memory.
*
* @sa lp_build_gather()
*/
LLVMValueRef
lp_build_gather_elem_ptr(struct gallivm_state *gallivm,
unsigned length,
LLVMValueRef base_ptr,
LLVMValueRef offsets,
unsigned i)
{
LLVMValueRef offset;
LLVMValueRef ptr;
assert(LLVMTypeOf(base_ptr) == LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0));
if (length == 1) {
assert(i == 0);
offset = offsets;
} else {
LLVMValueRef index = lp_build_const_int32(gallivm, i);
offset = LLVMBuildExtractElement(gallivm->builder, offsets, index, "");
}
ptr = LLVMBuildGEP(gallivm->builder, base_ptr, &offset, 1, "");
return ptr;
}
/**
* Fetch a pixel into a 4 float AoS.
*
* \param format_desc describes format of the image we're fetching from
* \param ptr address of the pixel block (or the texel if uncompressed)
* \param i, j the sub-block pixel coordinates. For non-compressed formats
* these will always be (0, 0).
* \return a 4 element vector with the pixel's RGBA values.
*/
LLVMValueRef
lp_build_fetch_rgba_aos(struct gallivm_state *gallivm,
const struct util_format_description *format_desc,
struct lp_type type,
LLVMValueRef base_ptr,
LLVMValueRef offset,
LLVMValueRef i,
LLVMValueRef j)
{
LLVMBuilderRef builder = gallivm->builder;
unsigned num_pixels = type.length / 4;
struct lp_build_context bld;
assert(type.length <= LP_MAX_VECTOR_LENGTH);
assert(type.length % 4 == 0);
lp_build_context_init(&bld, gallivm, type);
/*
* Trivial case
*
* The format matches the type (apart of a swizzle) so no need for
* scaling or converting.
*/
if (format_matches_type(format_desc, type) &&
format_desc->block.bits <= type.width * 4 &&
util_is_power_of_two(format_desc->block.bits)) {
LLVMValueRef packed;
/*
* The format matches the type (apart of a swizzle) so no need for
* scaling or converting.
*/
packed = lp_build_gather(gallivm, type.length/4,
format_desc->block.bits, type.width*4,
base_ptr, offset);
assert(format_desc->block.bits <= type.width * type.length);
packed = LLVMBuildBitCast(gallivm->builder, packed,
lp_build_vec_type(gallivm, type), "");
return lp_build_format_swizzle_aos(format_desc, &bld, packed);
}
/*
* Bit arithmetic
*/
if (format_desc->layout == UTIL_FORMAT_LAYOUT_PLAIN &&
(format_desc->colorspace == UTIL_FORMAT_COLORSPACE_RGB ||
format_desc->colorspace == UTIL_FORMAT_COLORSPACE_ZS) &&
format_desc->block.width == 1 &&
format_desc->block.height == 1 &&
util_is_power_of_two(format_desc->block.bits) &&
format_desc->block.bits <= 32 &&
format_desc->is_bitmask &&
!format_desc->is_mixed &&
(format_desc->channel[0].type == UTIL_FORMAT_TYPE_UNSIGNED ||
format_desc->channel[1].type == UTIL_FORMAT_TYPE_UNSIGNED)) {
LLVMValueRef tmps[LP_MAX_VECTOR_LENGTH/4];
LLVMValueRef res;
unsigned k;
/*
* Unpack a pixel at a time into a <4 x float> RGBA vector
*/
for (k = 0; k < num_pixels; ++k) {
LLVMValueRef packed;
packed = lp_build_gather_elem(gallivm, num_pixels,
format_desc->block.bits, 32,
base_ptr, offset, k);
tmps[k] = lp_build_unpack_arith_rgba_aos(gallivm,
format_desc,
packed);
}
/*
* Type conversion.
*
* TODO: We could avoid floating conversion for integer to
* integer conversions.
*/
if (gallivm_debug & GALLIVM_DEBUG_PERF && !type.floating) {
debug_printf("%s: unpacking %s with floating point\n",
__FUNCTION__, format_desc->short_name);
}
lp_build_conv(gallivm,
lp_float32_vec4_type(),
type,
tmps, num_pixels, &res, 1);
return lp_build_format_swizzle_aos(format_desc, &bld, res);
}
/*
* YUV / subsampled formats
*/
if (format_desc->layout == UTIL_FORMAT_LAYOUT_SUBSAMPLED) {
struct lp_type tmp_type;
LLVMValueRef tmp;
memset(&tmp_type, 0, sizeof tmp_type);
tmp_type.width = 8;
tmp_type.length = num_pixels * 4;
tmp_type.norm = TRUE;
tmp = lp_build_fetch_subsampled_rgba_aos(gallivm,
format_desc,
num_pixels,
base_ptr,
offset,
i, j);
lp_build_conv(gallivm,
tmp_type, type,
&tmp, 1, &tmp, 1);
return tmp;
}
/*
* Fallback to util_format_description::fetch_rgba_8unorm().
*/
if (format_desc->fetch_rgba_8unorm &&
!type.floating && type.width == 8 && !type.sign && type.norm) {
/*
* Fallback to calling util_format_description::fetch_rgba_8unorm.
*
* This is definitely not the most efficient way of fetching pixels, as
* we miss the opportunity to do vectorization, but this it is a
* convenient for formats or scenarios for which there was no opportunity
* or incentive to optimize.
*/
LLVMModuleRef module = LLVMGetGlobalParent(LLVMGetBasicBlockParent(LLVMGetInsertBlock(gallivm->builder)));
char name[256];
LLVMTypeRef i8t = LLVMInt8TypeInContext(gallivm->context);
LLVMTypeRef pi8t = LLVMPointerType(i8t, 0);
LLVMTypeRef i32t = LLVMInt32TypeInContext(gallivm->context);
LLVMValueRef function;
LLVMValueRef tmp_ptr;
LLVMValueRef tmp;
LLVMValueRef res;
LLVMValueRef callee;
unsigned k;
util_snprintf(name, sizeof name, "util_format_%s_fetch_rgba_8unorm",
format_desc->short_name);
if (gallivm_debug & GALLIVM_DEBUG_PERF) {
debug_printf("%s: falling back to %s\n", __FUNCTION__, name);
}
/*
* Declare and bind format_desc->fetch_rgba_8unorm().
*/
function = LLVMGetNamedFunction(module, name);
if (!function) {
/*
* Function to call looks like:
* fetch(uint8_t *dst, const uint8_t *src, unsigned i, unsigned j)
*/
LLVMTypeRef ret_type;
LLVMTypeRef arg_types[4];
LLVMTypeRef function_type;
ret_type = LLVMVoidTypeInContext(gallivm->context);
arg_types[0] = pi8t;
arg_types[1] = pi8t;
arg_types[2] = i32t;
arg_types[3] = i32t;
function_type = LLVMFunctionType(ret_type, arg_types,
Elements(arg_types), 0);
function = LLVMAddFunction(module, name, function_type);
LLVMSetFunctionCallConv(function, LLVMCCallConv);
LLVMSetLinkage(function, LLVMExternalLinkage);
assert(LLVMIsDeclaration(function));
}
/* make const pointer for the C fetch_rgba_float function */
callee = lp_build_const_int_pointer(gallivm,
func_to_pointer((func_pointer) format_desc->fetch_rgba_8unorm));
/* cast the callee pointer to the function's type */
function = LLVMBuildBitCast(builder, callee,
LLVMTypeOf(function), "cast callee");
tmp_ptr = lp_build_alloca(gallivm, i32t, "");
res = LLVMGetUndef(LLVMVectorType(i32t, num_pixels));
/*
* Invoke format_desc->fetch_rgba_8unorm() for each pixel and insert the result
* in the SoA vectors.
*/
for (k = 0; k < num_pixels; ++k) {
LLVMValueRef index = lp_build_const_int32(gallivm, k);
LLVMValueRef args[4];
args[0] = LLVMBuildBitCast(builder, tmp_ptr, pi8t, "");
args[1] = lp_build_gather_elem_ptr(gallivm, num_pixels,
base_ptr, offset, k);
if (num_pixels == 1) {
args[2] = i;
args[3] = j;
}
else {
args[2] = LLVMBuildExtractElement(builder, i, index, "");
args[3] = LLVMBuildExtractElement(builder, j, index, "");
}
LLVMBuildCall(builder, function, args, Elements(args), "");
tmp = LLVMBuildLoad(builder, tmp_ptr, "");
if (num_pixels == 1) {
res = tmp;
}
else {
res = LLVMBuildInsertElement(builder, res, tmp, index, "");
}
}
/* Bitcast from <n x i32> to <4n x i8> */
res = LLVMBuildBitCast(builder, res, bld.vec_type, "");
return res;
}
/*
* Fallback to util_format_description::fetch_rgba_float().
*/
if (format_desc->fetch_rgba_float) {
/*
* Fallback to calling util_format_description::fetch_rgba_float.
*
* This is definitely not the most efficient way of fetching pixels, as
* we miss the opportunity to do vectorization, but this it is a
* convenient for formats or scenarios for which there was no opportunity
* or incentive to optimize.
*/
LLVMModuleRef module = LLVMGetGlobalParent(LLVMGetBasicBlockParent(LLVMGetInsertBlock(builder)));
char name[256];
LLVMTypeRef f32t = LLVMFloatTypeInContext(gallivm->context);
LLVMTypeRef f32x4t = LLVMVectorType(f32t, 4);
LLVMTypeRef pf32t = LLVMPointerType(f32t, 0);
LLVMTypeRef pi8t = LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0);
LLVMTypeRef i32t = LLVMInt32TypeInContext(gallivm->context);
LLVMValueRef function;
LLVMValueRef tmp_ptr;
LLVMValueRef tmps[LP_MAX_VECTOR_LENGTH/4];
LLVMValueRef res;
LLVMValueRef callee;
unsigned k;
util_snprintf(name, sizeof name, "util_format_%s_fetch_rgba_float",
format_desc->short_name);
if (gallivm_debug & GALLIVM_DEBUG_PERF) {
debug_printf("%s: falling back to %s\n", __FUNCTION__, name);
}
/*
* Declare and bind format_desc->fetch_rgba_float().
*/
function = LLVMGetNamedFunction(module, name);
if (!function) {
/*
* Function to call looks like:
* fetch(float *dst, const uint8_t *src, unsigned i, unsigned j)
*/
LLVMTypeRef ret_type;
LLVMTypeRef arg_types[4];
LLVMTypeRef function_type;
ret_type = LLVMVoidTypeInContext(gallivm->context);
arg_types[0] = pf32t;
arg_types[1] = pi8t;
arg_types[2] = i32t;
arg_types[3] = i32t;
function_type = LLVMFunctionType(ret_type, arg_types,
Elements(arg_types), 0);
function = LLVMAddFunction(module, name, function_type);
LLVMSetFunctionCallConv(function, LLVMCCallConv);
LLVMSetLinkage(function, LLVMExternalLinkage);
assert(LLVMIsDeclaration(function));
}
/* Note: we're using this casting here instead of LLVMAddGlobalMapping()
* to work around a bug in LLVM 2.6.
*/
/* make const pointer for the C fetch_rgba_float function */
callee = lp_build_const_int_pointer(gallivm,
func_to_pointer((func_pointer) format_desc->fetch_rgba_float));
/* cast the callee pointer to the function's type */
function = LLVMBuildBitCast(builder, callee,
LLVMTypeOf(function), "cast callee");
tmp_ptr = lp_build_alloca(gallivm, f32x4t, "");
/*
* Invoke format_desc->fetch_rgba_float() for each pixel and insert the result
* in the SoA vectors.
*/
for (k = 0; k < num_pixels; ++k) {
LLVMValueRef args[4];
args[0] = LLVMBuildBitCast(builder, tmp_ptr, pf32t, "");
args[1] = lp_build_gather_elem_ptr(gallivm, num_pixels,
base_ptr, offset, k);
if (num_pixels == 1) {
args[2] = i;
args[3] = j;
}
else {
LLVMValueRef index = lp_build_const_int32(gallivm, k);
args[2] = LLVMBuildExtractElement(builder, i, index, "");
args[3] = LLVMBuildExtractElement(builder, j, index, "");
}
LLVMBuildCall(builder, function, args, Elements(args), "");
tmps[k] = LLVMBuildLoad(builder, tmp_ptr, "");
}
lp_build_conv(gallivm,
lp_float32_vec4_type(),
type,
tmps, num_pixels, &res, 1);
return res;
}
assert(0);
return lp_build_undef(gallivm, type);
}
/*---------------------------- Compiler related functions.------------------------------*/
LLVMTypeRef getHeapPointerType() {
return LLVMPointerType(HeapType, 0);
}
/**
* Return true if the type already exists or if we are only being asked to
* generate a preliminary type, false otherwise.
*/
static bool setup_name(compile_t* c, ast_t* ast, gentype_t* g, bool prelim)
{
if(ast_id(ast) == TK_NOMINAL)
{
ast_t* def = (ast_t*)ast_data(ast);
g->underlying = ast_id(def);
// Find the primitive type, if there is one.
AST_GET_CHILDREN(ast, pkg, id);
const char* package = ast_name(pkg);
const char* name = ast_name(id);
if(package == c->str_builtin)
{
if(name == c->str_Bool)
g->primitive = c->i1;
else if(name == c->str_I8)
g->primitive = c->i8;
else if(name == c->str_U8)
g->primitive = c->i8;
else if(name == c->str_I16)
g->primitive = c->i16;
else if(name == c->str_U16)
g->primitive = c->i16;
else if(name == c->str_I32)
g->primitive = c->i32;
else if(name == c->str_U32)
g->primitive = c->i32;
else if(name == c->str_I64)
g->primitive = c->i64;
else if(name == c->str_U64)
g->primitive = c->i64;
else if(name == c->str_I128)
g->primitive = c->i128;
else if(name == c->str_U128)
g->primitive = c->i128;
#if defined(PLATFORM_IS_ILP32)
else if(name == c->str_ILong)
g->primitive = c->i32;
else if(name == c->str_ULong)
g->primitive = c->i32;
else if(name == c->str_ISize)
g->primitive = c->i32;
else if(name == c->str_USize)
g->primitive = c->i32;
#elif defined(PLATFORM_IS_LP64)
else if(name == c->str_ILong)
g->primitive = c->i64;
else if(name == c->str_ULong)
g->primitive = c->i64;
else if(name == c->str_ISize)
g->primitive = c->i64;
else if(name == c->str_USize)
g->primitive = c->i64;
#elif defined(PLATFORM_IS_LLP64)
else if(name == c->str_ILong)
g->primitive = c->i32;
else if(name == c->str_ULong)
g->primitive = c->i32;
else if(name == c->str_ISize)
g->primitive = c->i64;
else if(name == c->str_USize)
g->primitive = c->i64;
#endif
else if(name == c->str_F32)
g->primitive = c->f32;
else if(name == c->str_F64)
g->primitive = c->f64;
else if(name == c->str_Pointer)
return genprim_pointer(c, g, prelim);
else if(name == c->str_Maybe)
return genprim_maybe(c, g, prelim);
else if(name == c->str_Platform)
return true;
}
} else {
g->underlying = TK_TUPLETYPE;
}
// Find or create the structure type.
g->structure = LLVMGetTypeByName(c->module, g->type_name);
if(g->structure == NULL)
g->structure = LLVMStructCreateNamed(c->context, g->type_name);
bool opaque = LLVMIsOpaqueStruct(g->structure) != 0;
if(g->underlying == TK_TUPLETYPE)
{
// This is actually our primitive type.
g->primitive = g->structure;
g->structure = NULL;
} else {
g->structure_ptr = LLVMPointerType(g->structure, 0);
}
// Fill in our global descriptor.
make_global_descriptor(c, g);
if(g->primitive != NULL)
{
// We're primitive, so use the primitive type.
g->use_type = g->primitive;
} else {
// We're not primitive, so use a pointer to our structure.
g->use_type = g->structure_ptr;
}
if(!opaque)
{
// Fill in our global instance if the type is not opaque.
make_global_instance(c, g);
// Fill in a box type if we need one.
make_box_type(c, g);
return true;
}
return prelim;
}
static bool make_struct(compile_t* c, gentype_t* g)
{
LLVMTypeRef type;
int extra = 0;
if(g->underlying != TK_TUPLETYPE)
{
type = g->structure;
if(g->underlying != TK_STRUCT)
extra++;
} else {
type = g->primitive;
}
if(g->underlying == TK_ACTOR)
extra++;
size_t buf_size = (g->field_count + extra) * sizeof(LLVMTypeRef);
LLVMTypeRef* elements = (LLVMTypeRef*)pool_alloc_size(buf_size);
// Create the type descriptor as element 0.
if(extra > 0)
elements[0] = LLVMPointerType(g->desc_type, 0);
// Create the actor pad as element 1.
if(g->underlying == TK_ACTOR)
elements[1] = c->actor_pad;
// Get a preliminary type for each field and set the struct body. This is
// needed in case a struct for the type being generated here is required when
// generating a field.
for(int i = 0; i < g->field_count; i++)
{
gentype_t field_g;
bool ok;
if((g->field_keys != NULL) && (g->field_keys[i] == TK_EMBED))
{
ok = gentype(c, g->fields[i], &field_g);
elements[i + extra] = field_g.structure;
} else {
ok = gentype_prelim(c, g->fields[i], &field_g);
elements[i + extra] = field_g.use_type;
}
if(!ok)
{
pool_free_size(buf_size, elements);
return false;
}
}
// An embedded field may have caused the current type to be fully generated
// at this point. If so, finish gracefully.
if(!LLVMIsOpaqueStruct(type))
{
g->done = true;
return true;
}
LLVMStructSetBody(type, elements, g->field_count + extra, false);
// Create a box type for tuples.
if(g->underlying == TK_TUPLETYPE)
make_box_type(c, g);
pool_free_size(buf_size, elements);
return true;
}
/**
* Increment the shader input attribute values.
* This is called when we move from one quad to the next.
*/
static void
attribs_update(struct lp_build_interp_soa_context *bld,
struct gallivm_state *gallivm,
LLVMValueRef loop_iter,
int start,
int end)
{
LLVMBuilderRef builder = gallivm->builder;
struct lp_build_context *coeff_bld = &bld->coeff_bld;
LLVMValueRef oow = NULL;
unsigned attrib;
unsigned chan;
for(attrib = start; attrib < end; ++attrib) {
const unsigned mask = bld->mask[attrib];
const unsigned interp = bld->interp[attrib];
for(chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
if(mask & (1 << chan)) {
LLVMValueRef a;
if (interp == LP_INTERP_CONSTANT ||
interp == LP_INTERP_FACING) {
a = LLVMBuildLoad(builder, bld->a[attrib][chan], "");
}
else if (interp == LP_INTERP_POSITION) {
assert(attrib > 0);
a = bld->attribs[0][chan];
}
else {
LLVMValueRef dadq;
a = bld->a[attrib][chan];
/*
* Broadcast the attribute value for this quad into all elements
*/
{
/* stored as vector load as float */
LLVMTypeRef ptr_type = LLVMPointerType(LLVMFloatTypeInContext(
gallivm->context), 0);
LLVMValueRef ptr;
a = LLVMBuildBitCast(builder, a, ptr_type, "");
ptr = LLVMBuildGEP(builder, a, &loop_iter, 1, "");
a = LLVMBuildLoad(builder, ptr, "");
a = lp_build_broadcast_scalar(&bld->coeff_bld, a);
}
/*
* Get the derivatives.
*/
dadq = bld->dadq[attrib][chan];
#if PERSPECTIVE_DIVIDE_PER_QUAD
if (interp == LP_INTERP_PERSPECTIVE) {
LLVMValueRef dwdq = bld->dadq[0][3];
if (oow == NULL) {
assert(bld->oow);
oow = LLVMBuildShuffleVector(coeff_bld->builder,
bld->oow, coeff_bld->undef,
shuffle, "");
}
dadq = lp_build_sub(coeff_bld,
dadq,
lp_build_mul(coeff_bld, a, dwdq));
dadq = lp_build_mul(coeff_bld, dadq, oow);
}
#endif
/*
* Add the derivatives
*/
a = lp_build_add(coeff_bld, a, dadq);
#if !PERSPECTIVE_DIVIDE_PER_QUAD
if (interp == LP_INTERP_PERSPECTIVE) {
if (oow == NULL) {
LLVMValueRef w = bld->attribs[0][3];
assert(attrib != 0);
assert(bld->mask[0] & TGSI_WRITEMASK_W);
oow = lp_build_rcp(coeff_bld, w);
}
a = lp_build_mul(coeff_bld, a, oow);
}
#endif
if (attrib == 0 && chan == 2) {
/* FIXME: Depth values can exceed 1.0, due to the fact that
* setup interpolation coefficients refer to (0,0) which causes
* precision loss. So we must clamp to 1.0 here to avoid artifacts
*/
a = lp_build_min(coeff_bld, a, coeff_bld->one);
}
attrib_name(a, attrib, chan, "");
}
bld->attribs[attrib][chan] = a;
}
}
}
}
/**
* Initialize the bld->a, dadq fields. This involves fetching
* those values from the arrays which are passed into the JIT function.
*/
static void
coeffs_init(struct lp_build_interp_soa_context *bld,
LLVMValueRef a0_ptr,
LLVMValueRef dadx_ptr,
LLVMValueRef dady_ptr)
{
struct lp_build_context *coeff_bld = &bld->coeff_bld;
struct lp_build_context *setup_bld = &bld->setup_bld;
struct gallivm_state *gallivm = coeff_bld->gallivm;
LLVMBuilderRef builder = gallivm->builder;
LLVMValueRef pixoffx, pixoffy;
unsigned attrib;
unsigned chan;
unsigned i;
pixoffx = coeff_bld->undef;
pixoffy = coeff_bld->undef;
for (i = 0; i < coeff_bld->type.length; i++) {
LLVMValueRef nr = lp_build_const_int32(gallivm, i);
LLVMValueRef pixxf = lp_build_const_float(gallivm, quad_offset_x[i]);
LLVMValueRef pixyf = lp_build_const_float(gallivm, quad_offset_y[i]);
pixoffx = LLVMBuildInsertElement(builder, pixoffx, pixxf, nr, "");
pixoffy = LLVMBuildInsertElement(builder, pixoffy, pixyf, nr, "");
}
for (attrib = 0; attrib < bld->num_attribs; ++attrib) {
const unsigned mask = bld->mask[attrib];
const unsigned interp = bld->interp[attrib];
LLVMValueRef index = lp_build_const_int32(gallivm,
attrib * TGSI_NUM_CHANNELS);
LLVMValueRef ptr;
LLVMValueRef dadxaos = setup_bld->zero;
LLVMValueRef dadyaos = setup_bld->zero;
LLVMValueRef a0aos = setup_bld->zero;
/* always fetch all 4 values for performance/simplicity */
switch (interp) {
case LP_INTERP_PERSPECTIVE:
/* fall-through */
case LP_INTERP_LINEAR:
ptr = LLVMBuildGEP(builder, dadx_ptr, &index, 1, "");
ptr = LLVMBuildBitCast(builder, ptr,
LLVMPointerType(setup_bld->vec_type, 0), "");
dadxaos = LLVMBuildLoad(builder, ptr, "");
ptr = LLVMBuildGEP(builder, dady_ptr, &index, 1, "");
ptr = LLVMBuildBitCast(builder, ptr,
LLVMPointerType(setup_bld->vec_type, 0), "");
dadyaos = LLVMBuildLoad(builder, ptr, "");
attrib_name(dadxaos, attrib, 0, ".dadxaos");
attrib_name(dadyaos, attrib, 0, ".dadyaos");
/* fall-through */
case LP_INTERP_CONSTANT:
case LP_INTERP_FACING:
ptr = LLVMBuildGEP(builder, a0_ptr, &index, 1, "");
ptr = LLVMBuildBitCast(builder, ptr,
LLVMPointerType(setup_bld->vec_type, 0), "");
a0aos = LLVMBuildLoad(builder, ptr, "");
attrib_name(a0aos, attrib, 0, ".a0aos");
break;
case LP_INTERP_POSITION:
/* Nothing to do as the position coeffs are already setup in slot 0 */
continue;
default:
assert(0);
break;
}
/*
* a = a0 + (x * dadx + y * dady)
* a0aos is the attrib value at top left corner of stamp
*/
if (interp != LP_INTERP_CONSTANT &&
interp != LP_INTERP_FACING) {
LLVMValueRef x = lp_build_broadcast_scalar(setup_bld, bld->x);
LLVMValueRef y = lp_build_broadcast_scalar(setup_bld, bld->y);
a0aos = lp_build_fmuladd(builder, x, dadxaos, a0aos);
a0aos = lp_build_fmuladd(builder, y, dadyaos, a0aos);
}
/*
* dadq = {0, dadx, dady, dadx + dady}
* for two quads (side by side) this is:
* {0, dadx, dady, dadx+dady, 2*dadx, 2*dadx+dady, 3*dadx+dady}
*/
for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
/* this generates a CRAPLOAD of shuffles... */
if (mask & (1 << chan)) {
LLVMValueRef dadx, dady;
LLVMValueRef dadq, dadq2;
LLVMValueRef a;
LLVMValueRef chan_index = lp_build_const_int32(gallivm, chan);
if (attrib == 0 && chan == 0) {
a = bld->x;
if (bld->pos_offset) {
a = LLVMBuildFAdd(builder, a, lp_build_const_float(gallivm, bld->pos_offset), "");
}
a = lp_build_broadcast_scalar(coeff_bld, a);
dadx = coeff_bld->one;
dady = coeff_bld->zero;
}
else if (attrib == 0 && chan == 1) {
a = bld->y;
if (bld->pos_offset) {
a = LLVMBuildFAdd(builder, a, lp_build_const_float(gallivm, bld->pos_offset), "");
}
a = lp_build_broadcast_scalar(coeff_bld, a);
dady = coeff_bld->one;
dadx = coeff_bld->zero;
}
else {
dadx = lp_build_extract_broadcast(gallivm, setup_bld->type,
coeff_bld->type, dadxaos, chan_index);
dady = lp_build_extract_broadcast(gallivm, setup_bld->type,
coeff_bld->type, dadyaos, chan_index);
/*
* a = {a, a, a, a}
*/
a = lp_build_extract_broadcast(gallivm, setup_bld->type,
coeff_bld->type, a0aos, chan_index);
}
dadx = LLVMBuildFMul(builder, dadx, pixoffx, "");
dady = LLVMBuildFMul(builder, dady, pixoffy, "");
dadq = LLVMBuildFAdd(builder, dadx, dady, "");
/*
* Compute the attrib values on the upper-left corner of each
* group of quads.
* Note that if we process 2 quads at once this doesn't
* really exactly to what we want.
* We need to access elem 0 and 2 respectively later if we process
* 2 quads at once.
*/
if (interp != LP_INTERP_CONSTANT &&
interp != LP_INTERP_FACING) {
dadq2 = LLVMBuildFAdd(builder, dadq, dadq, "");
a = LLVMBuildFAdd(builder, a, dadq2, "");
}
#if PERSPECTIVE_DIVIDE_PER_QUAD
/*
* a *= 1 / w
*/
/*
* XXX since we're only going to access elements 0,2 out of 8
* if we have 8-wide vectors we should do the division only 4-wide.
* a is really a 2-elements in a 4-wide vector disguised as 8-wide
* in this case.
*/
if (interp == LP_INTERP_PERSPECTIVE) {
LLVMValueRef w = bld->a[0][3];
assert(attrib != 0);
assert(bld->mask[0] & TGSI_WRITEMASK_W);
if (!bld->oow) {
bld->oow = lp_build_rcp(coeff_bld, w);
lp_build_name(bld->oow, "oow");
}
a = lp_build_mul(coeff_bld, a, bld->oow);
}
#endif
attrib_name(a, attrib, chan, ".a");
attrib_name(dadq, attrib, chan, ".dadq");
bld->a[attrib][chan] = lp_build_alloca(gallivm,
LLVMTypeOf(a), "");
LLVMBuildStore(builder, a, bld->a[attrib][chan]);
bld->dadq[attrib][chan] = dadq;
}
}
}
}
/* Much easier, and significantly less instructions in the per-stamp
* part (less than half) but overall more instructions so a loss if
* most quads are active. Might be a win though with larger vectors.
* No ability to do per-quad divide (doable but not implemented)
* Could be made to work with passed in pixel offsets (i.e. active quad merging).
*/
static void
coeffs_init_simple(struct lp_build_interp_soa_context *bld,
LLVMValueRef a0_ptr,
LLVMValueRef dadx_ptr,
LLVMValueRef dady_ptr)
{
struct lp_build_context *coeff_bld = &bld->coeff_bld;
struct lp_build_context *setup_bld = &bld->setup_bld;
struct gallivm_state *gallivm = coeff_bld->gallivm;
LLVMBuilderRef builder = gallivm->builder;
unsigned attrib;
for (attrib = 0; attrib < bld->num_attribs; ++attrib) {
/*
* always fetch all 4 values for performance/simplicity
* Note: we do that here because it seems to generate better
* code. It generates a lot of moves initially but less
* moves later. As far as I can tell this looks like a
* llvm issue, instead of simply reloading the values from
* the passed in pointers it if it runs out of registers
* it spills/reloads them. Maybe some optimization passes
* would help.
* Might want to investigate this again later.
*/
const unsigned interp = bld->interp[attrib];
LLVMValueRef index = lp_build_const_int32(gallivm,
attrib * TGSI_NUM_CHANNELS);
LLVMValueRef ptr;
LLVMValueRef dadxaos = setup_bld->zero;
LLVMValueRef dadyaos = setup_bld->zero;
LLVMValueRef a0aos = setup_bld->zero;
switch (interp) {
case LP_INTERP_PERSPECTIVE:
/* fall-through */
case LP_INTERP_LINEAR:
ptr = LLVMBuildGEP(builder, dadx_ptr, &index, 1, "");
ptr = LLVMBuildBitCast(builder, ptr,
LLVMPointerType(setup_bld->vec_type, 0), "");
dadxaos = LLVMBuildLoad(builder, ptr, "");
ptr = LLVMBuildGEP(builder, dady_ptr, &index, 1, "");
ptr = LLVMBuildBitCast(builder, ptr,
LLVMPointerType(setup_bld->vec_type, 0), "");
dadyaos = LLVMBuildLoad(builder, ptr, "");
attrib_name(dadxaos, attrib, 0, ".dadxaos");
attrib_name(dadyaos, attrib, 0, ".dadyaos");
/* fall-through */
case LP_INTERP_CONSTANT:
case LP_INTERP_FACING:
ptr = LLVMBuildGEP(builder, a0_ptr, &index, 1, "");
ptr = LLVMBuildBitCast(builder, ptr,
LLVMPointerType(setup_bld->vec_type, 0), "");
a0aos = LLVMBuildLoad(builder, ptr, "");
attrib_name(a0aos, attrib, 0, ".a0aos");
break;
case LP_INTERP_POSITION:
/* Nothing to do as the position coeffs are already setup in slot 0 */
continue;
default:
assert(0);
break;
}
bld->a0aos[attrib] = a0aos;
bld->dadxaos[attrib] = dadxaos;
bld->dadyaos[attrib] = dadyaos;
}
}
LLVMTypeRef ett_llvm_type(EagleComplexType *type)
{
switch(type->type)
{
case ETVoid:
return LLVMVoidTypeInContext(utl_get_current_context());
case ETFloat:
return LLVMFloatTypeInContext(utl_get_current_context());
case ETDouble:
return LLVMDoubleTypeInContext(utl_get_current_context());
case ETInt1:
return LLVMInt1TypeInContext(utl_get_current_context());
case ETGeneric: // In practice this doesn't matter
case ETAny:
case ETInt8:
case ETUInt8:
return LLVMInt8TypeInContext(utl_get_current_context());
case ETInt16:
case ETUInt16:
return LLVMInt16TypeInContext(utl_get_current_context());
case ETInt32:
case ETUInt32:
return LLVMInt32TypeInContext(utl_get_current_context());
case ETInt64:
case ETUInt64:
return LLVMInt64TypeInContext(utl_get_current_context());
case ETCString:
return LLVMPointerType(LLVMInt8TypeInContext(utl_get_current_context()), 0);
case ETEnum:
return LLVMInt64TypeInContext(utl_get_current_context());
case ETGenerator:
{
if(generator_type)
return generator_type;
LLVMTypeRef ptmp[2];
ptmp[0] = LLVMPointerType(LLVMInt8TypeInContext(utl_get_current_context()), 0);
ptmp[1] = LLVMPointerType(LLVMInt8TypeInContext(utl_get_current_context()), 0);
generator_type = LLVMStructCreateNamed(utl_get_current_context(), "__egl_gen_strct");
LLVMStructSetBody(generator_type, ptmp, 2, 0);
return generator_type;
}
case ETClass:
case ETStruct:
{
EagleStructType *st = (EagleStructType *)type;
LLVMTypeRef loaded = LLVMGetTypeByName(the_module, st->name);
if(loaded)
return loaded;
return NULL;
// LLVMTypeRef ty = LLVMStructTypeInContext(utl_get_current_context(),
}
case ETInterface:
{
return LLVMInt8TypeInContext(utl_get_current_context());
}
case ETPointer:
{
EaglePointerType *pt = (EaglePointerType *)type;
if(pt->counted || pt->weak)
{
LLVMTypeRef ptmp[2];
ptmp[0] = LLVMPointerType(LLVMInt8TypeInContext(utl_get_current_context()), 0);
ptmp[1] = LLVMInt1TypeInContext(utl_get_current_context());
LLVMTypeRef tys[6];
tys[0] = LLVMInt64TypeInContext(utl_get_current_context());
tys[1] = LLVMInt16TypeInContext(utl_get_current_context());
tys[2] = LLVMInt16TypeInContext(utl_get_current_context());
tys[3] = LLVMPointerType(LLVMInt8TypeInContext(utl_get_current_context()), 0);
tys[4] = LLVMPointerType(LLVMFunctionType(LLVMVoidTypeInContext(utl_get_current_context()), ptmp, 2, 0), 0);
tys[5] = ett_llvm_type(pt->to);
return LLVMPointerType(ty_get_counted(LLVMStructTypeInContext(utl_get_current_context(), tys, 6, 0)), 0);
}
return LLVMPointerType(ett_llvm_type(((EaglePointerType *)type)->to), 0);
}
case ETArray:
{
EagleArrayType *at = (EagleArrayType *)type;
if(at->ct < 0)
return LLVMPointerType(ett_llvm_type(at->of), 0);
else
return LLVMArrayType(ett_llvm_type(at->of), at->ct);
}
case ETFunction:
{
EagleFunctionType *ft = (EagleFunctionType *)type;
if(ET_IS_CLOSURE(type))
{
LLVMTypeRef tys[2];
tys[0] = LLVMPointerType(LLVMInt8TypeInContext(utl_get_current_context()), 0);
tys[1] = LLVMPointerType(LLVMInt8TypeInContext(utl_get_current_context()), 0);
return LLVMStructTypeInContext(utl_get_current_context(), tys, 2, 0);
}
LLVMTypeRef *tys = malloc(sizeof(LLVMTypeRef) * ft->pct);
int i;
for(i = 0; i < ft->pct; i++)
tys[i] = ett_llvm_type(ft->params[i]);
LLVMTypeRef out = LLVMFunctionType(ett_llvm_type(ft->retType), tys, ft->pct, 0);
free(tys);
return out;
}
default:
return NULL;
}
}
static void
lp_jit_create_types(struct lp_fragment_shader_variant *lp)
{
struct gallivm_state *gallivm = lp->gallivm;
LLVMContextRef lc = gallivm->context;
LLVMTypeRef viewport_type, texture_type, sampler_type;
/* struct lp_jit_viewport */
{
LLVMTypeRef elem_types[LP_JIT_VIEWPORT_NUM_FIELDS];
elem_types[LP_JIT_VIEWPORT_MIN_DEPTH] =
elem_types[LP_JIT_VIEWPORT_MAX_DEPTH] = LLVMFloatTypeInContext(lc);
viewport_type = LLVMStructTypeInContext(lc, elem_types,
Elements(elem_types), 0);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_viewport, min_depth,
gallivm->target, viewport_type,
LP_JIT_VIEWPORT_MIN_DEPTH);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_viewport, max_depth,
gallivm->target, viewport_type,
LP_JIT_VIEWPORT_MAX_DEPTH);
LP_CHECK_STRUCT_SIZE(struct lp_jit_viewport,
gallivm->target, viewport_type);
}
/* struct lp_jit_texture */
{
LLVMTypeRef elem_types[LP_JIT_TEXTURE_NUM_FIELDS];
elem_types[LP_JIT_TEXTURE_WIDTH] =
elem_types[LP_JIT_TEXTURE_HEIGHT] =
elem_types[LP_JIT_TEXTURE_DEPTH] =
elem_types[LP_JIT_TEXTURE_FIRST_LEVEL] =
elem_types[LP_JIT_TEXTURE_LAST_LEVEL] = LLVMInt32TypeInContext(lc);
elem_types[LP_JIT_TEXTURE_BASE] = LLVMPointerType(LLVMInt8TypeInContext(lc), 0);
elem_types[LP_JIT_TEXTURE_ROW_STRIDE] =
elem_types[LP_JIT_TEXTURE_IMG_STRIDE] =
elem_types[LP_JIT_TEXTURE_MIP_OFFSETS] =
LLVMArrayType(LLVMInt32TypeInContext(lc), LP_MAX_TEXTURE_LEVELS);
texture_type = LLVMStructTypeInContext(lc, elem_types,
Elements(elem_types), 0);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, width,
gallivm->target, texture_type,
LP_JIT_TEXTURE_WIDTH);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, height,
gallivm->target, texture_type,
LP_JIT_TEXTURE_HEIGHT);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, depth,
gallivm->target, texture_type,
LP_JIT_TEXTURE_DEPTH);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, first_level,
gallivm->target, texture_type,
LP_JIT_TEXTURE_FIRST_LEVEL);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, last_level,
gallivm->target, texture_type,
LP_JIT_TEXTURE_LAST_LEVEL);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, base,
gallivm->target, texture_type,
LP_JIT_TEXTURE_BASE);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, row_stride,
gallivm->target, texture_type,
LP_JIT_TEXTURE_ROW_STRIDE);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, img_stride,
gallivm->target, texture_type,
LP_JIT_TEXTURE_IMG_STRIDE);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, mip_offsets,
gallivm->target, texture_type,
LP_JIT_TEXTURE_MIP_OFFSETS);
LP_CHECK_STRUCT_SIZE(struct lp_jit_texture,
gallivm->target, texture_type);
}
/* struct lp_jit_sampler */
{
LLVMTypeRef elem_types[LP_JIT_SAMPLER_NUM_FIELDS];
elem_types[LP_JIT_SAMPLER_MIN_LOD] =
elem_types[LP_JIT_SAMPLER_MAX_LOD] =
elem_types[LP_JIT_SAMPLER_LOD_BIAS] = LLVMFloatTypeInContext(lc);
elem_types[LP_JIT_SAMPLER_BORDER_COLOR] =
LLVMArrayType(LLVMFloatTypeInContext(lc), 4);
sampler_type = LLVMStructTypeInContext(lc, elem_types,
Elements(elem_types), 0);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_sampler, min_lod,
gallivm->target, sampler_type,
LP_JIT_SAMPLER_MIN_LOD);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_sampler, max_lod,
gallivm->target, sampler_type,
LP_JIT_SAMPLER_MAX_LOD);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_sampler, lod_bias,
gallivm->target, sampler_type,
LP_JIT_SAMPLER_LOD_BIAS);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_sampler, border_color,
gallivm->target, sampler_type,
LP_JIT_SAMPLER_BORDER_COLOR);
LP_CHECK_STRUCT_SIZE(struct lp_jit_sampler,
gallivm->target, sampler_type);
}
/* struct lp_jit_context */
{
LLVMTypeRef elem_types[LP_JIT_CTX_COUNT];
LLVMTypeRef context_type;
elem_types[LP_JIT_CTX_CONSTANTS] =
LLVMArrayType(LLVMPointerType(LLVMFloatTypeInContext(lc), 0), LP_MAX_TGSI_CONST_BUFFERS);
elem_types[LP_JIT_CTX_NUM_CONSTANTS] =
LLVMArrayType(LLVMInt32TypeInContext(lc), LP_MAX_TGSI_CONST_BUFFERS);
elem_types[LP_JIT_CTX_ALPHA_REF] = LLVMFloatTypeInContext(lc);
elem_types[LP_JIT_CTX_STENCIL_REF_FRONT] =
elem_types[LP_JIT_CTX_STENCIL_REF_BACK] = LLVMInt32TypeInContext(lc);
elem_types[LP_JIT_CTX_U8_BLEND_COLOR] = LLVMPointerType(LLVMInt8TypeInContext(lc), 0);
elem_types[LP_JIT_CTX_F_BLEND_COLOR] = LLVMPointerType(LLVMFloatTypeInContext(lc), 0);
elem_types[LP_JIT_CTX_VIEWPORTS] = LLVMPointerType(viewport_type, 0);
elem_types[LP_JIT_CTX_TEXTURES] = LLVMArrayType(texture_type,
PIPE_MAX_SHADER_SAMPLER_VIEWS);
elem_types[LP_JIT_CTX_SAMPLERS] = LLVMArrayType(sampler_type,
PIPE_MAX_SAMPLERS);
context_type = LLVMStructTypeInContext(lc, elem_types,
Elements(elem_types), 0);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_context, constants,
gallivm->target, context_type,
LP_JIT_CTX_CONSTANTS);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_context, num_constants,
gallivm->target, context_type,
LP_JIT_CTX_NUM_CONSTANTS);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_context, alpha_ref_value,
gallivm->target, context_type,
LP_JIT_CTX_ALPHA_REF);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_context, stencil_ref_front,
gallivm->target, context_type,
LP_JIT_CTX_STENCIL_REF_FRONT);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_context, stencil_ref_back,
gallivm->target, context_type,
LP_JIT_CTX_STENCIL_REF_BACK);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_context, u8_blend_color,
gallivm->target, context_type,
LP_JIT_CTX_U8_BLEND_COLOR);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_context, f_blend_color,
gallivm->target, context_type,
LP_JIT_CTX_F_BLEND_COLOR);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_context, viewports,
gallivm->target, context_type,
LP_JIT_CTX_VIEWPORTS);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_context, textures,
gallivm->target, context_type,
LP_JIT_CTX_TEXTURES);
LP_CHECK_MEMBER_OFFSET(struct lp_jit_context, samplers,
gallivm->target, context_type,
LP_JIT_CTX_SAMPLERS);
LP_CHECK_STRUCT_SIZE(struct lp_jit_context,
gallivm->target, context_type);
lp->jit_context_ptr_type = LLVMPointerType(context_type, 0);
}
/* struct lp_jit_thread_data */
{
LLVMTypeRef elem_types[LP_JIT_THREAD_DATA_COUNT];
LLVMTypeRef thread_data_type;
elem_types[LP_JIT_THREAD_DATA_COUNTER] = LLVMInt64TypeInContext(lc);
elem_types[LP_JIT_THREAD_DATA_RASTER_STATE_VIEWPORT_INDEX] =
LLVMInt32TypeInContext(lc);
thread_data_type = LLVMStructTypeInContext(lc, elem_types,
Elements(elem_types), 0);
lp->jit_thread_data_ptr_type = LLVMPointerType(thread_data_type, 0);
}
if (gallivm_debug & GALLIVM_DEBUG_IR) {
LLVMDumpModule(gallivm->module);
}
}
/**
* @brief lp_build_fetch_rgba_aos_array
*
* \param format_desc describes format of the image we're fetching from
* \param dst_type output type
* \param base_ptr address of the pixel block (or the texel if uncompressed)
* \param offset ptr offset
*/
LLVMValueRef
lp_build_fetch_rgba_aos_array(struct gallivm_state *gallivm,
const struct util_format_description *format_desc,
struct lp_type dst_type,
LLVMValueRef base_ptr,
LLVMValueRef offset)
{
struct lp_build_context bld;
LLVMBuilderRef builder = gallivm->builder;
LLVMTypeRef src_vec_type;
LLVMValueRef ptr, res = NULL;
struct lp_type src_type;
boolean pure_integer = format_desc->channel[0].pure_integer;
struct lp_type tmp_type;
lp_type_from_format_desc(&src_type, format_desc);
assert(src_type.length <= dst_type.length);
src_vec_type = lp_build_vec_type(gallivm, src_type);
/* Read whole vector from memory, unaligned */
ptr = LLVMBuildGEP(builder, base_ptr, &offset, 1, "");
ptr = LLVMBuildPointerCast(builder, ptr, LLVMPointerType(src_vec_type, 0), "");
res = LLVMBuildLoad(builder, ptr, "");
LLVMSetAlignment(res, src_type.width / 8);
/* Truncate doubles to float */
if (src_type.floating && src_type.width == 64) {
src_type.width = 32;
src_vec_type = lp_build_vec_type(gallivm, src_type);
res = LLVMBuildFPTrunc(builder, res, src_vec_type, "");
}
/* Expand to correct length */
if (src_type.length < dst_type.length) {
res = lp_build_pad_vector(gallivm, res, dst_type.length);
src_type.length = dst_type.length;
}
tmp_type = dst_type;
if (pure_integer) {
/* some callers expect (fake) floats other real ints. */
tmp_type.floating = 0;
tmp_type.sign = src_type.sign;
}
/* Convert to correct format */
lp_build_conv(gallivm, src_type, tmp_type, &res, 1, &res, 1);
/* Swizzle it */
lp_build_context_init(&bld, gallivm, tmp_type);
res = lp_build_format_swizzle_aos(format_desc, &bld, res);
/* Bitcast to floats (for pure integers) when requested */
if (pure_integer && dst_type.floating) {
res = LLVMBuildBitCast(builder, res, lp_build_vec_type(gallivm, dst_type), "");
}
return res;
}