static void
store_cached_block(struct gallivm_state *gallivm,
LLVMValueRef *col,
LLVMValueRef tag_value,
LLVMValueRef hash_index,
LLVMValueRef cache)
{
LLVMBuilderRef builder = gallivm->builder;
LLVMValueRef ptr, indices[3];
LLVMTypeRef type_ptr4x32;
unsigned count;
type_ptr4x32 = LLVMPointerType(LLVMVectorType(LLVMInt32TypeInContext(gallivm->context), 4), 0);
indices[0] = lp_build_const_int32(gallivm, 0);
indices[1] = lp_build_const_int32(gallivm, LP_BUILD_FORMAT_CACHE_MEMBER_TAGS);
indices[2] = hash_index;
ptr = LLVMBuildGEP(builder, cache, indices, ARRAY_SIZE(indices), "");
LLVMBuildStore(builder, tag_value, ptr);
indices[1] = lp_build_const_int32(gallivm, LP_BUILD_FORMAT_CACHE_MEMBER_DATA);
hash_index = LLVMBuildMul(builder, hash_index,
lp_build_const_int32(gallivm, 16), "");
for (count = 0; count < 4; count++) {
indices[2] = hash_index;
ptr = LLVMBuildGEP(builder, cache, indices, ARRAY_SIZE(indices), "");
ptr = LLVMBuildBitCast(builder, ptr, type_ptr4x32, "");
LLVMBuildStore(builder, col[count], ptr);
hash_index = LLVMBuildAdd(builder, hash_index,
lp_build_const_int32(gallivm, 4), "");
}
}
/**
* Generate color blending and color output.
* \param rt the render target index (to index blend, colormask state)
* \param type the pixel color type
* \param context_ptr pointer to the runtime JIT context
* \param mask execution mask (active fragment/pixel mask)
* \param src colors from the fragment shader
* \param dst_ptr the destination color buffer pointer
*/
static void
generate_blend(const struct pipe_blend_state *blend,
unsigned rt,
LLVMBuilderRef builder,
struct lp_type type,
LLVMValueRef context_ptr,
LLVMValueRef mask,
LLVMValueRef *src,
LLVMValueRef dst_ptr)
{
struct lp_build_context bld;
struct lp_build_flow_context *flow;
struct lp_build_mask_context mask_ctx;
LLVMTypeRef vec_type;
LLVMValueRef const_ptr;
LLVMValueRef con[4];
LLVMValueRef dst[4];
LLVMValueRef res[4];
unsigned chan;
lp_build_context_init(&bld, builder, type);
flow = lp_build_flow_create(builder);
/* we'll use this mask context to skip blending if all pixels are dead */
lp_build_mask_begin(&mask_ctx, flow, type, mask);
vec_type = lp_build_vec_type(type);
const_ptr = lp_jit_context_blend_color(builder, context_ptr);
const_ptr = LLVMBuildBitCast(builder, const_ptr,
LLVMPointerType(vec_type, 0), "");
/* load constant blend color and colors from the dest color buffer */
for(chan = 0; chan < 4; ++chan) {
LLVMValueRef index = LLVMConstInt(LLVMInt32Type(), chan, 0);
con[chan] = LLVMBuildLoad(builder, LLVMBuildGEP(builder, const_ptr, &index, 1, ""), "");
dst[chan] = LLVMBuildLoad(builder, LLVMBuildGEP(builder, dst_ptr, &index, 1, ""), "");
lp_build_name(con[chan], "con.%c", "rgba"[chan]);
lp_build_name(dst[chan], "dst.%c", "rgba"[chan]);
}
/* do blend */
lp_build_blend_soa(builder, blend, type, rt, src, dst, con, res);
/* store results to color buffer */
for(chan = 0; chan < 4; ++chan) {
if(blend->rt[rt].colormask & (1 << chan)) {
LLVMValueRef index = LLVMConstInt(LLVMInt32Type(), chan, 0);
lp_build_name(res[chan], "res.%c", "rgba"[chan]);
res[chan] = lp_build_select(&bld, mask, res[chan], dst[chan]);
LLVMBuildStore(builder, res[chan], LLVMBuildGEP(builder, dst_ptr, &index, 1, ""));
}
}
lp_build_mask_end(&mask_ctx);
lp_build_flow_destroy(flow);
}
/**
* Generate color blending and color output.
*/
static void
generate_blend(const struct pipe_blend_state *blend,
LLVMBuilderRef builder,
struct lp_type type,
LLVMValueRef context_ptr,
LLVMValueRef mask,
LLVMValueRef *src,
LLVMValueRef dst_ptr)
{
struct lp_build_context bld;
struct lp_build_flow_context *flow;
struct lp_build_mask_context mask_ctx;
LLVMTypeRef vec_type;
LLVMTypeRef int_vec_type;
LLVMValueRef const_ptr;
LLVMValueRef con[4];
LLVMValueRef dst[4];
LLVMValueRef res[4];
unsigned chan;
lp_build_context_init(&bld, builder, type);
flow = lp_build_flow_create(builder);
lp_build_mask_begin(&mask_ctx, flow, type, mask);
vec_type = lp_build_vec_type(type);
int_vec_type = lp_build_int_vec_type(type);
const_ptr = lp_jit_context_blend_color(builder, context_ptr);
const_ptr = LLVMBuildBitCast(builder, const_ptr,
LLVMPointerType(vec_type, 0), "");
for(chan = 0; chan < 4; ++chan) {
LLVMValueRef index = LLVMConstInt(LLVMInt32Type(), chan, 0);
con[chan] = LLVMBuildLoad(builder, LLVMBuildGEP(builder, const_ptr, &index, 1, ""), "");
dst[chan] = LLVMBuildLoad(builder, LLVMBuildGEP(builder, dst_ptr, &index, 1, ""), "");
lp_build_name(con[chan], "con.%c", "rgba"[chan]);
lp_build_name(dst[chan], "dst.%c", "rgba"[chan]);
}
lp_build_blend_soa(builder, blend, type, src, dst, con, res);
for(chan = 0; chan < 4; ++chan) {
if(blend->colormask & (1 << chan)) {
LLVMValueRef index = LLVMConstInt(LLVMInt32Type(), chan, 0);
lp_build_name(res[chan], "res.%c", "rgba"[chan]);
res[chan] = lp_build_select(&bld, mask, res[chan], dst[chan]);
LLVMBuildStore(builder, res[chan], LLVMBuildGEP(builder, dst_ptr, &index, 1, ""));
}
}
lp_build_mask_end(&mask_ctx);
lp_build_flow_destroy(flow);
}
static LLVMValueRef
add_conv_test(struct gallivm_state *gallivm,
struct lp_type src_type, unsigned num_srcs,
struct lp_type dst_type, unsigned num_dsts)
{
LLVMModuleRef module = gallivm->module;
LLVMContextRef context = gallivm->context;
LLVMBuilderRef builder = gallivm->builder;
LLVMTypeRef args[2];
LLVMValueRef func;
LLVMValueRef src_ptr;
LLVMValueRef dst_ptr;
LLVMBasicBlockRef block;
LLVMValueRef src[LP_MAX_VECTOR_LENGTH];
LLVMValueRef dst[LP_MAX_VECTOR_LENGTH];
unsigned i;
args[0] = LLVMPointerType(lp_build_vec_type(gallivm, src_type), 0);
args[1] = LLVMPointerType(lp_build_vec_type(gallivm, dst_type), 0);
func = LLVMAddFunction(module, "test",
LLVMFunctionType(LLVMVoidTypeInContext(context),
args, 2, 0));
LLVMSetFunctionCallConv(func, LLVMCCallConv);
src_ptr = LLVMGetParam(func, 0);
dst_ptr = LLVMGetParam(func, 1);
block = LLVMAppendBasicBlockInContext(context, func, "entry");
LLVMPositionBuilderAtEnd(builder, block);
for(i = 0; i < num_srcs; ++i) {
LLVMValueRef index = LLVMConstInt(LLVMInt32TypeInContext(context), i, 0);
LLVMValueRef ptr = LLVMBuildGEP(builder, src_ptr, &index, 1, "");
src[i] = LLVMBuildLoad(builder, ptr, "");
}
lp_build_conv(gallivm, src_type, dst_type, src, num_srcs, dst, num_dsts);
for(i = 0; i < num_dsts; ++i) {
LLVMValueRef index = LLVMConstInt(LLVMInt32TypeInContext(context), i, 0);
LLVMValueRef ptr = LLVMBuildGEP(builder, dst_ptr, &index, 1, "");
LLVMBuildStore(builder, dst[i], ptr);
}
LLVMBuildRetVoid(builder);;
gallivm_verify_function(gallivm, func);
return func;
}
static LLVMValueRef
emit_fetch_temporary(
struct lp_build_tgsi_context *bld_base,
const struct tgsi_full_src_register *reg,
enum tgsi_opcode_type type,
unsigned swizzle)
{
struct lp_build_tgsi_soa_context *bld = lp_soa_context(bld_base);
LLVMBuilderRef builder = bld_base->base.gallivm->builder;
if (swizzle == ~0) {
LLVMValueRef values[TGSI_NUM_CHANNELS] = {};
unsigned chan;
for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
values[chan] = emit_fetch_temporary(bld_base, reg, type, chan);
}
return lp_build_gather_values(bld_base->base.gallivm, values,
TGSI_NUM_CHANNELS);
}
if (reg->Register.Indirect) {
LLVMValueRef array_index = emit_array_index(bld, reg, swizzle);
LLVMValueRef ptr = LLVMBuildGEP(builder, bld->temps_array, &array_index,
1, "");
return LLVMBuildLoad(builder, ptr, "");
} else {
LLVMValueRef temp_ptr;
temp_ptr = lp_get_temp_ptr_soa(bld, reg->Register.Index, swizzle);
return bitcast(bld_base,type,LLVMBuildLoad(builder, temp_ptr, ""));
}
}
LLVMValueRef build_utcb_load(
struct llvm_ctx *ctx,
LLVMValueRef ix,
const char *name)
{
return LLVMBuildLoad(ctx->builder,
LLVMBuildGEP(ctx->builder, ctx->utcb, &ix, 1, "utcb.addr.in"),
name);
}
void
lp_build_pointer_set(LLVMBuilderRef builder,
LLVMValueRef ptr,
LLVMValueRef index,
LLVMValueRef value)
{
LLVMValueRef element_ptr;
element_ptr = LLVMBuildGEP(builder, ptr, &index, 1, "");
LLVMBuildStore(builder, value, element_ptr);
}
/**
* @brief lp_build_fetch_rgba_aos_array
*
* \param format_desc describes format of the image we're fetching from
* \param dst_type output type
* \param base_ptr address of the pixel block (or the texel if uncompressed)
* \param offset ptr offset
*/
LLVMValueRef
lp_build_fetch_rgba_aos_array(struct gallivm_state *gallivm,
const struct util_format_description *format_desc,
struct lp_type dst_type,
LLVMValueRef base_ptr,
LLVMValueRef offset)
{
struct lp_build_context bld;
LLVMBuilderRef builder = gallivm->builder;
LLVMTypeRef src_elem_type, src_vec_type;
LLVMValueRef ptr, res = NULL;
struct lp_type src_type;
memset(&src_type, 0, sizeof src_type);
src_type.floating = format_desc->channel[0].type == UTIL_FORMAT_TYPE_FLOAT;
src_type.fixed = format_desc->channel[0].type == UTIL_FORMAT_TYPE_FIXED;
src_type.sign = format_desc->channel[0].type != UTIL_FORMAT_TYPE_UNSIGNED;
src_type.norm = format_desc->channel[0].normalized;
src_type.width = format_desc->channel[0].size;
src_type.length = format_desc->nr_channels;
assert(src_type.length <= dst_type.length);
src_elem_type = lp_build_elem_type(gallivm, src_type);
src_vec_type = lp_build_vec_type(gallivm, src_type);
/* Read whole vector from memory, unaligned */
if (!res) {
ptr = LLVMBuildGEP(builder, base_ptr, &offset, 1, "");
ptr = LLVMBuildPointerCast(builder, ptr, LLVMPointerType(src_vec_type, 0), "");
res = LLVMBuildLoad(builder, ptr, "");
lp_set_load_alignment(res, src_type.width / 8);
}
/* Truncate doubles to float */
if (src_type.floating && src_type.width == 64) {
src_type.width = 32;
src_vec_type = lp_build_vec_type(gallivm, src_type);
res = LLVMBuildFPTrunc(builder, res, src_vec_type, "");
}
/* Expand to correct length */
if (src_type.length < dst_type.length) {
res = lp_build_pad_vector(gallivm, res, src_type, dst_type.length);
src_type.length = dst_type.length;
}
/* Convert to correct format */
lp_build_conv(gallivm, src_type, dst_type, &res, 1, &res, 1);
/* Swizzle it */
lp_build_context_init(&bld, gallivm, dst_type);
return lp_build_format_swizzle_aos(format_desc, &bld, res);
}
LLVMValueRef gendesc_fieldinfo(compile_t* c, LLVMValueRef desc, size_t index)
{
LLVMValueRef fields = desc_field(c, desc, DESC_FIELDS);
LLVMValueRef gep[2];
gep[0] = LLVMConstInt(c->i32, 0, false);
gep[1] = LLVMConstInt(c->i32, index, false);
LLVMValueRef field_desc = LLVMBuildGEP(c->builder, fields, gep, 2, "");
return LLVMBuildLoad(c->builder, field_desc, "");
}
LLVMValueRef gendesc_vtable(compile_t* c, LLVMValueRef object, size_t colour)
{
LLVMValueRef desc = gendesc_fetch(c, object);
LLVMValueRef vtable = LLVMBuildStructGEP(c->builder, desc, DESC_VTABLE, "");
LLVMValueRef gep[2];
gep[0] = LLVMConstInt(c->i32, 0, false);
gep[1] = LLVMConstInt(c->i32, colour, false);
LLVMValueRef func_ptr = LLVMBuildGEP(c->builder, vtable, gep, 2, "");
return LLVMBuildLoad(c->builder, func_ptr, "");
}
LLVMValueRef gendesc_istrait(compile_t* c, LLVMValueRef desc, ast_t* type)
{
// Get the trait identifier.
reach_type_t* t = reach_type(c->reach, type);
assert(t != NULL);
LLVMValueRef trait_id = LLVMConstInt(c->i32, t->type_id, false);
// Read the count and the trait list from the descriptor.
LLVMValueRef count = desc_field(c, desc, DESC_TRAIT_COUNT);
LLVMValueRef list = desc_field(c, desc, DESC_TRAITS);
LLVMBasicBlockRef entry_block = LLVMGetInsertBlock(c->builder);
LLVMBasicBlockRef cond_block = codegen_block(c, "cond");
LLVMBasicBlockRef body_block = codegen_block(c, "body");
LLVMBasicBlockRef post_block = codegen_block(c, "post");
LLVMBuildBr(c->builder, cond_block);
// While the index is less than the count, check an ID.
LLVMPositionBuilderAtEnd(c->builder, cond_block);
LLVMValueRef phi = LLVMBuildPhi(c->builder, c->i32, "");
LLVMValueRef zero = LLVMConstInt(c->i32, 0, false);
LLVMAddIncoming(phi, &zero, &entry_block, 1);
LLVMValueRef test = LLVMBuildICmp(c->builder, LLVMIntULT, phi, count, "");
LLVMBuildCondBr(c->builder, test, body_block, post_block);
// The phi node is the index. Get ID and compare it.
LLVMPositionBuilderAtEnd(c->builder, body_block);
LLVMValueRef gep[2];
gep[0] = LLVMConstInt(c->i32, 0, false);
gep[1] = phi;
LLVMValueRef id_ptr = LLVMBuildGEP(c->builder, list, gep, 2, "");
LLVMValueRef id = LLVMBuildLoad(c->builder, id_ptr, "");
LLVMValueRef test_id = LLVMBuildICmp(c->builder, LLVMIntEQ, id, trait_id,
"");
// Add one to the phi node.
LLVMValueRef one = LLVMConstInt(c->i32, 1, false);
LLVMValueRef inc = LLVMBuildAdd(c->builder, phi, one, "");
LLVMAddIncoming(phi, &inc, &body_block, 1);
// Either to the post block or back to the condition.
LLVMBuildCondBr(c->builder, test_id, post_block, cond_block);
LLVMPositionBuilderAtEnd(c->builder, post_block);
LLVMValueRef result = LLVMBuildPhi(c->builder, c->i1, "");
LLVMAddIncoming(result, &test, &cond_block, 1);
LLVMAddIncoming(result, &test_id, &body_block, 1);
return result;
}
LLVMValueRef
ac_build_gep0(struct ac_llvm_context *ctx,
LLVMValueRef base_ptr,
LLVMValueRef index)
{
LLVMValueRef indices[2] = {
LLVMConstInt(ctx->i32, 0, 0),
index,
};
return LLVMBuildGEP(ctx->builder, base_ptr,
indices, 2, "");
}
void
lp_build_pointer_set_unaligned(LLVMBuilderRef builder,
LLVMValueRef ptr,
LLVMValueRef index,
LLVMValueRef value,
unsigned alignment)
{
LLVMValueRef element_ptr;
LLVMValueRef instr;
element_ptr = LLVMBuildGEP(builder, ptr, &index, 1, "");
instr = LLVMBuildStore(builder, value, element_ptr);
lp_set_store_alignment(instr, alignment);
}
/**
* Return pointer to a single mipmap level.
* \param data_array array of pointers to mipmap levels
* \param level integer mipmap level
*/
LLVMValueRef
lp_build_get_mipmap_level(struct lp_build_sample_context *bld,
LLVMValueRef level)
{
LLVMBuilderRef builder = bld->gallivm->builder;
LLVMValueRef indexes[2], data_ptr;
indexes[0] = lp_build_const_int32(bld->gallivm, 0);
indexes[1] = level;
data_ptr = LLVMBuildGEP(builder, bld->data_array, indexes, 2, "");
data_ptr = LLVMBuildLoad(builder, data_ptr, "");
return data_ptr;
}
/**
* Dereference stride_array[mipmap_level] array to get a stride.
* Return stride as a vector.
*/
static LLVMValueRef
lp_build_get_level_stride_vec(struct lp_build_sample_context *bld,
LLVMValueRef stride_array, LLVMValueRef level)
{
LLVMBuilderRef builder = bld->gallivm->builder;
LLVMValueRef indexes[2], stride;
indexes[0] = lp_build_const_int32(bld->gallivm, 0);
indexes[1] = level;
stride = LLVMBuildGEP(builder, stride_array, indexes, 2, "");
stride = LLVMBuildLoad(builder, stride, "");
stride = lp_build_broadcast_scalar(&bld->int_coord_bld, stride);
return stride;
}
static LLVMValueRef
lookup_tag_data(struct gallivm_state *gallivm,
LLVMValueRef ptr,
LLVMValueRef index)
{
LLVMBuilderRef builder = gallivm->builder;
LLVMValueRef member_ptr, indices[3];
indices[0] = lp_build_const_int32(gallivm, 0);
indices[1] = lp_build_const_int32(gallivm, LP_BUILD_FORMAT_CACHE_MEMBER_TAGS);
indices[2] = index;
member_ptr = LLVMBuildGEP(builder, ptr, indices, ARRAY_SIZE(indices), "");
return LLVMBuildLoad(builder, member_ptr, "tag_data");
}
/**
* Build an LLVM bytecode indexed load using LLVMBuildGEP + LLVMBuildLoad
*
* @param offset The offset parameter specifies the number of
* elements to offset, not the number of bytes or dwords. An element is the
* the type pointed to by the base_ptr parameter (e.g. int is the element of
* an int* pointer)
*
* When LLVM lowers the load instruction, it will convert the element offset
* into a dword offset automatically.
*
*/
static LLVMValueRef build_indexed_load(
struct si_shader_context * si_shader_ctx,
LLVMValueRef base_ptr,
LLVMValueRef offset)
{
struct lp_build_context * base = &si_shader_ctx->radeon_bld.soa.bld_base.base;
LLVMValueRef computed_ptr = LLVMBuildGEP(
base->gallivm->builder, base_ptr, &offset, 1, "");
LLVMValueRef result = LLVMBuildLoad(base->gallivm->builder, computed_ptr, "");
LLVMSetMetadata(result, 1, si_shader_ctx->const_md);
return result;
}
LLVMValueRef
lp_build_pointer_get(LLVMBuilderRef builder,
LLVMValueRef ptr,
LLVMValueRef index)
{
LLVMValueRef element_ptr;
LLVMValueRef res;
assert(LLVMGetTypeKind(LLVMTypeOf(ptr)) == LLVMPointerTypeKind);
element_ptr = LLVMBuildGEP(builder, ptr, &index, 1, "");
res = LLVMBuildLoad(builder, element_ptr, "");
#ifdef DEBUG
lp_build_name(res, "%s[%s]", LLVMGetValueName(ptr), LLVMGetValueName(index));
#endif
return res;
}
static void trace_array_elements(compile_t* c, reach_type_t* t,
LLVMValueRef ctx, LLVMValueRef object, LLVMValueRef pointer)
{
// Get the type argument for the array. This will be used to generate the
// per-element trace call.
ast_t* typeargs = ast_childidx(t->ast, 2);
ast_t* typearg = ast_child(typeargs);
if(!gentrace_needed(typearg))
return;
reach_type_t* t_elem = reach_type(c->reach, typearg);
pointer = LLVMBuildBitCast(c->builder, pointer,
LLVMPointerType(t_elem->use_type, 0), "");
LLVMBasicBlockRef entry_block = LLVMGetInsertBlock(c->builder);
LLVMBasicBlockRef cond_block = codegen_block(c, "cond");
LLVMBasicBlockRef body_block = codegen_block(c, "body");
LLVMBasicBlockRef post_block = codegen_block(c, "post");
// Read the size.
LLVMValueRef size = field_value(c, object, 1);
LLVMBuildBr(c->builder, cond_block);
// While the index is less than the size, trace an element. The initial
// index when coming from the entry block is zero.
LLVMPositionBuilderAtEnd(c->builder, cond_block);
LLVMValueRef phi = LLVMBuildPhi(c->builder, c->intptr, "");
LLVMValueRef zero = LLVMConstInt(c->intptr, 0, false);
LLVMAddIncoming(phi, &zero, &entry_block, 1);
LLVMValueRef test = LLVMBuildICmp(c->builder, LLVMIntULT, phi, size, "");
LLVMBuildCondBr(c->builder, test, body_block, post_block);
// The phi node is the index. Get the element and trace it.
LLVMPositionBuilderAtEnd(c->builder, body_block);
LLVMValueRef elem_ptr = LLVMBuildGEP(c->builder, pointer, &phi, 1, "elem");
LLVMValueRef elem = LLVMBuildLoad(c->builder, elem_ptr, "");
gentrace(c, ctx, elem, typearg);
// Add one to the phi node and branch back to the cond block.
LLVMValueRef one = LLVMConstInt(c->intptr, 1, false);
LLVMValueRef inc = LLVMBuildAdd(c->builder, phi, one, "");
body_block = LLVMGetInsertBlock(c->builder);
LLVMAddIncoming(phi, &inc, &body_block, 1);
LLVMBuildBr(c->builder, cond_block);
LLVMPositionBuilderAtEnd(c->builder, post_block);
}
static LLVMValueRef llvm_load_const_buffer(
struct lp_build_tgsi_context * bld_base,
LLVMValueRef OffsetValue,
unsigned ConstantAddressSpace)
{
LLVMValueRef offset[2] = {
LLVMConstInt(LLVMInt64TypeInContext(bld_base->base.gallivm->context), 0, false),
OffsetValue
};
LLVMTypeRef const_ptr_type = LLVMPointerType(LLVMArrayType(LLVMVectorType(bld_base->base.elem_type, 4), 1024),
ConstantAddressSpace);
LLVMValueRef const_ptr = LLVMBuildIntToPtr(bld_base->base.gallivm->builder, lp_build_const_int32(bld_base->base.gallivm, 0), const_ptr_type, "");
LLVMValueRef ptr = LLVMBuildGEP(bld_base->base.gallivm->builder, const_ptr, offset, 2, "");
return LLVMBuildLoad(bld_base->base.gallivm->builder, ptr, "");
}
LLVMValueRef
lp_build_struct_get_ptr(struct gallivm_state *gallivm,
LLVMValueRef ptr,
unsigned member,
const char *name)
{
LLVMValueRef indices[2];
LLVMValueRef member_ptr;
assert(LLVMGetTypeKind(LLVMTypeOf(ptr)) == LLVMPointerTypeKind);
assert(LLVMGetTypeKind(LLVMGetElementType(LLVMTypeOf(ptr))) == LLVMStructTypeKind);
indices[0] = lp_build_const_int32(gallivm, 0);
indices[1] = lp_build_const_int32(gallivm, member);
member_ptr = LLVMBuildGEP(gallivm->builder, ptr, indices, Elements(indices), "");
lp_build_name(member_ptr, "%s.%s_ptr", LLVMGetValueName(ptr), name);
return member_ptr;
}
LLVMValueRef
lp_build_array_get_ptr(struct gallivm_state *gallivm,
LLVMValueRef ptr,
LLVMValueRef index)
{
LLVMValueRef indices[2];
LLVMValueRef element_ptr;
assert(LLVMGetTypeKind(LLVMTypeOf(ptr)) == LLVMPointerTypeKind);
assert(LLVMGetTypeKind(LLVMGetElementType(LLVMTypeOf(ptr))) == LLVMArrayTypeKind);
indices[0] = lp_build_const_int32(gallivm, 0);
indices[1] = index;
element_ptr = LLVMBuildGEP(gallivm->builder, ptr, indices, Elements(indices), "");
#ifdef DEBUG
lp_build_name(element_ptr, "&%s[%s]",
LLVMGetValueName(ptr), LLVMGetValueName(index));
#endif
return element_ptr;
}
static LLVMValueRef
emit_fetch_output(
struct lp_build_tgsi_context *bld_base,
const struct tgsi_full_src_register *reg,
enum tgsi_opcode_type type,
unsigned swizzle)
{
struct lp_build_tgsi_soa_context *bld = lp_soa_context(bld_base);
LLVMBuilderRef builder = bld_base->base.gallivm->builder;
if (reg->Register.Indirect) {
LLVMValueRef array_index = emit_array_index(bld, reg, swizzle);
LLVMValueRef ptr = LLVMBuildGEP(builder, bld->outputs_array, &array_index,
1, "");
return LLVMBuildLoad(builder, ptr, "");
} else {
LLVMValueRef temp_ptr;
temp_ptr = lp_get_output_ptr(bld, reg->Register.Index, swizzle);
return LLVMBuildLoad(builder, temp_ptr, "");
}
}
/**
* lp_build_printf.
*
* Build printf call in LLVM IR. The output goes to stdout.
* The additional variable arguments need to have type
* LLVMValueRef.
*/
LLVMValueRef
lp_build_printf(struct gallivm_state *gallivm, const char *fmt, ...)
{
va_list arglist;
int i = 0;
int argcount = lp_get_printf_arg_count(fmt);
LLVMBuilderRef builder = gallivm->builder;
LLVMContextRef context = gallivm->context;
LLVMModuleRef module = gallivm->module;
LLVMValueRef params[50];
LLVMValueRef fmtarg = lp_build_const_string_variable(module, context,
fmt, strlen(fmt) + 1);
LLVMValueRef int0 = lp_build_const_int32(gallivm, 0);
LLVMValueRef index[2];
LLVMValueRef func_printf = LLVMGetNamedFunction(module, "printf");
assert(Elements(params) >= argcount + 1);
index[0] = index[1] = int0;
if (!func_printf) {
LLVMTypeRef printf_type = LLVMFunctionType(LLVMIntTypeInContext(context, 32), NULL, 0, 1);
func_printf = LLVMAddFunction(module, "printf", printf_type);
}
params[0] = LLVMBuildGEP(builder, fmtarg, index, 2, "");
va_start(arglist, fmt);
for (i = 1; i <= argcount; i++) {
LLVMValueRef val = va_arg(arglist, LLVMValueRef);
LLVMTypeRef type = LLVMTypeOf(val);
/* printf wants doubles, so lets convert so that
* we can actually print them */
if (LLVMGetTypeKind(type) == LLVMFloatTypeKind)
val = LLVMBuildFPExt(builder, val, LLVMDoubleTypeInContext(context), "");
params[i] = val;
}
va_end(arglist);
return LLVMBuildCall(builder, func_printf, params, argcount + 1, "");
}
static void pointer_update(compile_t* c, reach_type_t* t, reach_type_t* t_elem)
{
FIND_METHOD("_update");
LLVMTypeRef params[3];
params[0] = t->use_type;
params[1] = c->intptr;
params[2] = t_elem->use_type;
start_function(c, m, t_elem->use_type, params, 3);
LLVMValueRef ptr = LLVMGetParam(m->func, 0);
LLVMValueRef index = LLVMGetParam(m->func, 1);
LLVMValueRef elem_ptr = LLVMBuildBitCast(c->builder, ptr,
LLVMPointerType(t_elem->use_type, 0), "");
LLVMValueRef loc = LLVMBuildGEP(c->builder, elem_ptr, &index, 1, "");
LLVMValueRef result = LLVMBuildLoad(c->builder, loc, "");
LLVMBuildStore(c->builder, LLVMGetParam(m->func, 2), loc);
LLVMBuildRet(c->builder, result);
codegen_finishfun(c);
}
/**
* Gather elements from scatter positions in memory into a single vector.
*
* @param src_width src element width
* @param dst_width result element width (source will be expanded to fit)
* @param length length of the offsets,
* @param base_ptr base pointer, should be a i8 pointer type.
* @param offsets vector with offsets
*/
LLVMValueRef
lp_build_gather(LLVMBuilderRef builder,
unsigned length,
unsigned src_width,
unsigned dst_width,
LLVMValueRef base_ptr,
LLVMValueRef offsets)
{
LLVMTypeRef src_type = LLVMIntType(src_width);
LLVMTypeRef src_ptr_type = LLVMPointerType(src_type, 0);
LLVMTypeRef dst_elem_type = LLVMIntType(dst_width);
LLVMTypeRef dst_vec_type = LLVMVectorType(dst_elem_type, length);
LLVMValueRef res;
unsigned i;
res = LLVMGetUndef(dst_vec_type);
for(i = 0; i < length; ++i) {
LLVMValueRef index = LLVMConstInt(LLVMInt32Type(), i, 0);
LLVMValueRef elem_offset;
LLVMValueRef elem_ptr;
LLVMValueRef elem;
elem_offset = LLVMBuildExtractElement(builder, offsets, index, "");
elem_ptr = LLVMBuildGEP(builder, base_ptr, &elem_offset, 1, "");
elem_ptr = LLVMBuildBitCast(builder, elem_ptr, src_ptr_type, "");
elem = LLVMBuildLoad(builder, elem_ptr, "");
assert(src_width <= dst_width);
if(src_width > dst_width)
elem = LLVMBuildTrunc(builder, elem, dst_elem_type, "");
if(src_width < dst_width)
elem = LLVMBuildZExt(builder, elem, dst_elem_type, "");
res = LLVMBuildInsertElement(builder, res, elem, index, "");
}
return res;
}
/**
* Fetch the specified member of the lp_jit_texture structure.
* \param emit_load if TRUE, emit the LLVM load instruction to actually
* fetch the field's value. Otherwise, just emit the
* GEP code to address the field.
*
* @sa http://llvm.org/docs/GetElementPtr.html
*/
static LLVMValueRef
lp_llvm_texture_member(const struct lp_sampler_dynamic_state *base,
struct gallivm_state *gallivm,
unsigned unit,
unsigned member_index,
const char *member_name,
boolean emit_load)
{
struct llvmpipe_sampler_dynamic_state *state =
(struct llvmpipe_sampler_dynamic_state *)base;
LLVMBuilderRef builder = gallivm->builder;
LLVMValueRef indices[4];
LLVMValueRef ptr;
LLVMValueRef res;
assert(unit < PIPE_MAX_SAMPLERS);
/* context[0] */
indices[0] = lp_build_const_int32(gallivm, 0);
/* context[0].textures */
indices[1] = lp_build_const_int32(gallivm, LP_JIT_CTX_TEXTURES);
/* context[0].textures[unit] */
indices[2] = lp_build_const_int32(gallivm, unit);
/* context[0].textures[unit].member */
indices[3] = lp_build_const_int32(gallivm, member_index);
ptr = LLVMBuildGEP(builder, state->context_ptr, indices, Elements(indices), "");
if (emit_load)
res = LLVMBuildLoad(builder, ptr, "");
else
res = ptr;
lp_build_name(res, "context.texture%u.%s", unit, member_name);
return res;
}
/**
* Get the pointer to one element from scatter positions in memory.
*
* @sa lp_build_gather()
*/
LLVMValueRef
lp_build_gather_elem_ptr(struct gallivm_state *gallivm,
unsigned length,
LLVMValueRef base_ptr,
LLVMValueRef offsets,
unsigned i)
{
LLVMValueRef offset;
LLVMValueRef ptr;
assert(LLVMTypeOf(base_ptr) == LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0));
if (length == 1) {
assert(i == 0);
offset = offsets;
} else {
LLVMValueRef index = lp_build_const_int32(gallivm, i);
offset = LLVMBuildExtractElement(gallivm->builder, offsets, index, "");
}
ptr = LLVMBuildGEP(gallivm->builder, base_ptr, &offset, 1, "");
return ptr;
}
/**
* Initialize fragment shader input attribute info.
*/
void
lp_build_interp_soa_init(struct lp_build_interp_soa_context *bld,
struct gallivm_state *gallivm,
unsigned num_inputs,
const struct lp_shader_input *inputs,
boolean pixel_center_integer,
LLVMBuilderRef builder,
struct lp_type type,
LLVMValueRef a0_ptr,
LLVMValueRef dadx_ptr,
LLVMValueRef dady_ptr,
LLVMValueRef x0,
LLVMValueRef y0)
{
struct lp_type coeff_type;
struct lp_type setup_type;
unsigned attrib;
unsigned chan;
memset(bld, 0, sizeof *bld);
memset(&coeff_type, 0, sizeof coeff_type);
coeff_type.floating = TRUE;
coeff_type.sign = TRUE;
coeff_type.width = 32;
coeff_type.length = type.length;
memset(&setup_type, 0, sizeof setup_type);
setup_type.floating = TRUE;
setup_type.sign = TRUE;
setup_type.width = 32;
setup_type.length = TGSI_NUM_CHANNELS;
/* XXX: we don't support interpolating into any other types */
assert(memcmp(&coeff_type, &type, sizeof coeff_type) == 0);
lp_build_context_init(&bld->coeff_bld, gallivm, coeff_type);
lp_build_context_init(&bld->setup_bld, gallivm, setup_type);
/* For convenience */
bld->pos = bld->attribs[0];
bld->inputs = (const LLVMValueRef (*)[TGSI_NUM_CHANNELS]) bld->attribs[1];
/* Position */
bld->mask[0] = TGSI_WRITEMASK_XYZW;
bld->interp[0] = LP_INTERP_LINEAR;
/* Inputs */
for (attrib = 0; attrib < num_inputs; ++attrib) {
bld->mask[1 + attrib] = inputs[attrib].usage_mask;
bld->interp[1 + attrib] = inputs[attrib].interp;
}
bld->num_attribs = 1 + num_inputs;
/* Ensure all masked out input channels have a valid value */
for (attrib = 0; attrib < bld->num_attribs; ++attrib) {
for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
bld->attribs[attrib][chan] = bld->coeff_bld.undef;
}
}
if (pixel_center_integer) {
bld->pos_offset = 0.0;
} else {
bld->pos_offset = 0.5;
}
pos_init(bld, x0, y0);
/*
* Simple method (single step interpolation) may be slower if vector length
* is just 4, but the results are different (generally less accurate) with
* the other method, so always use more accurate version.
*/
if (1) {
bld->simple_interp = TRUE;
{
/* XXX this should use a global static table */
unsigned i;
unsigned num_loops = 16 / type.length;
LLVMValueRef pixoffx, pixoffy, index;
LLVMValueRef ptr;
bld->xoffset_store = lp_build_array_alloca(gallivm,
lp_build_vec_type(gallivm, type),
lp_build_const_int32(gallivm, num_loops),
"");
bld->yoffset_store = lp_build_array_alloca(gallivm,
lp_build_vec_type(gallivm, type),
lp_build_const_int32(gallivm, num_loops),
"");
for (i = 0; i < num_loops; i++) {
index = lp_build_const_int32(gallivm, i);
calc_offsets(&bld->coeff_bld, i*type.length/4, &pixoffx, &pixoffy);
ptr = LLVMBuildGEP(builder, bld->xoffset_store, &index, 1, "");
LLVMBuildStore(builder, pixoffx, ptr);
ptr = LLVMBuildGEP(builder, bld->yoffset_store, &index, 1, "");
LLVMBuildStore(builder, pixoffy, ptr);
}
}
coeffs_init_simple(bld, a0_ptr, dadx_ptr, dady_ptr);
}
else {
bld->simple_interp = FALSE;
coeffs_init(bld, a0_ptr, dadx_ptr, dady_ptr);
}
}
/**
* Increment the shader input attribute values.
* This is called when we move from one quad to the next.
*/
static void
attribs_update(struct lp_build_interp_soa_context *bld,
struct gallivm_state *gallivm,
LLVMValueRef loop_iter,
int start,
int end)
{
LLVMBuilderRef builder = gallivm->builder;
struct lp_build_context *coeff_bld = &bld->coeff_bld;
LLVMValueRef oow = NULL;
unsigned attrib;
unsigned chan;
for(attrib = start; attrib < end; ++attrib) {
const unsigned mask = bld->mask[attrib];
const unsigned interp = bld->interp[attrib];
for(chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
if(mask & (1 << chan)) {
LLVMValueRef a;
if (interp == LP_INTERP_CONSTANT ||
interp == LP_INTERP_FACING) {
a = LLVMBuildLoad(builder, bld->a[attrib][chan], "");
}
else if (interp == LP_INTERP_POSITION) {
assert(attrib > 0);
a = bld->attribs[0][chan];
}
else {
LLVMValueRef dadq;
a = bld->a[attrib][chan];
/*
* Broadcast the attribute value for this quad into all elements
*/
{
/* stored as vector load as float */
LLVMTypeRef ptr_type = LLVMPointerType(LLVMFloatTypeInContext(
gallivm->context), 0);
LLVMValueRef ptr;
a = LLVMBuildBitCast(builder, a, ptr_type, "");
ptr = LLVMBuildGEP(builder, a, &loop_iter, 1, "");
a = LLVMBuildLoad(builder, ptr, "");
a = lp_build_broadcast_scalar(&bld->coeff_bld, a);
}
/*
* Get the derivatives.
*/
dadq = bld->dadq[attrib][chan];
#if PERSPECTIVE_DIVIDE_PER_QUAD
if (interp == LP_INTERP_PERSPECTIVE) {
LLVMValueRef dwdq = bld->dadq[0][3];
if (oow == NULL) {
assert(bld->oow);
oow = LLVMBuildShuffleVector(coeff_bld->builder,
bld->oow, coeff_bld->undef,
shuffle, "");
}
dadq = lp_build_sub(coeff_bld,
dadq,
lp_build_mul(coeff_bld, a, dwdq));
dadq = lp_build_mul(coeff_bld, dadq, oow);
}
#endif
/*
* Add the derivatives
*/
a = lp_build_add(coeff_bld, a, dadq);
#if !PERSPECTIVE_DIVIDE_PER_QUAD
if (interp == LP_INTERP_PERSPECTIVE) {
if (oow == NULL) {
LLVMValueRef w = bld->attribs[0][3];
assert(attrib != 0);
assert(bld->mask[0] & TGSI_WRITEMASK_W);
oow = lp_build_rcp(coeff_bld, w);
}
a = lp_build_mul(coeff_bld, a, oow);
}
#endif
if (attrib == 0 && chan == 2) {
/* FIXME: Depth values can exceed 1.0, due to the fact that
* setup interpolation coefficients refer to (0,0) which causes
* precision loss. So we must clamp to 1.0 here to avoid artifacts
*/
a = lp_build_min(coeff_bld, a, coeff_bld->one);
}
attrib_name(a, attrib, chan, "");
}
bld->attribs[attrib][chan] = a;
}
}
}
}
