static void llvm_load_system_value(
struct radeon_llvm_context * ctx,
unsigned index,
const struct tgsi_full_declaration *decl)
{
unsigned chan;
switch (decl->Semantic.Name) {
case TGSI_SEMANTIC_INSTANCEID: chan = 3; break;
case TGSI_SEMANTIC_VERTEXID: chan = 0; break;
default: assert(!"unknown system value");
}
#if HAVE_LLVM >= 0x0304
ctx->system_values[index] = LLVMBuildExtractElement(ctx->gallivm.builder,
LLVMGetParam(ctx->main_fn, 0), lp_build_const_int32(&(ctx->gallivm), chan),
"");
#else
LLVMValueRef reg = lp_build_const_int32(
ctx->soa.bld_base.base.gallivm, chan);
ctx->system_values[index] = build_intrinsic(
ctx->soa.bld_base.base.gallivm->builder,
"llvm.R600.load.input",
ctx->soa.bld_base.base.elem_type, ®, 1,
LLVMReadNoneAttribute);
#endif
}
/**
* Expands src vector from src.length to dst_length
*/
LLVMValueRef
lp_build_pad_vector(struct gallivm_state *gallivm,
LLVMValueRef src,
struct lp_type src_type,
unsigned dst_length)
{
LLVMValueRef undef = LLVMGetUndef(lp_build_vec_type(gallivm, src_type));
LLVMValueRef elems[LP_MAX_VECTOR_LENGTH];
unsigned i;
assert(dst_length <= Elements(elems));
assert(dst_length > src_type.length);
if (src_type.length == dst_length)
return src;
/* If its a single scalar type, no need to reinvent the wheel */
if (src_type.length == 1) {
return lp_build_broadcast(gallivm, LLVMVectorType(lp_build_elem_type(gallivm, src_type), dst_length), src);
}
/* All elements from src vector */
for (i = 0; i < src_type.length; ++i)
elems[i] = lp_build_const_int32(gallivm, i);
/* Undef fill remaining space */
for (i = src_type.length; i < dst_length; ++i)
elems[i] = lp_build_const_int32(gallivm, src_type.length);
/* Combine the two vectors */
return LLVMBuildShuffleVector(gallivm->builder, src, undef, LLVMConstVector(elems, dst_length), "");
}
/* Find the last bit set. */
static void emit_umsb(const struct lp_build_tgsi_action *action,
struct lp_build_tgsi_context *bld_base,
struct lp_build_emit_data *emit_data)
{
struct gallivm_state *gallivm = bld_base->base.gallivm;
LLVMBuilderRef builder = gallivm->builder;
LLVMValueRef args[2] = {
emit_data->args[0],
/* Don't generate code for handling zero: */
LLVMConstInt(LLVMInt1TypeInContext(gallivm->context), 1, 0)
};
LLVMValueRef msb =
lp_build_intrinsic(builder, "llvm.ctlz.i32",
emit_data->dst_type, args, ARRAY_SIZE(args),
LLVMReadNoneAttribute);
/* The HW returns the last bit index from MSB, but TGSI wants
* the index from LSB. Invert it by doing "31 - msb". */
msb = LLVMBuildSub(builder, lp_build_const_int32(gallivm, 31),
msb, "");
/* Check for zero: */
emit_data->output[emit_data->chan] =
LLVMBuildSelect(builder,
LLVMBuildICmp(builder, LLVMIntEQ, args[0],
bld_base->uint_bld.zero, ""),
lp_build_const_int32(gallivm, -1), msb, "");
}
static void llvm_emit_tex(
const struct lp_build_tgsi_action * action,
struct lp_build_tgsi_context * bld_base,
struct lp_build_emit_data * emit_data)
{
struct gallivm_state * gallivm = bld_base->base.gallivm;
LLVMValueRef args[6];
unsigned c, sampler_src;
assert(emit_data->arg_count + 2 <= Elements(args));
for (c = 0; c < emit_data->arg_count; ++c)
args[c] = emit_data->args[c];
sampler_src = emit_data->inst->Instruction.NumSrcRegs-1;
args[c++] = lp_build_const_int32(gallivm,
emit_data->inst->Src[sampler_src].Register.Index + R600_MAX_CONST_BUFFERS);
args[c++] = lp_build_const_int32(gallivm,
emit_data->inst->Src[sampler_src].Register.Index);
args[c++] = lp_build_const_int32(gallivm,
emit_data->inst->Texture.Texture);
emit_data->output[0] = build_intrinsic(gallivm->builder,
action->intr_name,
emit_data->dst_type, args, c, LLVMReadNoneAttribute);
}
static void
store_cached_block(struct gallivm_state *gallivm,
LLVMValueRef *col,
LLVMValueRef tag_value,
LLVMValueRef hash_index,
LLVMValueRef cache)
{
LLVMBuilderRef builder = gallivm->builder;
LLVMValueRef ptr, indices[3];
LLVMTypeRef type_ptr4x32;
unsigned count;
type_ptr4x32 = LLVMPointerType(LLVMVectorType(LLVMInt32TypeInContext(gallivm->context), 4), 0);
indices[0] = lp_build_const_int32(gallivm, 0);
indices[1] = lp_build_const_int32(gallivm, LP_BUILD_FORMAT_CACHE_MEMBER_TAGS);
indices[2] = hash_index;
ptr = LLVMBuildGEP(builder, cache, indices, ARRAY_SIZE(indices), "");
LLVMBuildStore(builder, tag_value, ptr);
indices[1] = lp_build_const_int32(gallivm, LP_BUILD_FORMAT_CACHE_MEMBER_DATA);
hash_index = LLVMBuildMul(builder, hash_index,
lp_build_const_int32(gallivm, 16), "");
for (count = 0; count < 4; count++) {
indices[2] = hash_index;
ptr = LLVMBuildGEP(builder, cache, indices, ARRAY_SIZE(indices), "");
ptr = LLVMBuildBitCast(builder, ptr, type_ptr4x32, "");
LLVMBuildStore(builder, col[count], ptr);
hash_index = LLVMBuildAdd(builder, hash_index,
lp_build_const_int32(gallivm, 4), "");
}
}
/* Find the last bit opposite of the sign bit. */
static void emit_imsb(const struct lp_build_tgsi_action *action,
struct lp_build_tgsi_context *bld_base,
struct lp_build_emit_data *emit_data)
{
struct gallivm_state *gallivm = bld_base->base.gallivm;
LLVMBuilderRef builder = gallivm->builder;
LLVMValueRef arg = emit_data->args[0];
LLVMValueRef msb =
lp_build_intrinsic(builder, "llvm.AMDGPU.flbit.i32",
emit_data->dst_type, &arg, 1,
LLVMReadNoneAttribute);
/* The HW returns the last bit index from MSB, but TGSI wants
* the index from LSB. Invert it by doing "31 - msb". */
msb = LLVMBuildSub(builder, lp_build_const_int32(gallivm, 31),
msb, "");
/* If arg == 0 || arg == -1 (0xffffffff), return -1. */
LLVMValueRef all_ones = lp_build_const_int32(gallivm, -1);
LLVMValueRef cond =
LLVMBuildOr(builder,
LLVMBuildICmp(builder, LLVMIntEQ, arg,
bld_base->uint_bld.zero, ""),
LLVMBuildICmp(builder, LLVMIntEQ, arg,
all_ones, ""), "");
emit_data->output[emit_data->chan] =
LLVMBuildSelect(builder, cond, all_ones, msb, "");
}
static void declare_input_vs(
struct si_shader_context * si_shader_ctx,
unsigned input_index,
const struct tgsi_full_declaration *decl)
{
struct lp_build_context * base = &si_shader_ctx->radeon_bld.soa.bld_base.base;
unsigned divisor = si_shader_ctx->shader->key.vs.instance_divisors[input_index];
unsigned chan;
LLVMValueRef t_list_ptr;
LLVMValueRef t_offset;
LLVMValueRef t_list;
LLVMValueRef attribute_offset;
LLVMValueRef buffer_index;
LLVMValueRef args[3];
LLVMTypeRef vec4_type;
LLVMValueRef input;
/* Load the T list */
t_list_ptr = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn, SI_PARAM_VERTEX_BUFFER);
t_offset = lp_build_const_int32(base->gallivm, input_index);
t_list = build_indexed_load(si_shader_ctx, t_list_ptr, t_offset);
/* Build the attribute offset */
attribute_offset = lp_build_const_int32(base->gallivm, 0);
if (divisor) {
/* Build index from instance ID, start instance and divisor */
si_shader_ctx->shader->shader.uses_instanceid = true;
buffer_index = get_instance_index(&si_shader_ctx->radeon_bld, divisor);
} else {
/* Load the buffer index, which is always stored in VGPR0
* for Vertex Shaders */
buffer_index = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn, SI_PARAM_VERTEX_ID);
}
vec4_type = LLVMVectorType(base->elem_type, 4);
args[0] = t_list;
args[1] = attribute_offset;
args[2] = buffer_index;
input = build_intrinsic(base->gallivm->builder,
"llvm.SI.vs.load.input", vec4_type, args, 3,
LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
/* Break up the vec4 into individual components */
for (chan = 0; chan < 4; chan++) {
LLVMValueRef llvm_chan = lp_build_const_int32(base->gallivm, chan);
/* XXX: Use a helper function for this. There is one in
* tgsi_llvm.c. */
si_shader_ctx->radeon_bld.inputs[radeon_llvm_reg_index_soa(input_index, chan)] =
LLVMBuildExtractElement(base->gallivm->builder,
input, llvm_chan, "");
}
}
static LLVMValueRef
lookup_tag_data(struct gallivm_state *gallivm,
LLVMValueRef ptr,
LLVMValueRef index)
{
LLVMBuilderRef builder = gallivm->builder;
LLVMValueRef member_ptr, indices[3];
indices[0] = lp_build_const_int32(gallivm, 0);
indices[1] = lp_build_const_int32(gallivm, LP_BUILD_FORMAT_CACHE_MEMBER_TAGS);
indices[2] = index;
member_ptr = LLVMBuildGEP(builder, ptr, indices, ARRAY_SIZE(indices), "");
return LLVMBuildLoad(builder, member_ptr, "tag_data");
}
/*
* Helper for building packed ddx/ddy vector for one coord (scalar per quad
* values). The vector will look like this (8-wide):
* ds1dx ds1dy dt1dx dt1dy ds2dx ds2dy dt2dx dt2dy
* This only needs 2 (v)shufps.
*/
LLVMValueRef
lp_build_packed_ddx_ddy_twocoord(struct lp_build_context *bld,
LLVMValueRef a, LLVMValueRef b)
{
struct gallivm_state *gallivm = bld->gallivm;
LLVMBuilderRef builder = gallivm->builder;
LLVMValueRef shuffles1[LP_MAX_VECTOR_LENGTH/4];
LLVMValueRef shuffles2[LP_MAX_VECTOR_LENGTH/4];
LLVMValueRef vec1, vec2;
unsigned length, num_quads, i;
/* XXX: do hsub version */
length = bld->type.length;
num_quads = length / 4;
for (i = 0; i < num_quads; i++) {
unsigned s1 = 4 * i;
unsigned s2 = 4 * i + length;
shuffles1[4*i + 0] = lp_build_const_int32(gallivm, LP_BLD_QUAD_TOP_LEFT + s1);
shuffles1[4*i + 1] = lp_build_const_int32(gallivm, LP_BLD_QUAD_TOP_LEFT + s1);
shuffles1[4*i + 2] = lp_build_const_int32(gallivm, LP_BLD_QUAD_TOP_LEFT + s2);
shuffles1[4*i + 3] = lp_build_const_int32(gallivm, LP_BLD_QUAD_TOP_LEFT + s2);
shuffles2[4*i + 0] = lp_build_const_int32(gallivm, LP_BLD_QUAD_TOP_RIGHT + s1);
shuffles2[4*i + 1] = lp_build_const_int32(gallivm, LP_BLD_QUAD_BOTTOM_LEFT + s1);
shuffles2[4*i + 2] = lp_build_const_int32(gallivm, LP_BLD_QUAD_TOP_RIGHT + s2);
shuffles2[4*i + 3] = lp_build_const_int32(gallivm, LP_BLD_QUAD_BOTTOM_LEFT + s2);
}
vec1 = LLVMBuildShuffleVector(builder, a, b,
LLVMConstVector(shuffles1, length), "");
vec2 = LLVMBuildShuffleVector(builder, a, b,
LLVMConstVector(shuffles2, length), "");
if (bld->type.floating)
return LLVMBuildFSub(builder, vec2, vec1, "ddxddyddxddy");
else
return LLVMBuildSub(builder, vec2, vec1, "ddxddyddxddy");
}
static void llvm_load_input(
struct radeon_llvm_context * ctx,
unsigned input_index,
const struct tgsi_full_declaration *decl)
{
const struct r600_shader_io * input = &ctx->r600_inputs[input_index];
unsigned chan;
int two_side = (ctx->two_side && input->name == TGSI_SEMANTIC_COLOR);
LLVMValueRef v;
boolean require_interp_intrinsic = ctx->chip_class >= EVERGREEN &&
ctx->type == TGSI_PROCESSOR_FRAGMENT;
if (require_interp_intrinsic && input->spi_sid) {
v = llvm_load_input_vector(ctx, input->lds_pos, input->ij_index,
(input->interpolate > 0));
} else
v = LLVMGetParam(ctx->main_fn, input->gpr);
if (two_side) {
struct r600_shader_io * back_input =
&ctx->r600_inputs[input->back_color_input];
LLVMValueRef v2;
LLVMValueRef face = LLVMGetParam(ctx->main_fn, ctx->face_gpr);
face = LLVMBuildExtractElement(ctx->gallivm.builder, face,
lp_build_const_int32(&(ctx->gallivm), 0), "");
if (require_interp_intrinsic && back_input->spi_sid)
v2 = llvm_load_input_vector(ctx, back_input->lds_pos,
back_input->ij_index, (back_input->interpolate > 0));
else
v2 = LLVMGetParam(ctx->main_fn, back_input->gpr);
v = llvm_face_select_helper(ctx, face, v, v2);
}
for (chan = 0; chan < 4; chan++) {
unsigned soa_index = radeon_llvm_reg_index_soa(input_index, chan);
ctx->inputs[soa_index] = LLVMBuildExtractElement(ctx->gallivm.builder, v,
lp_build_const_int32(&(ctx->gallivm), chan), "");
if (input->name == TGSI_SEMANTIC_POSITION &&
ctx->type == TGSI_PROCESSOR_FRAGMENT && chan == 3) {
/* RCP for fragcoord.w */
ctx->inputs[soa_index] = LLVMBuildFDiv(ctx->gallivm.builder,
lp_build_const_float(&(ctx->gallivm), 1.0f),
ctx->inputs[soa_index], "");
}
}
}
/** Helper used by lp_build_cube_lookup()
* Return (major_coord >= 0) ? pos_face : neg_face;
*/
static LLVMValueRef
lp_build_cube_face(struct lp_build_sample_context *bld,
LLVMValueRef major_coord,
unsigned pos_face, unsigned neg_face)
{
struct gallivm_state *gallivm = bld->gallivm;
LLVMBuilderRef builder = gallivm->builder;
LLVMValueRef cmp = LLVMBuildFCmp(builder, LLVMRealUGE,
major_coord,
bld->float_bld.zero, "");
LLVMValueRef pos = lp_build_const_int32(gallivm, pos_face);
LLVMValueRef neg = lp_build_const_int32(gallivm, neg_face);
LLVMValueRef res = LLVMBuildSelect(builder, cmp, pos, neg, "");
return res;
}
/**
* Concatenates several (must be a power of 2) vectors (of same type)
* into a larger one.
* Most useful for building up a 256bit sized vector out of two 128bit ones.
*/
LLVMValueRef
lp_build_concat(struct gallivm_state *gallivm,
LLVMValueRef src[],
struct lp_type src_type,
unsigned num_vectors)
{
unsigned new_length, i;
LLVMValueRef tmp[LP_MAX_VECTOR_LENGTH/2];
LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH];
assert(src_type.length * num_vectors <= Elements(shuffles));
assert(util_is_power_of_two(num_vectors));
new_length = src_type.length;
for (i = 0; i < num_vectors; i++)
tmp[i] = src[i];
while (num_vectors > 1) {
num_vectors >>= 1;
new_length <<= 1;
for (i = 0; i < new_length; i++) {
shuffles[i] = lp_build_const_int32(gallivm, i);
}
for (i = 0; i < num_vectors; i++) {
tmp[i] = LLVMBuildShuffleVector(gallivm->builder, tmp[i*2], tmp[i*2 + 1],
LLVMConstVector(shuffles, new_length), "");
}
}
return tmp[0];
}
static LLVMValueRef
emit_array_fetch(
struct lp_build_tgsi_context *bld_base,
unsigned File, enum tgsi_opcode_type type,
struct tgsi_declaration_range range,
unsigned swizzle)
{
struct lp_build_tgsi_soa_context *bld = lp_soa_context(bld_base);
struct gallivm_state * gallivm = bld->bld_base.base.gallivm;
LLVMBuilderRef builder = bld_base->base.gallivm->builder;
unsigned i, size = range.Last - range.First + 1;
LLVMTypeRef vec = LLVMVectorType(tgsi2llvmtype(bld_base, type), size);
LLVMValueRef result = LLVMGetUndef(vec);
struct tgsi_full_src_register tmp_reg = {};
tmp_reg.Register.File = File;
for (i = 0; i < size; ++i) {
tmp_reg.Register.Index = i + range.First;
LLVMValueRef temp = emit_fetch(bld_base, &tmp_reg, type, swizzle);
result = LLVMBuildInsertElement(builder, result, temp,
lp_build_const_int32(gallivm, i), "");
}
return result;
}
static void llvm_emit_epilogue(struct lp_build_tgsi_context * bld_base)
{
struct radeon_llvm_context * ctx = radeon_llvm_context(bld_base);
struct lp_build_context * base = &bld_base->base;
unsigned i;
/* Add the necessary export instructions */
for (i = 0; i < ctx->output_reg_count; i++) {
unsigned chan;
for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
LLVMValueRef output;
LLVMValueRef store_output;
unsigned adjusted_reg_idx = i +
ctx->reserved_reg_count;
LLVMValueRef reg_index = lp_build_const_int32(
base->gallivm,
radeon_llvm_reg_index_soa(adjusted_reg_idx, chan));
output = LLVMBuildLoad(base->gallivm->builder,
ctx->soa.outputs[i][chan], "");
store_output = lp_build_intrinsic_binary(
base->gallivm->builder,
"llvm.AMDGPU.store.output",
base->elem_type,
output, reg_index);
lp_build_intrinsic_unary(base->gallivm->builder,
"llvm.AMDGPU.export.reg",
LLVMVoidTypeInContext(base->gallivm->context),
store_output);
}
}
}
LLVMValueRef
lp_build_struct_get_ptr(struct gallivm_state *gallivm,
LLVMValueRef ptr,
unsigned member,
const char *name)
{
LLVMValueRef indices[2];
LLVMValueRef member_ptr;
assert(LLVMGetTypeKind(LLVMTypeOf(ptr)) == LLVMPointerTypeKind);
assert(LLVMGetTypeKind(LLVMGetElementType(LLVMTypeOf(ptr))) == LLVMStructTypeKind);
indices[0] = lp_build_const_int32(gallivm, 0);
indices[1] = lp_build_const_int32(gallivm, member);
member_ptr = LLVMBuildGEP(gallivm->builder, ptr, indices, Elements(indices), "");
lp_build_name(member_ptr, "%s.%s_ptr", LLVMGetValueName(ptr), name);
return member_ptr;
}
static void emit_pk2h(const struct lp_build_tgsi_action *action,
struct lp_build_tgsi_context *bld_base,
struct lp_build_emit_data *emit_data)
{
LLVMBuilderRef builder = bld_base->base.gallivm->builder;
LLVMContextRef context = bld_base->base.gallivm->context;
struct lp_build_context *uint_bld = &bld_base->uint_bld;
LLVMTypeRef fp16, i16;
LLVMValueRef const16, comp[2];
unsigned i;
fp16 = LLVMHalfTypeInContext(context);
i16 = LLVMInt16TypeInContext(context);
const16 = lp_build_const_int32(uint_bld->gallivm, 16);
for (i = 0; i < 2; i++) {
comp[i] = LLVMBuildFPTrunc(builder, emit_data->args[i], fp16, "");
comp[i] = LLVMBuildBitCast(builder, comp[i], i16, "");
comp[i] = LLVMBuildZExt(builder, comp[i], uint_bld->elem_type, "");
}
comp[1] = LLVMBuildShl(builder, comp[1], const16, "");
comp[0] = LLVMBuildOr(builder, comp[0], comp[1], "");
emit_data->output[emit_data->chan] = comp[0];
}
LLVMValueRef
lp_build_intrinsic_map(struct gallivm_state *gallivm,
const char *name,
LLVMTypeRef ret_type,
LLVMValueRef *args,
unsigned num_args)
{
LLVMBuilderRef builder = gallivm->builder;
LLVMTypeRef ret_elem_type = LLVMGetElementType(ret_type);
unsigned n = LLVMGetVectorSize(ret_type);
unsigned i, j;
LLVMValueRef res;
assert(num_args <= LP_MAX_FUNC_ARGS);
res = LLVMGetUndef(ret_type);
for(i = 0; i < n; ++i) {
LLVMValueRef index = lp_build_const_int32(gallivm, i);
LLVMValueRef arg_elems[LP_MAX_FUNC_ARGS];
LLVMValueRef res_elem;
for(j = 0; j < num_args; ++j)
arg_elems[j] = LLVMBuildExtractElement(builder, args[j], index, "");
res_elem = lp_build_intrinsic(builder, name, ret_elem_type, arg_elems, num_args, 0);
res = LLVMBuildInsertElement(builder, res, res_elem, index, "");
}
return res;
}
/**
* Gather elements from scatter positions in memory into a single vector.
* Use for fetching texels from a texture.
* For SSE, typical values are length=4, src_width=32, dst_width=32.
*
* @param length length of the offsets
* @param src_width src element width in bits
* @param dst_width result element width in bits (src will be expanded to fit)
* @param base_ptr base pointer, should be a i8 pointer type.
* @param offsets vector with offsets
*/
LLVMValueRef
lp_build_gather(struct gallivm_state *gallivm,
unsigned length,
unsigned src_width,
unsigned dst_width,
LLVMValueRef base_ptr,
LLVMValueRef offsets)
{
LLVMValueRef res;
if (length == 1) {
/* Scalar */
return lp_build_gather_elem(gallivm, length,
src_width, dst_width,
base_ptr, offsets, 0);
} else {
/* Vector */
LLVMTypeRef dst_elem_type = LLVMIntTypeInContext(gallivm->context, dst_width);
LLVMTypeRef dst_vec_type = LLVMVectorType(dst_elem_type, length);
unsigned i;
res = LLVMGetUndef(dst_vec_type);
for (i = 0; i < length; ++i) {
LLVMValueRef index = lp_build_const_int32(gallivm, i);
LLVMValueRef elem;
elem = lp_build_gather_elem(gallivm, length,
src_width, dst_width,
base_ptr, offsets, i);
res = LLVMBuildInsertElement(gallivm->builder, res, elem, index, "");
}
}
return res;
}
/**
* Print a intt[4] vector.
*/
LLVMValueRef
lp_build_print_ivec4(struct gallivm_state *gallivm,
const char *msg, LLVMValueRef vec)
{
LLVMBuilderRef builder = gallivm->builder;
char format[1000];
LLVMValueRef x, y, z, w;
x = LLVMBuildExtractElement(builder, vec, lp_build_const_int32(gallivm, 0), "");
y = LLVMBuildExtractElement(builder, vec, lp_build_const_int32(gallivm, 1), "");
z = LLVMBuildExtractElement(builder, vec, lp_build_const_int32(gallivm, 2), "");
w = LLVMBuildExtractElement(builder, vec, lp_build_const_int32(gallivm, 3), "");
util_snprintf(format, sizeof(format), "%s %%i %%i %%i %%i\n", msg);
return lp_build_printf(gallivm, format, x, y, z, w);
}
static void
emit_store(
struct lp_build_tgsi_context * bld_base,
const struct tgsi_full_instruction * inst,
const struct tgsi_opcode_info * info,
LLVMValueRef dst[4])
{
struct lp_build_tgsi_soa_context *bld = lp_soa_context(bld_base);
struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
struct lp_build_context base = bld->bld_base.base;
const struct tgsi_full_dst_register *reg = &inst->Dst[0];
LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
LLVMValueRef temp_ptr;
unsigned chan, chan_index;
boolean is_vec_store = FALSE;
if (dst[0]) {
LLVMTypeKind k = LLVMGetTypeKind(LLVMTypeOf(dst[0]));
is_vec_store = (k == LLVMVectorTypeKind);
}
if (is_vec_store) {
LLVMValueRef values[4] = {};
TGSI_FOR_EACH_DST0_ENABLED_CHANNEL(inst, chan) {
LLVMValueRef index = lp_build_const_int32(gallivm, chan);
values[chan] = LLVMBuildExtractElement(gallivm->builder,
dst[0], index, "");
}
bld_base->emit_store(bld_base, inst, info, values);
return;
}
/* this is ffs in C */
static void emit_lsb(const struct lp_build_tgsi_action *action,
struct lp_build_tgsi_context *bld_base,
struct lp_build_emit_data *emit_data)
{
struct gallivm_state *gallivm = bld_base->base.gallivm;
LLVMBuilderRef builder = gallivm->builder;
LLVMValueRef args[2] = {
emit_data->args[0],
/* The value of 1 means that ffs(x=0) = undef, so LLVM won't
* add special code to check for x=0. The reason is that
* the LLVM behavior for x=0 is different from what we
* need here. However, LLVM also assumes that ffs(x) is
* in [0, 31], but GLSL expects that ffs(0) = -1, so
* a conditional assignment to handle 0 is still required.
*/
LLVMConstInt(LLVMInt1TypeInContext(gallivm->context), 1, 0)
};
LLVMValueRef lsb =
lp_build_intrinsic(gallivm->builder, "llvm.cttz.i32",
emit_data->dst_type, args, ARRAY_SIZE(args),
LLVMReadNoneAttribute);
/* TODO: We need an intrinsic to skip this conditional. */
/* Check for zero: */
emit_data->output[emit_data->chan] =
LLVMBuildSelect(builder,
LLVMBuildICmp(builder, LLVMIntEQ, args[0],
bld_base->uint_bld.zero, ""),
lp_build_const_int32(gallivm, -1), lsb, "");
}
LLVMValueRef
lp_build_get_const_mipmap_level(struct lp_build_sample_context *bld,
int level)
{
LLVMValueRef lvl = lp_build_const_int32(bld->gallivm, level);
return lp_build_get_mipmap_level(bld, lvl);
}
static void
calc_offsets(struct lp_build_context *coeff_bld,
unsigned quad_start_index,
LLVMValueRef *pixoffx,
LLVMValueRef *pixoffy)
{
unsigned i;
unsigned num_pix = coeff_bld->type.length;
struct gallivm_state *gallivm = coeff_bld->gallivm;
LLVMBuilderRef builder = coeff_bld->gallivm->builder;
LLVMValueRef nr, pixxf, pixyf;
*pixoffx = coeff_bld->undef;
*pixoffy = coeff_bld->undef;
for (i = 0; i < num_pix; i++) {
nr = lp_build_const_int32(gallivm, i);
pixxf = lp_build_const_float(gallivm, quad_offset_x[i % num_pix] +
(quad_start_index & 1) * 2);
pixyf = lp_build_const_float(gallivm, quad_offset_y[i % num_pix] +
(quad_start_index & 2));
*pixoffx = LLVMBuildInsertElement(builder, *pixoffx, pixxf, nr, "");
*pixoffy = LLVMBuildInsertElement(builder, *pixoffy, pixyf, nr, "");
}
}
/**
* Swizzle a vector consisting of an array of XYZW structs.
*
* This fills a vector of dst_len length with the swizzled channels from src.
*
* e.g. with swizzles = { 2, 1, 0 } and swizzle_count = 6 results in
* RGBA RGBA = BGR BGR BG
*
* @param swizzles the swizzle array
* @param num_swizzles the number of elements in swizzles
* @param dst_len the length of the result
*/
LLVMValueRef
lp_build_swizzle_aos_n(struct gallivm_state* gallivm,
LLVMValueRef src,
const unsigned char* swizzles,
unsigned num_swizzles,
unsigned dst_len)
{
LLVMBuilderRef builder = gallivm->builder;
LLVMValueRef shuffles[LP_MAX_VECTOR_WIDTH];
unsigned i;
assert(dst_len < LP_MAX_VECTOR_WIDTH);
for (i = 0; i < dst_len; ++i) {
int swizzle = swizzles[i % num_swizzles];
if (swizzle == LP_BLD_SWIZZLE_DONTCARE) {
shuffles[i] = LLVMGetUndef(LLVMInt32TypeInContext(gallivm->context));
} else {
shuffles[i] = lp_build_const_int32(gallivm, swizzle);
}
}
return LLVMBuildShuffleVector(builder, src, LLVMGetUndef(LLVMTypeOf(src)), LLVMConstVector(shuffles, dst_len), "");
}
/**
* Build shuffle vectors that match PACKxx (SSE) instructions or
* VPERM (Altivec).
*/
static LLVMValueRef
lp_build_const_pack_shuffle(struct gallivm_state *gallivm, unsigned n)
{
LLVMValueRef elems[LP_MAX_VECTOR_LENGTH];
unsigned i;
assert(n <= LP_MAX_VECTOR_LENGTH);
for(i = 0; i < n; ++i)
#ifdef PIPE_ARCH_LITTLE_ENDIAN
elems[i] = lp_build_const_int32(gallivm, 2*i);
#else
elems[i] = lp_build_const_int32(gallivm, 2*i+1);
#endif
return LLVMConstVector(elems, n);
}
static void llvm_emit_tex(
const struct lp_build_tgsi_action * action,
struct lp_build_tgsi_context * bld_base,
struct lp_build_emit_data * emit_data)
{
struct gallivm_state * gallivm = bld_base->base.gallivm;
LLVMValueRef args[3];
args[0] = emit_data->args[0];
args[1] = lp_build_const_int32(gallivm,
emit_data->inst->Src[1].Register.Index);
args[2] = lp_build_const_int32(gallivm,
emit_data->inst->Texture.Texture);
emit_data->output[0] = lp_build_intrinsic(gallivm->builder,
action->intr_name,
emit_data->dst_type, args, 3);
}
/**
* Print a LLVM value of any type
*/
LLVMValueRef
lp_build_print_value(struct gallivm_state *gallivm,
const char *msg,
LLVMValueRef value)
{
LLVMBuilderRef builder = gallivm->builder;
LLVMTypeKind type_kind;
LLVMTypeRef type_ref;
LLVMValueRef params[2 + LP_MAX_VECTOR_LENGTH];
char type_fmt[4] = " %x";
char format[2 + 3 * LP_MAX_VECTOR_LENGTH + 2] = "%s";
unsigned length;
unsigned i;
type_ref = LLVMTypeOf(value);
type_kind = LLVMGetTypeKind(type_ref);
if (type_kind == LLVMVectorTypeKind) {
length = LLVMGetVectorSize(type_ref);
type_ref = LLVMGetElementType(type_ref);
type_kind = LLVMGetTypeKind(type_ref);
} else {
length = 1;
}
if (type_kind == LLVMFloatTypeKind || type_kind == LLVMDoubleTypeKind) {
type_fmt[2] = 'f';
} else if (type_kind == LLVMIntegerTypeKind) {
if (LLVMGetIntTypeWidth(type_ref) == 8) {
type_fmt[2] = 'u';
} else {
type_fmt[2] = 'i';
}
} else {
/* Unsupported type */
assert(0);
}
/* Create format string and arguments */
assert(strlen(format) + strlen(type_fmt) * length + 2 <= sizeof format);
params[1] = lp_build_const_string(gallivm, msg);
if (length == 1) {
util_strncat(format, type_fmt, sizeof(format) - strlen(format) - 1);
params[2] = value;
} else {
for (i = 0; i < length; ++i) {
util_strncat(format, type_fmt, sizeof(format) - strlen(format) - 1);
params[2 + i] = LLVMBuildExtractElement(builder, value, lp_build_const_int32(gallivm, i), "");
}
}
util_strncat(format, "\n", sizeof(format) - strlen(format) - 1);
params[0] = lp_build_const_string(gallivm, format);
return lp_build_print_args(gallivm, 2 + length, params);
}
void
lp_emit_declaration_aos(
struct lp_build_tgsi_aos_context *bld,
const struct tgsi_full_declaration *decl)
{
struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
LLVMTypeRef vec_type = lp_build_vec_type(bld->bld_base.base.gallivm, bld->bld_base.base.type);
unsigned first = decl->Range.First;
unsigned last = decl->Range.Last;
unsigned idx;
for (idx = first; idx <= last; ++idx) {
switch (decl->Declaration.File) {
case TGSI_FILE_TEMPORARY:
assert(idx < LP_MAX_INLINED_TEMPS);
if (bld->indirect_files & (1 << TGSI_FILE_TEMPORARY)) {
LLVMValueRef array_size = lp_build_const_int32(gallivm, last + 1);
bld->temps_array = lp_build_array_alloca(bld->bld_base.base.gallivm,
vec_type, array_size, "");
} else {
bld->temps[idx] = lp_build_alloca(gallivm, vec_type, "");
}
break;
case TGSI_FILE_OUTPUT:
bld->outputs[idx] = lp_build_alloca(gallivm, vec_type, "");
break;
case TGSI_FILE_ADDRESS:
assert(idx < LP_MAX_TGSI_ADDRS);
bld->addr[idx] = lp_build_alloca(gallivm, vec_type, "");
break;
case TGSI_FILE_PREDICATE:
assert(idx < LP_MAX_TGSI_PREDS);
bld->preds[idx] = lp_build_alloca(gallivm, vec_type, "");
break;
case TGSI_FILE_SAMPLER_VIEW:
/*
* The target stored here MUST match whatever there actually
* is in the set sampler views (what about return type?).
*/
assert(last < PIPE_MAX_SHADER_SAMPLER_VIEWS);
for (idx = first; idx <= last; ++idx) {
bld->sv[idx] = decl->SamplerView;
}
break;
default:
/* don't need to declare other vars */
break;
}
}
}
/**
* Build shuffle vectors that match PUNPCKLxx and PUNPCKHxx instructions.
*/
static LLVMValueRef
lp_build_const_unpack_shuffle(struct gallivm_state *gallivm,
unsigned n, unsigned lo_hi)
{
LLVMValueRef elems[LP_MAX_VECTOR_LENGTH];
unsigned i, j;
assert(n <= LP_MAX_VECTOR_LENGTH);
assert(lo_hi < 2);
/* TODO: cache results in a static table */
for(i = 0, j = lo_hi*n/2; i < n; i += 2, ++j) {
elems[i + 0] = lp_build_const_int32(gallivm, 0 + j);
elems[i + 1] = lp_build_const_int32(gallivm, n + j);
}
return LLVMConstVector(elems, n);
}
static LLVMValueRef
emit_array_index(
struct lp_build_tgsi_soa_context *bld,
const struct tgsi_full_src_register *reg,
unsigned swizzle)
{
struct gallivm_state * gallivm = bld->bld_base.base.gallivm;
LLVMValueRef addr = LLVMBuildLoad(gallivm->builder,
bld->addr[reg->Indirect.Index][swizzle], "");
LLVMValueRef offset = lp_build_const_int32(gallivm, reg->Register.Index);
LLVMValueRef hw_index = LLVMBuildAdd(gallivm->builder, addr, offset, "");
LLVMValueRef soa_index = LLVMBuildMul(gallivm->builder, hw_index,
lp_build_const_int32(gallivm, 4), "");
LLVMValueRef array_index = LLVMBuildAdd(gallivm->builder, soa_index,
lp_build_const_int32(gallivm, swizzle), "");
return array_index;
}
