void
lp_build_context_init(struct lp_build_context *bld,
struct gallivm_state *gallivm,
struct lp_type type)
{
bld->gallivm = gallivm;
bld->type = type;
bld->int_elem_type = lp_build_int_elem_type(gallivm, type);
if (type.floating)
bld->elem_type = lp_build_elem_type(gallivm, type);
else
bld->elem_type = bld->int_elem_type;
if (type.length == 1) {
bld->int_vec_type = bld->int_elem_type;
bld->vec_type = bld->elem_type;
}
else {
bld->int_vec_type = LLVMVectorType(bld->int_elem_type, type.length);
bld->vec_type = LLVMVectorType(bld->elem_type, type.length);
}
bld->undef = LLVMGetUndef(bld->vec_type);
bld->zero = LLVMConstNull(bld->vec_type);
bld->one = lp_build_one(gallivm, type);
}
/*
* Derive from the quad's upper left scalar coordinates the coordinates for
* all other quad pixels
*/
static void
generate_pos0(LLVMBuilderRef builder,
LLVMValueRef x,
LLVMValueRef y,
LLVMValueRef *x0,
LLVMValueRef *y0)
{
LLVMTypeRef int_elem_type = LLVMInt32Type();
LLVMTypeRef int_vec_type = LLVMVectorType(int_elem_type, QUAD_SIZE);
LLVMTypeRef elem_type = LLVMFloatType();
LLVMTypeRef vec_type = LLVMVectorType(elem_type, QUAD_SIZE);
LLVMValueRef x_offsets[QUAD_SIZE];
LLVMValueRef y_offsets[QUAD_SIZE];
unsigned i;
x = lp_build_broadcast(builder, int_vec_type, x);
y = lp_build_broadcast(builder, int_vec_type, y);
for(i = 0; i < QUAD_SIZE; ++i) {
x_offsets[i] = LLVMConstInt(int_elem_type, quad_offset_x[i], 0);
y_offsets[i] = LLVMConstInt(int_elem_type, quad_offset_y[i], 0);
}
x = LLVMBuildAdd(builder, x, LLVMConstVector(x_offsets, QUAD_SIZE), "");
y = LLVMBuildAdd(builder, y, LLVMConstVector(y_offsets, QUAD_SIZE), "");
*x0 = LLVMBuildSIToFP(builder, x, vec_type, "");
*y0 = LLVMBuildSIToFP(builder, y, vec_type, "");
}
static void
store_cached_block(struct gallivm_state *gallivm,
LLVMValueRef *col,
LLVMValueRef tag_value,
LLVMValueRef hash_index,
LLVMValueRef cache)
{
LLVMBuilderRef builder = gallivm->builder;
LLVMValueRef ptr, indices[3];
LLVMTypeRef type_ptr4x32;
unsigned count;
type_ptr4x32 = LLVMPointerType(LLVMVectorType(LLVMInt32TypeInContext(gallivm->context), 4), 0);
indices[0] = lp_build_const_int32(gallivm, 0);
indices[1] = lp_build_const_int32(gallivm, LP_BUILD_FORMAT_CACHE_MEMBER_TAGS);
indices[2] = hash_index;
ptr = LLVMBuildGEP(builder, cache, indices, ARRAY_SIZE(indices), "");
LLVMBuildStore(builder, tag_value, ptr);
indices[1] = lp_build_const_int32(gallivm, LP_BUILD_FORMAT_CACHE_MEMBER_DATA);
hash_index = LLVMBuildMul(builder, hash_index,
lp_build_const_int32(gallivm, 16), "");
for (count = 0; count < 4; count++) {
indices[2] = hash_index;
ptr = LLVMBuildGEP(builder, cache, indices, ARRAY_SIZE(indices), "");
ptr = LLVMBuildBitCast(builder, ptr, type_ptr4x32, "");
LLVMBuildStore(builder, col[count], ptr);
hash_index = LLVMBuildAdd(builder, hash_index,
lp_build_const_int32(gallivm, 4), "");
}
}
/**
* Expands src vector from src.length to dst_length
*/
LLVMValueRef
lp_build_pad_vector(struct gallivm_state *gallivm,
LLVMValueRef src,
struct lp_type src_type,
unsigned dst_length)
{
LLVMValueRef undef = LLVMGetUndef(lp_build_vec_type(gallivm, src_type));
LLVMValueRef elems[LP_MAX_VECTOR_LENGTH];
unsigned i;
assert(dst_length <= Elements(elems));
assert(dst_length > src_type.length);
if (src_type.length == dst_length)
return src;
/* If its a single scalar type, no need to reinvent the wheel */
if (src_type.length == 1) {
return lp_build_broadcast(gallivm, LLVMVectorType(lp_build_elem_type(gallivm, src_type), dst_length), src);
}
/* All elements from src vector */
for (i = 0; i < src_type.length; ++i)
elems[i] = lp_build_const_int32(gallivm, i);
/* Undef fill remaining space */
for (i = src_type.length; i < dst_length; ++i)
elems[i] = lp_build_const_int32(gallivm, src_type.length);
/* Combine the two vectors */
return LLVMBuildShuffleVector(gallivm->builder, src, undef, LLVMConstVector(elems, dst_length), "");
}
LLVMValueRef
ac_build_gather_values_extended(struct ac_llvm_context *ctx,
LLVMValueRef *values,
unsigned value_count,
unsigned value_stride,
bool load)
{
LLVMBuilderRef builder = ctx->builder;
LLVMValueRef vec = NULL;
unsigned i;
if (value_count == 1) {
if (load)
return LLVMBuildLoad(builder, values[0], "");
return values[0];
} else if (!value_count)
unreachable("value_count is 0");
for (i = 0; i < value_count; i++) {
LLVMValueRef value = values[i * value_stride];
if (load)
value = LLVMBuildLoad(builder, value, "");
if (!i)
vec = LLVMGetUndef( LLVMVectorType(LLVMTypeOf(value), value_count));
LLVMValueRef index = LLVMConstInt(ctx->i32, i, false);
vec = LLVMBuildInsertElement(builder, vec, value, index, "");
}
return vec;
}
/**
* Gather elements from scatter positions in memory into a single vector.
* Use for fetching texels from a texture.
* For SSE, typical values are length=4, src_width=32, dst_width=32.
*
* @param length length of the offsets
* @param src_width src element width in bits
* @param dst_width result element width in bits (src will be expanded to fit)
* @param base_ptr base pointer, should be a i8 pointer type.
* @param offsets vector with offsets
*/
LLVMValueRef
lp_build_gather(struct gallivm_state *gallivm,
unsigned length,
unsigned src_width,
unsigned dst_width,
LLVMValueRef base_ptr,
LLVMValueRef offsets)
{
LLVMValueRef res;
if (length == 1) {
/* Scalar */
return lp_build_gather_elem(gallivm, length,
src_width, dst_width,
base_ptr, offsets, 0);
} else {
/* Vector */
LLVMTypeRef dst_elem_type = LLVMIntTypeInContext(gallivm->context, dst_width);
LLVMTypeRef dst_vec_type = LLVMVectorType(dst_elem_type, length);
unsigned i;
res = LLVMGetUndef(dst_vec_type);
for (i = 0; i < length; ++i) {
LLVMValueRef index = lp_build_const_int32(gallivm, i);
LLVMValueRef elem;
elem = lp_build_gather_elem(gallivm, length,
src_width, dst_width,
base_ptr, offsets, i);
res = LLVMBuildInsertElement(gallivm->builder, res, elem, index, "");
}
}
return res;
}
LLVMValueRef
lp_build_broadcast(struct gallivm_state *gallivm,
LLVMTypeRef vec_type,
LLVMValueRef scalar)
{
LLVMValueRef res;
if (LLVMGetTypeKind(vec_type) != LLVMVectorTypeKind) {
/* scalar */
assert(vec_type == LLVMTypeOf(scalar));
res = scalar;
} else {
LLVMBuilderRef builder = gallivm->builder;
const unsigned length = LLVMGetVectorSize(vec_type);
LLVMValueRef undef = LLVMGetUndef(vec_type);
/* The shuffle vector is always made of int32 elements */
LLVMTypeRef i32_type = LLVMInt32TypeInContext(gallivm->context);
LLVMTypeRef i32_vec_type = LLVMVectorType(i32_type, length);
assert(LLVMGetElementType(vec_type) == LLVMTypeOf(scalar));
res = LLVMBuildInsertElement(builder, undef, scalar, LLVMConstNull(i32_type), "");
res = LLVMBuildShuffleVector(builder, res, undef, LLVMConstNull(i32_vec_type), "");
}
return res;
}
static LLVMValueRef
add_test(LLVMModuleRef module, const char *name, lp_func_t lp_func)
{
LLVMTypeRef v4sf = LLVMVectorType(LLVMFloatType(), 4);
LLVMTypeRef args[1] = { v4sf };
LLVMValueRef func = LLVMAddFunction(module, name, LLVMFunctionType(v4sf, args, 1, 0));
LLVMValueRef arg1 = LLVMGetParam(func, 0);
LLVMBuilderRef builder = LLVMCreateBuilder();
LLVMBasicBlockRef block = LLVMAppendBasicBlock(func, "entry");
LLVMValueRef ret;
struct lp_build_context bld;
bld.builder = builder;
bld.type.floating = 1;
bld.type.width = 32;
bld.type.length = 4;
LLVMSetFunctionCallConv(func, LLVMCCallConv);
LLVMPositionBuilderAtEnd(builder, block);
ret = lp_func(&bld, arg1);
LLVMBuildRet(builder, ret);
LLVMDisposeBuilder(builder);
return func;
}
static void txp_fetch_args(
struct lp_build_tgsi_context * bld_base,
struct lp_build_emit_data * emit_data)
{
const struct tgsi_full_instruction * inst = emit_data->inst;
LLVMValueRef src_w;
unsigned chan;
LLVMValueRef coords[5];
emit_data->dst_type = LLVMVectorType(bld_base->base.elem_type, 4);
src_w = lp_build_emit_fetch(bld_base, emit_data->inst, 0, TGSI_CHAN_W);
for (chan = 0; chan < 3; chan++ ) {
LLVMValueRef arg = lp_build_emit_fetch(bld_base,
emit_data->inst, 0, chan);
coords[chan] = lp_build_emit_llvm_binary(bld_base,
TGSI_OPCODE_DIV, arg, src_w);
}
coords[3] = bld_base->base.one;
if ((inst->Texture.Texture == TGSI_TEXTURE_CUBE ||
inst->Texture.Texture == TGSI_TEXTURE_CUBE_ARRAY ||
inst->Texture.Texture == TGSI_TEXTURE_SHADOWCUBE ||
inst->Texture.Texture == TGSI_TEXTURE_SHADOWCUBE_ARRAY) &&
inst->Instruction.Opcode != TGSI_OPCODE_TXQ &&
inst->Instruction.Opcode != TGSI_OPCODE_TXQ_LZ) {
radeon_llvm_emit_prepare_cube_coords(bld_base, emit_data, coords, NULL);
}
emit_data->args[0] = lp_build_gather_values(bld_base->base.gallivm,
coords, 4);
emit_data->arg_count = 1;
}
static LLVMValueRef
emit_array_fetch(
struct lp_build_tgsi_context *bld_base,
unsigned File, enum tgsi_opcode_type type,
struct tgsi_declaration_range range,
unsigned swizzle)
{
struct lp_build_tgsi_soa_context *bld = lp_soa_context(bld_base);
struct gallivm_state * gallivm = bld->bld_base.base.gallivm;
LLVMBuilderRef builder = bld_base->base.gallivm->builder;
unsigned i, size = range.Last - range.First + 1;
LLVMTypeRef vec = LLVMVectorType(tgsi2llvmtype(bld_base, type), size);
LLVMValueRef result = LLVMGetUndef(vec);
struct tgsi_full_src_register tmp_reg = {};
tmp_reg.Register.File = File;
for (i = 0; i < size; ++i) {
tmp_reg.Register.Index = i + range.First;
LLVMValueRef temp = emit_fetch(bld_base, &tmp_reg, type, swizzle);
result = LLVMBuildInsertElement(builder, result, temp,
lp_build_const_int32(gallivm, i), "");
}
return result;
}
/**
* Combined extract and broadcast (mere shuffle in most cases)
*/
LLVMValueRef
lp_build_extract_broadcast(struct gallivm_state *gallivm,
struct lp_type src_type,
struct lp_type dst_type,
LLVMValueRef vector,
LLVMValueRef index)
{
LLVMTypeRef i32t = LLVMInt32TypeInContext(gallivm->context);
LLVMValueRef res;
assert(src_type.floating == dst_type.floating);
assert(src_type.width == dst_type.width);
assert(lp_check_value(src_type, vector));
assert(LLVMTypeOf(index) == i32t);
if (src_type.length == 1) {
if (dst_type.length == 1) {
/*
* Trivial scalar -> scalar.
*/
res = vector;
}
else {
/*
* Broadcast scalar -> vector.
*/
res = lp_build_broadcast(gallivm,
lp_build_vec_type(gallivm, dst_type),
vector);
}
}
else {
if (dst_type.length > 1) {
/*
* shuffle - result can be of different length.
*/
LLVMValueRef shuffle;
shuffle = lp_build_broadcast(gallivm,
LLVMVectorType(i32t, dst_type.length),
index);
res = LLVMBuildShuffleVector(gallivm->builder, vector,
LLVMGetUndef(lp_build_vec_type(gallivm, src_type)),
shuffle, "");
}
else {
/*
* Trivial extract scalar from vector.
*/
res = LLVMBuildExtractElement(gallivm->builder, vector, index, "");
}
}
return res;
}
static void declare_input_vs(
struct si_shader_context * si_shader_ctx,
unsigned input_index,
const struct tgsi_full_declaration *decl)
{
struct lp_build_context * base = &si_shader_ctx->radeon_bld.soa.bld_base.base;
unsigned divisor = si_shader_ctx->shader->key.vs.instance_divisors[input_index];
unsigned chan;
LLVMValueRef t_list_ptr;
LLVMValueRef t_offset;
LLVMValueRef t_list;
LLVMValueRef attribute_offset;
LLVMValueRef buffer_index;
LLVMValueRef args[3];
LLVMTypeRef vec4_type;
LLVMValueRef input;
/* Load the T list */
t_list_ptr = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn, SI_PARAM_VERTEX_BUFFER);
t_offset = lp_build_const_int32(base->gallivm, input_index);
t_list = build_indexed_load(si_shader_ctx, t_list_ptr, t_offset);
/* Build the attribute offset */
attribute_offset = lp_build_const_int32(base->gallivm, 0);
if (divisor) {
/* Build index from instance ID, start instance and divisor */
si_shader_ctx->shader->shader.uses_instanceid = true;
buffer_index = get_instance_index(&si_shader_ctx->radeon_bld, divisor);
} else {
/* Load the buffer index, which is always stored in VGPR0
* for Vertex Shaders */
buffer_index = LLVMGetParam(si_shader_ctx->radeon_bld.main_fn, SI_PARAM_VERTEX_ID);
}
vec4_type = LLVMVectorType(base->elem_type, 4);
args[0] = t_list;
args[1] = attribute_offset;
args[2] = buffer_index;
input = build_intrinsic(base->gallivm->builder,
"llvm.SI.vs.load.input", vec4_type, args, 3,
LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
/* Break up the vec4 into individual components */
for (chan = 0; chan < 4; chan++) {
LLVMValueRef llvm_chan = lp_build_const_int32(base->gallivm, chan);
/* XXX: Use a helper function for this. There is one in
* tgsi_llvm.c. */
si_shader_ctx->radeon_bld.inputs[radeon_llvm_reg_index_soa(input_index, chan)] =
LLVMBuildExtractElement(base->gallivm->builder,
input, llvm_chan, "");
}
}
LLVMTypeRef
lp_build_vec_type(struct gallivm_state *gallivm,struct lp_type type)
{
LLVMTypeRef elem_type = lp_build_elem_type(gallivm, type);
if (type.length == 1)
return elem_type;
else
return LLVMVectorType(elem_type, type.length);
}
/* Initialize module-independent parts of the context.
*
* The caller is responsible for initializing ctx::module and ctx::builder.
*/
void
ac_llvm_context_init(struct ac_llvm_context *ctx, LLVMContextRef context)
{
LLVMValueRef args[1];
ctx->context = context;
ctx->module = NULL;
ctx->builder = NULL;
ctx->voidt = LLVMVoidTypeInContext(ctx->context);
ctx->i1 = LLVMInt1TypeInContext(ctx->context);
ctx->i8 = LLVMInt8TypeInContext(ctx->context);
ctx->i16 = LLVMIntTypeInContext(ctx->context, 16);
ctx->i32 = LLVMIntTypeInContext(ctx->context, 32);
ctx->i64 = LLVMIntTypeInContext(ctx->context, 64);
ctx->f16 = LLVMHalfTypeInContext(ctx->context);
ctx->f32 = LLVMFloatTypeInContext(ctx->context);
ctx->f64 = LLVMDoubleTypeInContext(ctx->context);
ctx->v4i32 = LLVMVectorType(ctx->i32, 4);
ctx->v4f32 = LLVMVectorType(ctx->f32, 4);
ctx->v8i32 = LLVMVectorType(ctx->i32, 8);
ctx->i32_0 = LLVMConstInt(ctx->i32, 0, false);
ctx->i32_1 = LLVMConstInt(ctx->i32, 1, false);
ctx->f32_0 = LLVMConstReal(ctx->f32, 0.0);
ctx->f32_1 = LLVMConstReal(ctx->f32, 1.0);
ctx->range_md_kind = LLVMGetMDKindIDInContext(ctx->context,
"range", 5);
ctx->invariant_load_md_kind = LLVMGetMDKindIDInContext(ctx->context,
"invariant.load", 14);
ctx->fpmath_md_kind = LLVMGetMDKindIDInContext(ctx->context, "fpmath", 6);
args[0] = LLVMConstReal(ctx->f32, 2.5);
ctx->fpmath_md_2p5_ulp = LLVMMDNodeInContext(ctx->context, args, 1);
ctx->uniform_md_kind = LLVMGetMDKindIDInContext(ctx->context,
"amdgpu.uniform", 14);
ctx->empty_md = LLVMMDNodeInContext(ctx->context, NULL, 0);
}
LLVMModuleRef r600_tgsi_llvm(
struct radeon_llvm_context * ctx,
const struct tgsi_token * tokens)
{
struct tgsi_shader_info shader_info;
struct lp_build_tgsi_context * bld_base = &ctx->soa.bld_base;
radeon_llvm_context_init(ctx);
#if HAVE_LLVM >= 0x0304
LLVMTypeRef Arguments[32];
unsigned ArgumentsCount = 0;
for (unsigned i = 0; i < ctx->inputs_count; i++)
Arguments[ArgumentsCount++] = LLVMVectorType(bld_base->base.elem_type, 4);
radeon_llvm_create_func(ctx, Arguments, ArgumentsCount);
for (unsigned i = 0; i < ctx->inputs_count; i++) {
LLVMValueRef P = LLVMGetParam(ctx->main_fn, i);
LLVMAddAttribute(P, LLVMInRegAttribute);
}
#else
radeon_llvm_create_func(ctx, NULL, 0);
#endif
tgsi_scan_shader(tokens, &shader_info);
bld_base->info = &shader_info;
bld_base->userdata = ctx;
bld_base->emit_fetch_funcs[TGSI_FILE_CONSTANT] = llvm_fetch_const;
bld_base->emit_prologue = llvm_emit_prologue;
bld_base->emit_epilogue = llvm_emit_epilogue;
ctx->userdata = ctx;
ctx->load_input = llvm_load_input;
ctx->load_system_value = llvm_load_system_value;
bld_base->op_actions[TGSI_OPCODE_DP2] = dot_action;
bld_base->op_actions[TGSI_OPCODE_DP3] = dot_action;
bld_base->op_actions[TGSI_OPCODE_DP4] = dot_action;
bld_base->op_actions[TGSI_OPCODE_DPH] = dot_action;
bld_base->op_actions[TGSI_OPCODE_DDX].emit = llvm_emit_tex;
bld_base->op_actions[TGSI_OPCODE_DDY].emit = llvm_emit_tex;
bld_base->op_actions[TGSI_OPCODE_TEX].emit = llvm_emit_tex;
bld_base->op_actions[TGSI_OPCODE_TEX2].emit = llvm_emit_tex;
bld_base->op_actions[TGSI_OPCODE_TXB].emit = llvm_emit_tex;
bld_base->op_actions[TGSI_OPCODE_TXB2].emit = llvm_emit_tex;
bld_base->op_actions[TGSI_OPCODE_TXD].emit = llvm_emit_tex;
bld_base->op_actions[TGSI_OPCODE_TXL].emit = llvm_emit_tex;
bld_base->op_actions[TGSI_OPCODE_TXL2].emit = llvm_emit_tex;
bld_base->op_actions[TGSI_OPCODE_TXF].emit = llvm_emit_tex;
bld_base->op_actions[TGSI_OPCODE_TXQ].emit = llvm_emit_tex;
bld_base->op_actions[TGSI_OPCODE_TXP].emit = llvm_emit_tex;
bld_base->op_actions[TGSI_OPCODE_CMP].emit = emit_cndlt;
lp_build_tgsi_llvm(bld_base, tokens);
radeon_llvm_finalize_module(ctx);
return ctx->gallivm.module;
}
static LLVMValueRef bitcast_to_float(struct ac_llvm_context *ctx,
LLVMValueRef value)
{
LLVMTypeRef type = LLVMTypeOf(value);
LLVMTypeRef new_type;
if (LLVMGetTypeKind(type) == LLVMVectorTypeKind)
new_type = LLVMVectorType(ctx->f32, LLVMGetVectorSize(type));
else
new_type = ctx->f32;
return LLVMBuildBitCast(ctx->builder, value, new_type, "");
}
static LLVMValueRef
llvm_load_input_vector(
struct radeon_llvm_context * ctx, unsigned location, unsigned ijregs,
boolean interp)
{
LLVMTypeRef VecType;
LLVMValueRef Args[3] = {
lp_build_const_int32(&(ctx->gallivm), location)
};
unsigned ArgCount = 1;
if (interp) {
VecType = LLVMVectorType(ctx->soa.bld_base.base.elem_type, 2);
LLVMValueRef IJIndex = LLVMGetParam(ctx->main_fn, ijregs / 2);
Args[ArgCount++] = LLVMBuildExtractElement(ctx->gallivm.builder, IJIndex,
lp_build_const_int32(&(ctx->gallivm), 2 * (ijregs % 2)), "");
Args[ArgCount++] = LLVMBuildExtractElement(ctx->gallivm.builder, IJIndex,
lp_build_const_int32(&(ctx->gallivm), 2 * (ijregs % 2) + 1), "");
LLVMValueRef HalfVec[2] = {
build_intrinsic(ctx->gallivm.builder, "llvm.R600.interp.xy",
VecType, Args, ArgCount, LLVMReadNoneAttribute),
build_intrinsic(ctx->gallivm.builder, "llvm.R600.interp.zw",
VecType, Args, ArgCount, LLVMReadNoneAttribute)
};
LLVMValueRef MaskInputs[4] = {
lp_build_const_int32(&(ctx->gallivm), 0),
lp_build_const_int32(&(ctx->gallivm), 1),
lp_build_const_int32(&(ctx->gallivm), 2),
lp_build_const_int32(&(ctx->gallivm), 3)
};
LLVMValueRef Mask = LLVMConstVector(MaskInputs, 4);
return LLVMBuildShuffleVector(ctx->gallivm.builder, HalfVec[0], HalfVec[1],
Mask, "");
} else {
VecType = LLVMVectorType(ctx->soa.bld_base.base.elem_type, 4);
return build_intrinsic(ctx->gallivm.builder, "llvm.R600.interp.const",
VecType, Args, ArgCount, LLVMReadNoneAttribute);
}
}
/**
* @param n is the number of pixels processed
* @param packed is a <n x i32> vector with the packed YUYV blocks
* @param i is a <n x i32> vector with the x pixel coordinate (0 or 1)
* @return a <4*n x i8> vector with the pixel RGBA values in AoS
*/
LLVMValueRef
lp_build_fetch_subsampled_rgba_aos(struct gallivm_state *gallivm,
const struct util_format_description *format_desc,
unsigned n,
LLVMValueRef base_ptr,
LLVMValueRef offset,
LLVMValueRef i,
LLVMValueRef j)
{
LLVMValueRef packed;
LLVMValueRef rgba;
struct lp_type fetch_type;
assert(format_desc->layout == UTIL_FORMAT_LAYOUT_SUBSAMPLED);
assert(format_desc->block.bits == 32);
assert(format_desc->block.width == 2);
assert(format_desc->block.height == 1);
fetch_type = lp_type_uint(32);
packed = lp_build_gather(gallivm, n, 32, fetch_type, TRUE, base_ptr, offset, FALSE);
(void)j;
switch (format_desc->format) {
case PIPE_FORMAT_UYVY:
rgba = uyvy_to_rgba_aos(gallivm, n, packed, i);
break;
case PIPE_FORMAT_YUYV:
rgba = yuyv_to_rgba_aos(gallivm, n, packed, i);
break;
case PIPE_FORMAT_R8G8_B8G8_UNORM:
rgba = rgbg_to_rgba_aos(gallivm, n, packed, i);
break;
case PIPE_FORMAT_G8R8_G8B8_UNORM:
rgba = grgb_to_rgba_aos(gallivm, n, packed, i);
break;
case PIPE_FORMAT_G8R8_B8R8_UNORM:
rgba = grbr_to_rgba_aos(gallivm, n, packed, i);
break;
case PIPE_FORMAT_R8G8_R8B8_UNORM:
rgba = rgrb_to_rgba_aos(gallivm, n, packed, i);
break;
default:
assert(0);
rgba = LLVMGetUndef(LLVMVectorType(LLVMInt8TypeInContext(gallivm->context), 4*n));
break;
}
return rgba;
}
LLVMValueRef
lp_build_gather_values(struct gallivm_state * gallivm,
LLVMValueRef * values,
unsigned value_count)
{
LLVMTypeRef vec_type = LLVMVectorType(LLVMTypeOf(values[0]), value_count);
LLVMBuilderRef builder = gallivm->builder;
LLVMValueRef vec = LLVMGetUndef(vec_type);
unsigned i;
for (i = 0; i < value_count; i++) {
LLVMValueRef index = lp_build_const_int32(gallivm, i);
vec = LLVMBuildInsertElement(builder, vec, values[i], index, "");
}
return vec;
}
static LLVMValueRef llvm_load_const_buffer(
struct lp_build_tgsi_context * bld_base,
LLVMValueRef OffsetValue,
unsigned ConstantAddressSpace)
{
LLVMValueRef offset[2] = {
LLVMConstInt(LLVMInt64TypeInContext(bld_base->base.gallivm->context), 0, false),
OffsetValue
};
LLVMTypeRef const_ptr_type = LLVMPointerType(LLVMArrayType(LLVMVectorType(bld_base->base.elem_type, 4), 1024),
ConstantAddressSpace);
LLVMValueRef const_ptr = LLVMBuildIntToPtr(bld_base->base.gallivm->builder, lp_build_const_int32(bld_base->base.gallivm, 0), const_ptr_type, "");
LLVMValueRef ptr = LLVMBuildGEP(bld_base->base.gallivm->builder, const_ptr, offset, 2, "");
return LLVMBuildLoad(bld_base->base.gallivm->builder, ptr, "");
}
/**
* Build int32[4] vector type
*/
LLVMTypeRef
lp_build_int32_vec4_type(struct gallivm_state *gallivm)
{
struct lp_type t;
LLVMTypeRef type;
memset(&t, 0, sizeof(t));
t.floating = FALSE; /* floating point values */
t.sign = TRUE; /* values are signed */
t.norm = FALSE; /* values are not limited to [0,1] or [-1,1] */
t.width = 32; /* 32-bit int */
t.length = 4; /* 4 elements per vector */
type = lp_build_int_elem_type(gallivm, t);
return LLVMVectorType(type, t.length);
}
LLVMValueRef ac_build_cvt_pkrtz_f16(struct ac_llvm_context *ctx,
LLVMValueRef args[2])
{
if (HAVE_LLVM >= 0x0500) {
LLVMTypeRef v2f16 =
LLVMVectorType(LLVMHalfTypeInContext(ctx->context), 2);
LLVMValueRef res =
ac_build_intrinsic(ctx, "llvm.amdgcn.cvt.pkrtz",
v2f16, args, 2,
AC_FUNC_ATTR_READNONE);
return LLVMBuildBitCast(ctx->builder, res, ctx->i32, "");
}
return ac_build_intrinsic(ctx, "llvm.SI.packf16", ctx->i32, args, 2,
AC_FUNC_ATTR_READNONE |
AC_FUNC_ATTR_LEGACY);
}
void ac_build_export(struct ac_llvm_context *ctx, struct ac_export_args *a)
{
LLVMValueRef args[9];
if (HAVE_LLVM >= 0x0500) {
args[0] = LLVMConstInt(ctx->i32, a->target, 0);
args[1] = LLVMConstInt(ctx->i32, a->enabled_channels, 0);
if (a->compr) {
LLVMTypeRef i16 = LLVMInt16TypeInContext(ctx->context);
LLVMTypeRef v2i16 = LLVMVectorType(i16, 2);
args[2] = LLVMBuildBitCast(ctx->builder, a->out[0],
v2i16, "");
args[3] = LLVMBuildBitCast(ctx->builder, a->out[1],
v2i16, "");
args[4] = LLVMConstInt(ctx->i1, a->done, 0);
args[5] = LLVMConstInt(ctx->i1, a->valid_mask, 0);
ac_build_intrinsic(ctx, "llvm.amdgcn.exp.compr.v2i16",
ctx->voidt, args, 6, 0);
} else {
args[2] = a->out[0];
args[3] = a->out[1];
args[4] = a->out[2];
args[5] = a->out[3];
args[6] = LLVMConstInt(ctx->i1, a->done, 0);
args[7] = LLVMConstInt(ctx->i1, a->valid_mask, 0);
ac_build_intrinsic(ctx, "llvm.amdgcn.exp.f32",
ctx->voidt, args, 8, 0);
}
return;
}
args[0] = LLVMConstInt(ctx->i32, a->enabled_channels, 0);
args[1] = LLVMConstInt(ctx->i32, a->valid_mask, 0);
args[2] = LLVMConstInt(ctx->i32, a->done, 0);
args[3] = LLVMConstInt(ctx->i32, a->target, 0);
args[4] = LLVMConstInt(ctx->i32, a->compr, 0);
memcpy(args + 5, a->out, sizeof(a->out[0]) * 4);
ac_build_intrinsic(ctx, "llvm.SI.export", ctx->voidt, args, 9,
AC_FUNC_ATTR_LEGACY);
}
static LLVMValueRef
rgb_to_rgba_aos(struct gallivm_state *gallivm,
unsigned n,
LLVMValueRef r, LLVMValueRef g, LLVMValueRef b)
{
LLVMBuilderRef builder = gallivm->builder;
struct lp_type type;
LLVMValueRef a;
LLVMValueRef rgba;
memset(&type, 0, sizeof type);
type.sign = TRUE;
type.width = 32;
type.length = n;
assert(lp_check_value(type, r));
assert(lp_check_value(type, g));
assert(lp_check_value(type, b));
/*
* Make a 4 x unorm8 vector
*/
#ifdef PIPE_ARCH_LITTLE_ENDIAN
r = r;
g = LLVMBuildShl(builder, g, lp_build_const_int_vec(gallivm, type, 8), "");
b = LLVMBuildShl(builder, b, lp_build_const_int_vec(gallivm, type, 16), "");
a = lp_build_const_int_vec(gallivm, type, 0xff000000);
#else
r = LLVMBuildShl(builder, r, lp_build_const_int_vec(gallivm, type, 24), "");
g = LLVMBuildShl(builder, g, lp_build_const_int_vec(gallivm, type, 16), "");
b = LLVMBuildShl(builder, b, lp_build_const_int_vec(gallivm, type, 8), "");
a = lp_build_const_int_vec(gallivm, type, 0x000000ff);
#endif
rgba = r;
rgba = LLVMBuildOr(builder, rgba, g, "");
rgba = LLVMBuildOr(builder, rgba, b, "");
rgba = LLVMBuildOr(builder, rgba, a, "");
rgba = LLVMBuildBitCast(builder, rgba,
LLVMVectorType(LLVMInt8TypeInContext(gallivm->context), 4*n), "");
return rgba;
}
static void txd_fetch_args(
struct lp_build_tgsi_context * bld_base,
struct lp_build_emit_data * emit_data)
{
const struct tgsi_full_instruction * inst = emit_data->inst;
LLVMValueRef coords[4];
unsigned chan, src;
for (src = 0; src < 3; src++) {
for (chan = 0; chan < 4; chan++)
coords[chan] = lp_build_emit_fetch(bld_base, inst, src, chan);
emit_data->args[src] = lp_build_gather_values(bld_base->base.gallivm,
coords, 4);
}
emit_data->arg_count = 3;
emit_data->dst_type = LLVMVectorType(bld_base->base.elem_type, 4);
}
LLVMValueRef
lp_build_one(struct lp_type type)
{
LLVMTypeRef elem_type;
LLVMValueRef elems[LP_MAX_VECTOR_LENGTH];
unsigned i;
assert(type.length <= LP_MAX_VECTOR_LENGTH);
elem_type = lp_build_elem_type(type);
if(type.floating)
elems[0] = LLVMConstReal(elem_type, 1.0);
else if(type.fixed)
elems[0] = LLVMConstInt(elem_type, 1LL << (type.width/2), 0);
else if(!type.norm)
elems[0] = LLVMConstInt(elem_type, 1, 0);
else if(type.sign)
elems[0] = LLVMConstInt(elem_type, (1LL << (type.width - 1)) - 1, 0);
else {
/* special case' -- 1.0 for normalized types is more easily attained if
* we start with a vector consisting of all bits set */
LLVMTypeRef vec_type = LLVMVectorType(elem_type, type.length);
LLVMValueRef vec = LLVMConstAllOnes(vec_type);
#if 0
if(type.sign)
/* TODO: Unfortunately this caused "Tried to create a shift operation
* on a non-integer type!" */
vec = LLVMConstLShr(vec, lp_build_const_int_vec(type, 1));
#endif
return vec;
}
for(i = 1; i < type.length; ++i)
elems[i] = elems[0];
if (type.length == 1)
return elems[0];
else
return LLVMConstVector(elems, type.length);
}
/**
* @param n is the number of pixels processed
* @param packed is a <n x i32> vector with the packed YUYV blocks
* @param i is a <n x i32> vector with the x pixel coordinate (0 or 1)
* @return a <4*n x i8> vector with the pixel RGBA values in AoS
*/
LLVMValueRef
lp_build_fetch_subsampled_rgba_aos(LLVMBuilderRef builder,
const struct util_format_description *format_desc,
unsigned n,
LLVMValueRef base_ptr,
LLVMValueRef offset,
LLVMValueRef i,
LLVMValueRef j)
{
LLVMValueRef packed;
LLVMValueRef rgba;
assert(format_desc->layout == UTIL_FORMAT_LAYOUT_SUBSAMPLED);
assert(format_desc->block.bits == 32);
assert(format_desc->block.width == 2);
assert(format_desc->block.height == 1);
packed = lp_build_gather(builder, n, 32, 32, base_ptr, offset);
(void)j;
switch (format_desc->format) {
case PIPE_FORMAT_UYVY:
rgba = uyvy_to_rgba_aos(builder, n, packed, i);
break;
case PIPE_FORMAT_YUYV:
rgba = yuyv_to_rgba_aos(builder, n, packed, i);
break;
case PIPE_FORMAT_R8G8_B8G8_UNORM:
rgba = rgbg_to_rgba_aos(builder, n, packed, i);
break;
case PIPE_FORMAT_G8R8_G8B8_UNORM:
rgba = grgb_to_rgba_aos(builder, n, packed, i);
break;
default:
assert(0);
rgba = LLVMGetUndef(LLVMVectorType(LLVMInt8Type(), 4*n));
break;
}
return rgba;
}
static void txp_fetch_args(
struct lp_build_tgsi_context * bld_base,
struct lp_build_emit_data * emit_data)
{
LLVMValueRef src_w;
unsigned chan;
LLVMValueRef coords[4];
emit_data->dst_type = LLVMVectorType(bld_base->base.elem_type, 4);
src_w = lp_build_emit_fetch(bld_base, emit_data->inst, 0, TGSI_CHAN_W);
for (chan = 0; chan < 3; chan++ ) {
LLVMValueRef arg = lp_build_emit_fetch(bld_base,
emit_data->inst, 0, chan);
coords[chan] = lp_build_emit_llvm_binary(bld_base,
TGSI_OPCODE_DIV, arg, src_w);
}
coords[3] = bld_base->base.one;
emit_data->args[0] = lp_build_gather_values(bld_base->base.gallivm,
coords, 4);
emit_data->arg_count = 1;
}
/**
* Gather elements from scatter positions in memory into a single vector.
*
* @param src_width src element width
* @param dst_width result element width (source will be expanded to fit)
* @param length length of the offsets,
* @param base_ptr base pointer, should be a i8 pointer type.
* @param offsets vector with offsets
*/
LLVMValueRef
lp_build_gather(LLVMBuilderRef builder,
unsigned length,
unsigned src_width,
unsigned dst_width,
LLVMValueRef base_ptr,
LLVMValueRef offsets)
{
LLVMTypeRef src_type = LLVMIntType(src_width);
LLVMTypeRef src_ptr_type = LLVMPointerType(src_type, 0);
LLVMTypeRef dst_elem_type = LLVMIntType(dst_width);
LLVMTypeRef dst_vec_type = LLVMVectorType(dst_elem_type, length);
LLVMValueRef res;
unsigned i;
res = LLVMGetUndef(dst_vec_type);
for(i = 0; i < length; ++i) {
LLVMValueRef index = LLVMConstInt(LLVMInt32Type(), i, 0);
LLVMValueRef elem_offset;
LLVMValueRef elem_ptr;
LLVMValueRef elem;
elem_offset = LLVMBuildExtractElement(builder, offsets, index, "");
elem_ptr = LLVMBuildGEP(builder, base_ptr, &elem_offset, 1, "");
elem_ptr = LLVMBuildBitCast(builder, elem_ptr, src_ptr_type, "");
elem = LLVMBuildLoad(builder, elem_ptr, "");
assert(src_width <= dst_width);
if(src_width > dst_width)
elem = LLVMBuildTrunc(builder, elem, dst_elem_type, "");
if(src_width < dst_width)
elem = LLVMBuildZExt(builder, elem, dst_elem_type, "");
res = LLVMBuildInsertElement(builder, res, elem, index, "");
}
return res;
}
/**
* Expands src vector from src.length to dst_length
*/
LLVMValueRef
lp_build_pad_vector(struct gallivm_state *gallivm,
LLVMValueRef src,
unsigned dst_length)
{
LLVMValueRef elems[LP_MAX_VECTOR_LENGTH];
LLVMValueRef undef;
LLVMTypeRef type;
unsigned i, src_length;
type = LLVMTypeOf(src);
if (LLVMGetTypeKind(type) != LLVMVectorTypeKind) {
/* Can't use ShuffleVector on non-vector type */
undef = LLVMGetUndef(LLVMVectorType(type, dst_length));
return LLVMBuildInsertElement(gallivm->builder, undef, src, lp_build_const_int32(gallivm, 0), "");
}
undef = LLVMGetUndef(type);
src_length = LLVMGetVectorSize(type);
assert(dst_length <= Elements(elems));
assert(dst_length >= src_length);
if (src_length == dst_length)
return src;
/* All elements from src vector */
for (i = 0; i < src_length; ++i)
elems[i] = lp_build_const_int32(gallivm, i);
/* Undef fill remaining space */
for (i = src_length; i < dst_length; ++i)
elems[i] = lp_build_const_int32(gallivm, src_length);
/* Combine the two vectors */
return LLVMBuildShuffleVector(gallivm->builder, src, undef, LLVMConstVector(elems, dst_length), "");
}
