/**
* Expand the bit width.
*
* This will only change the number of bits the values are represented, not the
* values themselves.
*/
void
lp_build_unpack(struct gallivm_state *gallivm,
struct lp_type src_type,
struct lp_type dst_type,
LLVMValueRef src,
LLVMValueRef *dst, unsigned num_dsts)
{
unsigned num_tmps;
unsigned i;
/* Register width must remain constant */
assert(src_type.width * src_type.length == dst_type.width * dst_type.length);
/* We must not loose or gain channels. Only precision */
assert(src_type.length == dst_type.length * num_dsts);
num_tmps = 1;
dst[0] = src;
while(src_type.width < dst_type.width) {
struct lp_type tmp_type = src_type;
tmp_type.width *= 2;
tmp_type.length /= 2;
for(i = num_tmps; i--; ) {
lp_build_unpack2(gallivm, src_type, tmp_type, dst[i], &dst[2*i + 0], &dst[2*i + 1]);
}
src_type = tmp_type;
num_tmps *= 2;
}
assert(num_tmps == num_dsts);
}
/**
* Sample a single texture image with nearest sampling.
* If sampling a cube texture, r = cube face in [0,5].
* Return filtered color as two vectors of 16-bit fixed point values.
*/
static void
lp_build_sample_image_nearest(struct lp_build_sample_context *bld,
LLVMValueRef int_size,
LLVMValueRef row_stride_vec,
LLVMValueRef img_stride_vec,
LLVMValueRef data_ptr,
LLVMValueRef s,
LLVMValueRef t,
LLVMValueRef r,
LLVMValueRef *colors_lo,
LLVMValueRef *colors_hi)
{
const unsigned dims = bld->dims;
LLVMBuilderRef builder = bld->gallivm->builder;
struct lp_build_context i32, h16, u8n;
LLVMTypeRef i32_vec_type, h16_vec_type, u8n_vec_type;
LLVMValueRef i32_c8;
LLVMValueRef width_vec, height_vec, depth_vec;
LLVMValueRef s_ipart, t_ipart = NULL, r_ipart = NULL;
LLVMValueRef x_stride;
LLVMValueRef x_offset, offset;
LLVMValueRef x_subcoord, y_subcoord, z_subcoord;
lp_build_context_init(&i32, bld->gallivm, lp_type_int_vec(32));
lp_build_context_init(&h16, bld->gallivm, lp_type_ufixed(16));
lp_build_context_init(&u8n, bld->gallivm, lp_type_unorm(8));
i32_vec_type = lp_build_vec_type(bld->gallivm, i32.type);
h16_vec_type = lp_build_vec_type(bld->gallivm, h16.type);
u8n_vec_type = lp_build_vec_type(bld->gallivm, u8n.type);
lp_build_extract_image_sizes(bld,
bld->int_size_type,
bld->int_coord_type,
int_size,
&width_vec,
&height_vec,
&depth_vec);
if (bld->static_state->normalized_coords) {
LLVMValueRef scaled_size;
LLVMValueRef flt_size;
/* scale size by 256 (8 fractional bits) */
scaled_size = lp_build_shl_imm(&bld->int_size_bld, int_size, 8);
flt_size = lp_build_int_to_float(&bld->float_size_bld, scaled_size);
lp_build_unnormalized_coords(bld, flt_size, &s, &t, &r);
}
else {
/* scale coords by 256 (8 fractional bits) */
s = lp_build_mul_imm(&bld->coord_bld, s, 256);
if (dims >= 2)
t = lp_build_mul_imm(&bld->coord_bld, t, 256);
if (dims >= 3)
r = lp_build_mul_imm(&bld->coord_bld, r, 256);
}
/* convert float to int */
s = LLVMBuildFPToSI(builder, s, i32_vec_type, "");
if (dims >= 2)
t = LLVMBuildFPToSI(builder, t, i32_vec_type, "");
if (dims >= 3)
r = LLVMBuildFPToSI(builder, r, i32_vec_type, "");
/* compute floor (shift right 8) */
i32_c8 = lp_build_const_int_vec(bld->gallivm, i32.type, 8);
s_ipart = LLVMBuildAShr(builder, s, i32_c8, "");
if (dims >= 2)
t_ipart = LLVMBuildAShr(builder, t, i32_c8, "");
if (dims >= 3)
r_ipart = LLVMBuildAShr(builder, r, i32_c8, "");
/* get pixel, row, image strides */
x_stride = lp_build_const_vec(bld->gallivm,
bld->int_coord_bld.type,
bld->format_desc->block.bits/8);
/* Do texcoord wrapping, compute texel offset */
lp_build_sample_wrap_nearest_int(bld,
bld->format_desc->block.width,
s_ipart, width_vec, x_stride,
bld->static_state->pot_width,
bld->static_state->wrap_s,
&x_offset, &x_subcoord);
offset = x_offset;
if (dims >= 2) {
LLVMValueRef y_offset;
lp_build_sample_wrap_nearest_int(bld,
bld->format_desc->block.height,
t_ipart, height_vec, row_stride_vec,
bld->static_state->pot_height,
bld->static_state->wrap_t,
&y_offset, &y_subcoord);
offset = lp_build_add(&bld->int_coord_bld, offset, y_offset);
if (dims >= 3) {
LLVMValueRef z_offset;
lp_build_sample_wrap_nearest_int(bld,
1, /* block length (depth) */
r_ipart, depth_vec, img_stride_vec,
bld->static_state->pot_height,
bld->static_state->wrap_r,
&z_offset, &z_subcoord);
offset = lp_build_add(&bld->int_coord_bld, offset, z_offset);
}
else if (bld->static_state->target == PIPE_TEXTURE_CUBE) {
LLVMValueRef z_offset;
/* The r coord is the cube face in [0,5] */
z_offset = lp_build_mul(&bld->int_coord_bld, r, img_stride_vec);
offset = lp_build_add(&bld->int_coord_bld, offset, z_offset);
}
}
/*
* Fetch the pixels as 4 x 32bit (rgba order might differ):
*
* rgba0 rgba1 rgba2 rgba3
*
* bit cast them into 16 x u8
*
* r0 g0 b0 a0 r1 g1 b1 a1 r2 g2 b2 a2 r3 g3 b3 a3
*
* unpack them into two 8 x i16:
*
* r0 g0 b0 a0 r1 g1 b1 a1
* r2 g2 b2 a2 r3 g3 b3 a3
*
* The higher 8 bits of the resulting elements will be zero.
*/
{
LLVMValueRef rgba8;
if (util_format_is_rgba8_variant(bld->format_desc)) {
/*
* Given the format is a rgba8, just read the pixels as is,
* without any swizzling. Swizzling will be done later.
*/
rgba8 = lp_build_gather(bld->gallivm,
bld->texel_type.length,
bld->format_desc->block.bits,
bld->texel_type.width,
data_ptr, offset);
rgba8 = LLVMBuildBitCast(builder, rgba8, u8n_vec_type, "");
}
else {
rgba8 = lp_build_fetch_rgba_aos(bld->gallivm,
bld->format_desc,
u8n.type,
data_ptr, offset,
x_subcoord,
y_subcoord);
}
/* Expand one 4*rgba8 to two 2*rgba16 */
lp_build_unpack2(bld->gallivm, u8n.type, h16.type,
rgba8,
colors_lo, colors_hi);
}
}
/**
* Sample a single texture image with (bi-)(tri-)linear sampling.
* Return filtered color as two vectors of 16-bit fixed point values.
*/
static void
lp_build_sample_image_linear(struct lp_build_sample_context *bld,
LLVMValueRef int_size,
LLVMValueRef row_stride_vec,
LLVMValueRef img_stride_vec,
LLVMValueRef data_ptr,
LLVMValueRef s,
LLVMValueRef t,
LLVMValueRef r,
LLVMValueRef *colors_lo,
LLVMValueRef *colors_hi)
{
const unsigned dims = bld->dims;
LLVMBuilderRef builder = bld->gallivm->builder;
struct lp_build_context i32, h16, u8n;
LLVMTypeRef i32_vec_type, h16_vec_type, u8n_vec_type;
LLVMValueRef i32_c8, i32_c128, i32_c255;
LLVMValueRef width_vec, height_vec, depth_vec;
LLVMValueRef s_ipart, s_fpart, s_fpart_lo, s_fpart_hi;
LLVMValueRef t_ipart = NULL, t_fpart = NULL, t_fpart_lo = NULL, t_fpart_hi = NULL;
LLVMValueRef r_ipart = NULL, r_fpart = NULL, r_fpart_lo = NULL, r_fpart_hi = NULL;
LLVMValueRef x_stride, y_stride, z_stride;
LLVMValueRef x_offset0, x_offset1;
LLVMValueRef y_offset0, y_offset1;
LLVMValueRef z_offset0, z_offset1;
LLVMValueRef offset[2][2][2]; /* [z][y][x] */
LLVMValueRef x_subcoord[2], y_subcoord[2], z_subcoord[2];
LLVMValueRef neighbors_lo[2][2][2]; /* [z][y][x] */
LLVMValueRef neighbors_hi[2][2][2]; /* [z][y][x] */
LLVMValueRef packed_lo, packed_hi;
unsigned x, y, z;
unsigned i, j, k;
unsigned numj, numk;
lp_build_context_init(&i32, bld->gallivm, lp_type_int_vec(32));
lp_build_context_init(&h16, bld->gallivm, lp_type_ufixed(16));
lp_build_context_init(&u8n, bld->gallivm, lp_type_unorm(8));
i32_vec_type = lp_build_vec_type(bld->gallivm, i32.type);
h16_vec_type = lp_build_vec_type(bld->gallivm, h16.type);
u8n_vec_type = lp_build_vec_type(bld->gallivm, u8n.type);
lp_build_extract_image_sizes(bld,
bld->int_size_type,
bld->int_coord_type,
int_size,
&width_vec,
&height_vec,
&depth_vec);
if (bld->static_state->normalized_coords) {
LLVMValueRef scaled_size;
LLVMValueRef flt_size;
/* scale size by 256 (8 fractional bits) */
scaled_size = lp_build_shl_imm(&bld->int_size_bld, int_size, 8);
flt_size = lp_build_int_to_float(&bld->float_size_bld, scaled_size);
lp_build_unnormalized_coords(bld, flt_size, &s, &t, &r);
}
else {
/* scale coords by 256 (8 fractional bits) */
s = lp_build_mul_imm(&bld->coord_bld, s, 256);
if (dims >= 2)
t = lp_build_mul_imm(&bld->coord_bld, t, 256);
if (dims >= 3)
r = lp_build_mul_imm(&bld->coord_bld, r, 256);
}
/* convert float to int */
s = LLVMBuildFPToSI(builder, s, i32_vec_type, "");
if (dims >= 2)
t = LLVMBuildFPToSI(builder, t, i32_vec_type, "");
if (dims >= 3)
r = LLVMBuildFPToSI(builder, r, i32_vec_type, "");
/* subtract 0.5 (add -128) */
i32_c128 = lp_build_const_int_vec(bld->gallivm, i32.type, -128);
s = LLVMBuildAdd(builder, s, i32_c128, "");
if (dims >= 2) {
t = LLVMBuildAdd(builder, t, i32_c128, "");
}
if (dims >= 3) {
r = LLVMBuildAdd(builder, r, i32_c128, "");
}
/* compute floor (shift right 8) */
i32_c8 = lp_build_const_int_vec(bld->gallivm, i32.type, 8);
s_ipart = LLVMBuildAShr(builder, s, i32_c8, "");
if (dims >= 2)
t_ipart = LLVMBuildAShr(builder, t, i32_c8, "");
if (dims >= 3)
r_ipart = LLVMBuildAShr(builder, r, i32_c8, "");
/* compute fractional part (AND with 0xff) */
i32_c255 = lp_build_const_int_vec(bld->gallivm, i32.type, 255);
s_fpart = LLVMBuildAnd(builder, s, i32_c255, "");
if (dims >= 2)
t_fpart = LLVMBuildAnd(builder, t, i32_c255, "");
if (dims >= 3)
r_fpart = LLVMBuildAnd(builder, r, i32_c255, "");
/* get pixel, row and image strides */
x_stride = lp_build_const_vec(bld->gallivm, bld->int_coord_bld.type,
bld->format_desc->block.bits/8);
y_stride = row_stride_vec;
z_stride = img_stride_vec;
/* do texcoord wrapping and compute texel offsets */
lp_build_sample_wrap_linear_int(bld,
bld->format_desc->block.width,
s_ipart, width_vec, x_stride,
bld->static_state->pot_width,
bld->static_state->wrap_s,
&x_offset0, &x_offset1,
&x_subcoord[0], &x_subcoord[1]);
for (z = 0; z < 2; z++) {
for (y = 0; y < 2; y++) {
offset[z][y][0] = x_offset0;
offset[z][y][1] = x_offset1;
}
}
if (dims >= 2) {
lp_build_sample_wrap_linear_int(bld,
bld->format_desc->block.height,
t_ipart, height_vec, y_stride,
bld->static_state->pot_height,
bld->static_state->wrap_t,
&y_offset0, &y_offset1,
&y_subcoord[0], &y_subcoord[1]);
for (z = 0; z < 2; z++) {
for (x = 0; x < 2; x++) {
offset[z][0][x] = lp_build_add(&bld->int_coord_bld,
offset[z][0][x], y_offset0);
offset[z][1][x] = lp_build_add(&bld->int_coord_bld,
offset[z][1][x], y_offset1);
}
}
}
if (dims >= 3) {
lp_build_sample_wrap_linear_int(bld,
bld->format_desc->block.height,
r_ipart, depth_vec, z_stride,
bld->static_state->pot_depth,
bld->static_state->wrap_r,
&z_offset0, &z_offset1,
&z_subcoord[0], &z_subcoord[1]);
for (y = 0; y < 2; y++) {
for (x = 0; x < 2; x++) {
offset[0][y][x] = lp_build_add(&bld->int_coord_bld,
offset[0][y][x], z_offset0);
offset[1][y][x] = lp_build_add(&bld->int_coord_bld,
offset[1][y][x], z_offset1);
}
}
}
else if (bld->static_state->target == PIPE_TEXTURE_CUBE) {
LLVMValueRef z_offset;
z_offset = lp_build_mul(&bld->int_coord_bld, r, img_stride_vec);
for (y = 0; y < 2; y++) {
for (x = 0; x < 2; x++) {
/* The r coord is the cube face in [0,5] */
offset[0][y][x] = lp_build_add(&bld->int_coord_bld,
offset[0][y][x], z_offset);
}
}
}
/*
* Transform 4 x i32 in
*
* s_fpart = {s0, s1, s2, s3}
*
* into 8 x i16
*
* s_fpart = {00, s0, 00, s1, 00, s2, 00, s3}
*
* into two 8 x i16
*
* s_fpart_lo = {s0, s0, s0, s0, s1, s1, s1, s1}
* s_fpart_hi = {s2, s2, s2, s2, s3, s3, s3, s3}
*
* and likewise for t_fpart. There is no risk of loosing precision here
* since the fractional parts only use the lower 8bits.
*/
s_fpart = LLVMBuildBitCast(builder, s_fpart, h16_vec_type, "");
if (dims >= 2)
t_fpart = LLVMBuildBitCast(builder, t_fpart, h16_vec_type, "");
if (dims >= 3)
r_fpart = LLVMBuildBitCast(builder, r_fpart, h16_vec_type, "");
{
LLVMTypeRef elem_type = LLVMInt32TypeInContext(bld->gallivm->context);
LLVMValueRef shuffles_lo[LP_MAX_VECTOR_LENGTH];
LLVMValueRef shuffles_hi[LP_MAX_VECTOR_LENGTH];
LLVMValueRef shuffle_lo;
LLVMValueRef shuffle_hi;
for (j = 0; j < h16.type.length; j += 4) {
#ifdef PIPE_ARCH_LITTLE_ENDIAN
unsigned subindex = 0;
#else
unsigned subindex = 1;
#endif
LLVMValueRef index;
index = LLVMConstInt(elem_type, j/2 + subindex, 0);
for (i = 0; i < 4; ++i)
shuffles_lo[j + i] = index;
index = LLVMConstInt(elem_type, h16.type.length/2 + j/2 + subindex, 0);
for (i = 0; i < 4; ++i)
shuffles_hi[j + i] = index;
}
shuffle_lo = LLVMConstVector(shuffles_lo, h16.type.length);
shuffle_hi = LLVMConstVector(shuffles_hi, h16.type.length);
s_fpart_lo = LLVMBuildShuffleVector(builder, s_fpart, h16.undef,
shuffle_lo, "");
s_fpart_hi = LLVMBuildShuffleVector(builder, s_fpart, h16.undef,
shuffle_hi, "");
if (dims >= 2) {
t_fpart_lo = LLVMBuildShuffleVector(builder, t_fpart, h16.undef,
shuffle_lo, "");
t_fpart_hi = LLVMBuildShuffleVector(builder, t_fpart, h16.undef,
shuffle_hi, "");
}
if (dims >= 3) {
r_fpart_lo = LLVMBuildShuffleVector(builder, r_fpart, h16.undef,
shuffle_lo, "");
r_fpart_hi = LLVMBuildShuffleVector(builder, r_fpart, h16.undef,
shuffle_hi, "");
}
}
/*
* Fetch the pixels as 4 x 32bit (rgba order might differ):
*
* rgba0 rgba1 rgba2 rgba3
*
* bit cast them into 16 x u8
*
* r0 g0 b0 a0 r1 g1 b1 a1 r2 g2 b2 a2 r3 g3 b3 a3
*
* unpack them into two 8 x i16:
*
* r0 g0 b0 a0 r1 g1 b1 a1
* r2 g2 b2 a2 r3 g3 b3 a3
*
* The higher 8 bits of the resulting elements will be zero.
*/
numj = 1 + (dims >= 2);
numk = 1 + (dims >= 3);
for (k = 0; k < numk; k++) {
for (j = 0; j < numj; j++) {
for (i = 0; i < 2; i++) {
LLVMValueRef rgba8;
if (util_format_is_rgba8_variant(bld->format_desc)) {
/*
* Given the format is a rgba8, just read the pixels as is,
* without any swizzling. Swizzling will be done later.
*/
rgba8 = lp_build_gather(bld->gallivm,
bld->texel_type.length,
bld->format_desc->block.bits,
bld->texel_type.width,
data_ptr, offset[k][j][i]);
rgba8 = LLVMBuildBitCast(builder, rgba8, u8n_vec_type, "");
}
else {
rgba8 = lp_build_fetch_rgba_aos(bld->gallivm,
bld->format_desc,
u8n.type,
data_ptr, offset[k][j][i],
x_subcoord[i],
y_subcoord[j]);
}
/* Expand one 4*rgba8 to two 2*rgba16 */
lp_build_unpack2(bld->gallivm, u8n.type, h16.type,
rgba8,
&neighbors_lo[k][j][i], &neighbors_hi[k][j][i]);
}
}
}
/*
* Linear interpolation with 8.8 fixed point.
*/
if (dims == 1) {
/* 1-D lerp */
packed_lo = lp_build_lerp(&h16,
s_fpart_lo,
neighbors_lo[0][0][0],
neighbors_lo[0][0][1]);
packed_hi = lp_build_lerp(&h16,
s_fpart_hi,
neighbors_hi[0][0][0],
neighbors_hi[0][0][1]);
}
else {
/* 2-D lerp */
packed_lo = lp_build_lerp_2d(&h16,
s_fpart_lo, t_fpart_lo,
neighbors_lo[0][0][0],
neighbors_lo[0][0][1],
neighbors_lo[0][1][0],
neighbors_lo[0][1][1]);
packed_hi = lp_build_lerp_2d(&h16,
s_fpart_hi, t_fpart_hi,
neighbors_hi[0][0][0],
neighbors_hi[0][0][1],
neighbors_hi[0][1][0],
neighbors_hi[0][1][1]);
if (dims >= 3) {
LLVMValueRef packed_lo2, packed_hi2;
/* lerp in the second z slice */
packed_lo2 = lp_build_lerp_2d(&h16,
s_fpart_lo, t_fpart_lo,
neighbors_lo[1][0][0],
neighbors_lo[1][0][1],
neighbors_lo[1][1][0],
neighbors_lo[1][1][1]);
packed_hi2 = lp_build_lerp_2d(&h16,
s_fpart_hi, t_fpart_hi,
neighbors_hi[1][0][0],
neighbors_hi[1][0][1],
neighbors_hi[1][1][0],
neighbors_hi[1][1][1]);
/* interp between two z slices */
packed_lo = lp_build_lerp(&h16, r_fpart_lo,
packed_lo, packed_lo2);
packed_hi = lp_build_lerp(&h16, r_fpart_hi,
packed_hi, packed_hi2);
}
}
*colors_lo = packed_lo;
*colors_hi = packed_hi;
}
/**
* Generate a * b
*/
LLVMValueRef
lp_build_mul(struct lp_build_context *bld,
LLVMValueRef a,
LLVMValueRef b)
{
const struct lp_type type = bld->type;
LLVMValueRef shift;
LLVMValueRef res;
if(a == bld->zero)
return bld->zero;
if(a == bld->one)
return b;
if(b == bld->zero)
return bld->zero;
if(b == bld->one)
return a;
if(a == bld->undef || b == bld->undef)
return bld->undef;
if(!type.floating && !type.fixed && type.norm) {
if(type.width == 8) {
struct lp_type i16_type = lp_wider_type(type);
LLVMValueRef al, ah, bl, bh, abl, abh, ab;
lp_build_unpack2(bld->builder, type, i16_type, a, &al, &ah);
lp_build_unpack2(bld->builder, type, i16_type, b, &bl, &bh);
/* PMULLW, PSRLW, PADDW */
abl = lp_build_mul_u8n(bld->builder, i16_type, al, bl);
abh = lp_build_mul_u8n(bld->builder, i16_type, ah, bh);
ab = lp_build_pack2(bld->builder, i16_type, type, abl, abh);
return ab;
}
/* FIXME */
assert(0);
}
if(type.fixed)
shift = lp_build_int_const_scalar(type, type.width/2);
else
shift = NULL;
if(LLVMIsConstant(a) && LLVMIsConstant(b)) {
res = LLVMConstMul(a, b);
if(shift) {
if(type.sign)
res = LLVMConstAShr(res, shift);
else
res = LLVMConstLShr(res, shift);
}
}
else {
res = LLVMBuildMul(bld->builder, a, b, "");
if(shift) {
if(type.sign)
res = LLVMBuildAShr(bld->builder, res, shift, "");
else
res = LLVMBuildLShr(bld->builder, res, shift, "");
}
}
return res;
}
