/**
* Check if the mask predicate is zero. If so, jump to the end of the block.
*/
void
lp_build_mask_check(struct lp_build_mask_context *mask)
{
LLVMBuilderRef builder = mask->skip.gallivm->builder;
LLVMValueRef value;
LLVMValueRef cond;
value = lp_build_mask_value(mask);
/*
* XXX this doesn't quite generate the most efficient code possible, if
* the masks are vectors which have all bits set to the same value
* in each element.
* movmskps/pmovmskb would be more efficient to get the required value
* into ordinary reg (certainly with 8 floats).
* Not sure if llvm could figure that out on its own.
*/
/* cond = (mask == 0) */
cond = LLVMBuildICmp(builder,
LLVMIntEQ,
LLVMBuildBitCast(builder, value, mask->reg_type, ""),
LLVMConstNull(mask->reg_type),
"");
/* if cond, goto end of block */
lp_build_flow_skip_cond_break(&mask->skip, cond);
}
/**
* Update boolean mask with given value (bitwise AND).
* Typically used to update the quad's pixel alive/killed mask
* after depth testing, alpha testing, TGSI_OPCODE_KILL_IF, etc.
*/
void
lp_build_mask_update(struct lp_build_mask_context *mask,
LLVMValueRef value)
{
value = LLVMBuildAnd(mask->skip.gallivm->builder,
lp_build_mask_value(mask),
value, "");
LLVMBuildStore(mask->skip.gallivm->builder, value, mask->var);
}
/**
* Generate code for performing depth and/or stencil tests.
* We operate on a vector of values (typically n 2x2 quads).
*
* \param depth the depth test state
* \param stencil the front/back stencil state
* \param type the data type of the fragment depth/stencil values
* \param format_desc description of the depth/stencil surface
* \param mask the alive/dead pixel mask for the quad (vector)
* \param stencil_refs the front/back stencil ref values (scalar)
* \param z_src the incoming depth/stencil values (n 2x2 quad values, float32)
* \param zs_dst the depth/stencil values in framebuffer
* \param face contains boolean value indicating front/back facing polygon
*/
void
lp_build_depth_stencil_test(struct gallivm_state *gallivm,
const struct pipe_depth_state *depth,
const struct pipe_stencil_state stencil[2],
struct lp_type z_src_type,
const struct util_format_description *format_desc,
struct lp_build_mask_context *mask,
LLVMValueRef stencil_refs[2],
LLVMValueRef z_src,
LLVMValueRef z_fb,
LLVMValueRef s_fb,
LLVMValueRef face,
LLVMValueRef *z_value,
LLVMValueRef *s_value,
boolean do_branch)
{
LLVMBuilderRef builder = gallivm->builder;
struct lp_type z_type;
struct lp_build_context z_bld;
struct lp_build_context s_bld;
struct lp_type s_type;
unsigned z_shift = 0, z_width = 0, z_mask = 0;
LLVMValueRef z_dst = NULL;
LLVMValueRef stencil_vals = NULL;
LLVMValueRef z_bitmask = NULL, stencil_shift = NULL;
LLVMValueRef z_pass = NULL, s_pass_mask = NULL;
LLVMValueRef orig_mask = lp_build_mask_value(mask);
LLVMValueRef front_facing = NULL;
boolean have_z, have_s;
/*
* Depths are expected to be between 0 and 1, even if they are stored in
* floats. Setting these bits here will ensure that the lp_build_conv() call
* below won't try to unnecessarily clamp the incoming values.
*/
if(z_src_type.floating) {
z_src_type.sign = FALSE;
z_src_type.norm = TRUE;
}
else {
assert(!z_src_type.sign);
assert(z_src_type.norm);
}
/* Pick the type matching the depth-stencil format. */
z_type = lp_depth_type(format_desc, z_src_type.length);
/* Pick the intermediate type for depth operations. */
z_type.width = z_src_type.width;
assert(z_type.length == z_src_type.length);
/* FIXME: for non-float depth/stencil might generate better code
* if we'd always split it up to use 128bit operations.
* For stencil we'd almost certainly want to pack to 8xi16 values,
* for z just run twice.
*/
/* Sanity checking */
{
const unsigned z_swizzle = format_desc->swizzle[0];
const unsigned s_swizzle = format_desc->swizzle[1];
assert(z_swizzle != UTIL_FORMAT_SWIZZLE_NONE ||
s_swizzle != UTIL_FORMAT_SWIZZLE_NONE);
assert(depth->enabled || stencil[0].enabled);
assert(format_desc->colorspace == UTIL_FORMAT_COLORSPACE_ZS);
assert(format_desc->block.width == 1);
assert(format_desc->block.height == 1);
if (stencil[0].enabled) {
assert(s_swizzle < 4);
assert(format_desc->channel[s_swizzle].type == UTIL_FORMAT_TYPE_UNSIGNED);
assert(format_desc->channel[s_swizzle].pure_integer);
assert(!format_desc->channel[s_swizzle].normalized);
assert(format_desc->channel[s_swizzle].size == 8);
}
if (depth->enabled) {
assert(z_swizzle < 4);
if (z_type.floating) {
assert(z_swizzle == 0);
assert(format_desc->channel[z_swizzle].type ==
UTIL_FORMAT_TYPE_FLOAT);
assert(format_desc->channel[z_swizzle].size == 32);
}
else {
assert(format_desc->channel[z_swizzle].type ==
UTIL_FORMAT_TYPE_UNSIGNED);
assert(format_desc->channel[z_swizzle].normalized);
assert(!z_type.fixed);
}
}
}
/* Setup build context for Z vals */
lp_build_context_init(&z_bld, gallivm, z_type);
/* Setup build context for stencil vals */
s_type = lp_int_type(z_type);
lp_build_context_init(&s_bld, gallivm, s_type);
/* Compute and apply the Z/stencil bitmasks and shifts.
*/
{
unsigned s_shift, s_mask;
z_dst = z_fb;
stencil_vals = s_fb;
have_z = get_z_shift_and_mask(format_desc, &z_shift, &z_width, &z_mask);
have_s = get_s_shift_and_mask(format_desc, &s_shift, &s_mask);
if (have_z) {
if (z_mask != 0xffffffff) {
z_bitmask = lp_build_const_int_vec(gallivm, z_type, z_mask);
}
/*
* Align the framebuffer Z 's LSB to the right.
*/
if (z_shift) {
LLVMValueRef shift = lp_build_const_int_vec(gallivm, z_type, z_shift);
z_dst = LLVMBuildLShr(builder, z_dst, shift, "z_dst");
} else if (z_bitmask) {
z_dst = LLVMBuildAnd(builder, z_dst, z_bitmask, "z_dst");
} else {
lp_build_name(z_dst, "z_dst");
}
}
if (have_s) {
if (s_shift) {
LLVMValueRef shift = lp_build_const_int_vec(gallivm, s_type, s_shift);
stencil_vals = LLVMBuildLShr(builder, stencil_vals, shift, "");
stencil_shift = shift; /* used below */
}
if (s_mask != 0xffffffff) {
LLVMValueRef mask = lp_build_const_int_vec(gallivm, s_type, s_mask);
stencil_vals = LLVMBuildAnd(builder, stencil_vals, mask, "");
}
lp_build_name(stencil_vals, "s_dst");
}
}
if (stencil[0].enabled) {
if (face) {
LLVMValueRef zero = lp_build_const_int32(gallivm, 0);
/* front_facing = face != 0 ? ~0 : 0 */
front_facing = LLVMBuildICmp(builder, LLVMIntNE, face, zero, "");
front_facing = LLVMBuildSExt(builder, front_facing,
LLVMIntTypeInContext(gallivm->context,
s_bld.type.length*s_bld.type.width),
"");
front_facing = LLVMBuildBitCast(builder, front_facing,
s_bld.int_vec_type, "");
}
/* convert scalar stencil refs into vectors */
stencil_refs[0] = lp_build_broadcast_scalar(&s_bld, stencil_refs[0]);
stencil_refs[1] = lp_build_broadcast_scalar(&s_bld, stencil_refs[1]);
s_pass_mask = lp_build_stencil_test(&s_bld, stencil,
stencil_refs, stencil_vals,
front_facing);
/* apply stencil-fail operator */
{
LLVMValueRef s_fail_mask = lp_build_andnot(&s_bld, orig_mask, s_pass_mask);
stencil_vals = lp_build_stencil_op(&s_bld, stencil, S_FAIL_OP,
stencil_refs, stencil_vals,
s_fail_mask, front_facing);
}
}
if (depth->enabled) {
/*
* Convert fragment Z to the desired type, aligning the LSB to the right.
*/
assert(z_type.width == z_src_type.width);
assert(z_type.length == z_src_type.length);
assert(lp_check_value(z_src_type, z_src));
if (z_src_type.floating) {
/*
* Convert from floating point values
*/
if (!z_type.floating) {
z_src = lp_build_clamped_float_to_unsigned_norm(gallivm,
z_src_type,
z_width,
z_src);
}
} else {
/*
* Convert from unsigned normalized values.
*/
assert(!z_src_type.sign);
assert(!z_src_type.fixed);
assert(z_src_type.norm);
assert(!z_type.floating);
if (z_src_type.width > z_width) {
LLVMValueRef shift = lp_build_const_int_vec(gallivm, z_src_type,
z_src_type.width - z_width);
z_src = LLVMBuildLShr(builder, z_src, shift, "");
}
}
assert(lp_check_value(z_type, z_src));
lp_build_name(z_src, "z_src");
/* compare src Z to dst Z, returning 'pass' mask */
z_pass = lp_build_cmp(&z_bld, depth->func, z_src, z_dst);
if (!stencil[0].enabled) {
/* We can potentially skip all remaining operations here, but only
* if stencil is disabled because we still need to update the stencil
* buffer values. Don't need to update Z buffer values.
*/
lp_build_mask_update(mask, z_pass);
if (do_branch) {
lp_build_mask_check(mask);
do_branch = FALSE;
}
}
if (depth->writemask) {
LLVMValueRef zselectmask;
/* mask off bits that failed Z test */
zselectmask = LLVMBuildAnd(builder, orig_mask, z_pass, "");
/* mask off bits that failed stencil test */
if (s_pass_mask) {
zselectmask = LLVMBuildAnd(builder, zselectmask, s_pass_mask, "");
}
/* Mix the old and new Z buffer values.
* z_dst[i] = zselectmask[i] ? z_src[i] : z_dst[i]
*/
z_dst = lp_build_select(&z_bld, zselectmask, z_src, z_dst);
}
if (stencil[0].enabled) {
/* update stencil buffer values according to z pass/fail result */
LLVMValueRef z_fail_mask, z_pass_mask;
/* apply Z-fail operator */
z_fail_mask = lp_build_andnot(&s_bld, orig_mask, z_pass);
stencil_vals = lp_build_stencil_op(&s_bld, stencil, Z_FAIL_OP,
stencil_refs, stencil_vals,
z_fail_mask, front_facing);
/* apply Z-pass operator */
z_pass_mask = LLVMBuildAnd(builder, orig_mask, z_pass, "");
stencil_vals = lp_build_stencil_op(&s_bld, stencil, Z_PASS_OP,
stencil_refs, stencil_vals,
z_pass_mask, front_facing);
}
}
else {
/* No depth test: apply Z-pass operator to stencil buffer values which
* passed the stencil test.
*/
s_pass_mask = LLVMBuildAnd(builder, orig_mask, s_pass_mask, "");
stencil_vals = lp_build_stencil_op(&s_bld, stencil, Z_PASS_OP,
stencil_refs, stencil_vals,
s_pass_mask, front_facing);
}
/* Put Z and stencil bits in the right place */
if (have_z && z_shift) {
LLVMValueRef shift = lp_build_const_int_vec(gallivm, z_type, z_shift);
z_dst = LLVMBuildShl(builder, z_dst, shift, "");
}
if (stencil_vals && stencil_shift)
stencil_vals = LLVMBuildShl(builder, stencil_vals,
stencil_shift, "");
/* Finally, merge the z/stencil values */
if (format_desc->block.bits <= 32) {
if (have_z && have_s)
*z_value = LLVMBuildOr(builder, z_dst, stencil_vals, "");
else if (have_z)
*z_value = z_dst;
else
*z_value = stencil_vals;
*s_value = *z_value;
}
else {
*z_value = z_dst;
*s_value = stencil_vals;
}
if (s_pass_mask)
lp_build_mask_update(mask, s_pass_mask);
if (depth->enabled && stencil[0].enabled)
lp_build_mask_update(mask, z_pass);
}
/**
* Store depth/stencil values.
* Incoming values are swizzled (typically n 2x2 quads), stored linear.
* If there's a mask it will do select/store otherwise just store.
*
* \param type the data type of the fragment depth/stencil values
* \param format_desc description of the depth/stencil surface
* \param mask the alive/dead pixel mask for the quad (vector)
* \param z_fb z values read from fb (with padding)
* \param s_fb s values read from fb (with padding)
* \param loop_counter the current loop iteration
* \param depth_ptr pointer to the depth/stencil values of this 4x4 block
* \param depth_stride stride of the depth/stencil buffer
* \param z_value the depth values to store (with padding)
* \param s_value the stencil values to store (with padding)
*/
void
lp_build_depth_stencil_write_swizzled(struct gallivm_state *gallivm,
struct lp_type z_src_type,
const struct util_format_description *format_desc,
struct lp_build_mask_context *mask,
LLVMValueRef z_fb,
LLVMValueRef s_fb,
LLVMValueRef loop_counter,
LLVMValueRef depth_ptr,
LLVMValueRef depth_stride,
LLVMValueRef z_value,
LLVMValueRef s_value)
{
struct lp_build_context z_bld;
LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH / 4];
LLVMBuilderRef builder = gallivm->builder;
LLVMValueRef mask_value = NULL;
LLVMValueRef zs_dst1, zs_dst2;
LLVMValueRef zs_dst_ptr1, zs_dst_ptr2;
LLVMValueRef depth_offset1, depth_offset2;
LLVMTypeRef load_ptr_type;
unsigned depth_bytes = format_desc->block.bits / 8;
struct lp_type zs_type = lp_depth_type(format_desc, z_src_type.length);
struct lp_type z_type = zs_type;
struct lp_type zs_load_type = zs_type;
zs_load_type.length = zs_load_type.length / 2;
load_ptr_type = LLVMPointerType(lp_build_vec_type(gallivm, zs_load_type), 0);
z_type.width = z_src_type.width;
lp_build_context_init(&z_bld, gallivm, z_type);
/*
* This is far from ideal, at least for late depth write we should do this
* outside the fs loop to avoid all the swizzle stuff.
*/
if (z_src_type.length == 4) {
LLVMValueRef looplsb = LLVMBuildAnd(builder, loop_counter,
lp_build_const_int32(gallivm, 1), "");
LLVMValueRef loopmsb = LLVMBuildAnd(builder, loop_counter,
lp_build_const_int32(gallivm, 2), "");
LLVMValueRef offset2 = LLVMBuildMul(builder, loopmsb,
depth_stride, "");
depth_offset1 = LLVMBuildMul(builder, looplsb,
lp_build_const_int32(gallivm, depth_bytes * 2), "");
depth_offset1 = LLVMBuildAdd(builder, depth_offset1, offset2, "");
}
else {
unsigned i;
LLVMValueRef loopx2 = LLVMBuildShl(builder, loop_counter,
lp_build_const_int32(gallivm, 1), "");
assert(z_src_type.length == 8);
depth_offset1 = LLVMBuildMul(builder, loopx2, depth_stride, "");
/*
* We load 2x4 values, and need to swizzle them (order
* 0,1,4,5,2,3,6,7) - not so hot with avx unfortunately.
*/
for (i = 0; i < 8; i++) {
shuffles[i] = lp_build_const_int32(gallivm, (i&1) + (i&2) * 2 + (i&4) / 2);
}
}
depth_offset2 = LLVMBuildAdd(builder, depth_offset1, depth_stride, "");
zs_dst_ptr1 = LLVMBuildGEP(builder, depth_ptr, &depth_offset1, 1, "");
zs_dst_ptr1 = LLVMBuildBitCast(builder, zs_dst_ptr1, load_ptr_type, "");
zs_dst_ptr2 = LLVMBuildGEP(builder, depth_ptr, &depth_offset2, 1, "");
zs_dst_ptr2 = LLVMBuildBitCast(builder, zs_dst_ptr2, load_ptr_type, "");
if (format_desc->block.bits > 32) {
s_value = LLVMBuildBitCast(builder, s_value, z_bld.vec_type, "");
}
if (mask) {
mask_value = lp_build_mask_value(mask);
z_value = lp_build_select(&z_bld, mask_value, z_value, z_fb);
if (format_desc->block.bits > 32) {
s_fb = LLVMBuildBitCast(builder, s_fb, z_bld.vec_type, "");
s_value = lp_build_select(&z_bld, mask_value, s_value, s_fb);
}
}
if (zs_type.width < z_src_type.width) {
/* Truncate ZS values (e.g., when writing to Z16_UNORM) */
z_value = LLVMBuildTrunc(builder, z_value,
lp_build_int_vec_type(gallivm, zs_type), "");
}
if (format_desc->block.bits <= 32) {
if (z_src_type.length == 4) {
zs_dst1 = lp_build_extract_range(gallivm, z_value, 0, 2);
zs_dst2 = lp_build_extract_range(gallivm, z_value, 2, 2);
}
else {
assert(z_src_type.length == 8);
zs_dst1 = LLVMBuildShuffleVector(builder, z_value, z_value,
LLVMConstVector(&shuffles[0],
zs_load_type.length), "");
zs_dst2 = LLVMBuildShuffleVector(builder, z_value, z_value,
LLVMConstVector(&shuffles[4],
zs_load_type.length), "");
}
}
else {
if (z_src_type.length == 4) {
zs_dst1 = lp_build_interleave2(gallivm, z_type,
z_value, s_value, 0);
zs_dst2 = lp_build_interleave2(gallivm, z_type,
z_value, s_value, 1);
}
else {
unsigned i;
LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH / 2];
assert(z_src_type.length == 8);
for (i = 0; i < 8; i++) {
shuffles[i*2] = lp_build_const_int32(gallivm, (i&1) + (i&2) * 2 + (i&4) / 2);
shuffles[i*2+1] = lp_build_const_int32(gallivm, (i&1) + (i&2) * 2 + (i&4) / 2 +
z_src_type.length);
}
zs_dst1 = LLVMBuildShuffleVector(builder, z_value, s_value,
LLVMConstVector(&shuffles[0],
z_src_type.length), "");
zs_dst2 = LLVMBuildShuffleVector(builder, z_value, s_value,
LLVMConstVector(&shuffles[8],
z_src_type.length), "");
}
zs_dst1 = LLVMBuildBitCast(builder, zs_dst1,
lp_build_vec_type(gallivm, zs_load_type), "");
zs_dst2 = LLVMBuildBitCast(builder, zs_dst2,
lp_build_vec_type(gallivm, zs_load_type), "");
}
LLVMBuildStore(builder, zs_dst1, zs_dst_ptr1);
LLVMBuildStore(builder, zs_dst2, zs_dst_ptr2);
}
/**
* End section of code which is predicated on a mask.
*/
LLVMValueRef
lp_build_mask_end(struct lp_build_mask_context *mask)
{
lp_build_flow_skip_end(&mask->skip);
return lp_build_mask_value(mask);
}
