/** * Compute the offset of a pixel. * * x, y, y_stride are vectors */ LLVMValueRef lp_build_sample_offset(struct lp_build_context *bld, const struct util_format_description *format_desc, LLVMValueRef x, LLVMValueRef y, LLVMValueRef y_stride, LLVMValueRef data_ptr) { LLVMValueRef x_stride; LLVMValueRef offset; x_stride = lp_build_const_scalar(bld->type, format_desc->block.bits/8); if(format_desc->colorspace == UTIL_FORMAT_COLORSPACE_ZS) { LLVMValueRef x_lo, x_hi; LLVMValueRef y_lo, y_hi; LLVMValueRef x_stride_lo, x_stride_hi; LLVMValueRef y_stride_lo, y_stride_hi; LLVMValueRef x_offset_lo, x_offset_hi; LLVMValueRef y_offset_lo, y_offset_hi; LLVMValueRef offset_lo, offset_hi; x_lo = LLVMBuildAnd(bld->builder, x, bld->one, ""); y_lo = LLVMBuildAnd(bld->builder, y, bld->one, ""); x_hi = LLVMBuildLShr(bld->builder, x, bld->one, ""); y_hi = LLVMBuildLShr(bld->builder, y, bld->one, ""); x_stride_lo = x_stride; y_stride_lo = lp_build_const_scalar(bld->type, 2*format_desc->block.bits/8); x_stride_hi = lp_build_const_scalar(bld->type, 4*format_desc->block.bits/8); y_stride_hi = LLVMBuildShl(bld->builder, y_stride, bld->one, ""); x_offset_lo = lp_build_mul(bld, x_lo, x_stride_lo); y_offset_lo = lp_build_mul(bld, y_lo, y_stride_lo); offset_lo = lp_build_add(bld, x_offset_lo, y_offset_lo); x_offset_hi = lp_build_mul(bld, x_hi, x_stride_hi); y_offset_hi = lp_build_mul(bld, y_hi, y_stride_hi); offset_hi = lp_build_add(bld, x_offset_hi, y_offset_hi); offset = lp_build_add(bld, offset_hi, offset_lo); } else { LLVMValueRef x_offset; LLVMValueRef y_offset; x_offset = lp_build_mul(bld, x, x_stride); y_offset = lp_build_mul(bld, y, y_stride); offset = lp_build_add(bld, x_offset, y_offset); } return offset; }
/** * Generate polynomial. * Ex: coeffs[0] + x * coeffs[1] + x^2 * coeffs[2]. */ static LLVMValueRef lp_build_polynomial(struct lp_build_context *bld, LLVMValueRef x, const double *coeffs, unsigned num_coeffs) { const struct lp_type type = bld->type; LLVMValueRef res = NULL; unsigned i; /* TODO: optimize the constant case */ if(LLVMIsConstant(x)) debug_printf("%s: inefficient/imprecise constant arithmetic\n", __FUNCTION__); for (i = num_coeffs; i--; ) { LLVMValueRef coeff = lp_build_const_scalar(type, coeffs[i]); if(res) res = lp_build_add(bld, coeff, lp_build_mul(bld, x, res)); else res = coeff; } if(res) return res; else return bld->undef; }
/** * Compute the offset of a pixel block. * * x, y, z, y_stride, z_stride are vectors, and they refer to pixels. * * Returns the relative offset and i,j sub-block coordinates */ void lp_build_sample_offset(struct lp_build_context *bld, const struct util_format_description *format_desc, LLVMValueRef x, LLVMValueRef y, LLVMValueRef z, LLVMValueRef y_stride, LLVMValueRef z_stride, LLVMValueRef *out_offset, LLVMValueRef *out_i, LLVMValueRef *out_j) { LLVMValueRef x_stride; LLVMValueRef offset; x_stride = lp_build_const_vec(bld->gallivm, bld->type, format_desc->block.bits/8); lp_build_sample_partial_offset(bld, format_desc->block.width, x, x_stride, &offset, out_i); if (y && y_stride) { LLVMValueRef y_offset; lp_build_sample_partial_offset(bld, format_desc->block.height, y, y_stride, &y_offset, out_j); offset = lp_build_add(bld, offset, y_offset); } else { *out_j = bld->zero; } if (z && z_stride) { LLVMValueRef z_offset; LLVMValueRef k; lp_build_sample_partial_offset(bld, 1, /* pixel blocks are always 2D */ z, z_stride, &z_offset, &k); offset = lp_build_add(bld, offset, z_offset); } *out_offset = offset; }
/** * For PIPE_TEX_MIPFILTER_LINEAR, convert float LOD to integer to * two (adjacent) mipmap level indexes. Later, we'll sample from those * two mipmap levels and interpolate between them. */ void lp_build_linear_mip_levels(struct lp_build_sample_context *bld, unsigned unit, LLVMValueRef lod_ipart, LLVMValueRef *lod_fpart_inout, LLVMValueRef *level0_out, LLVMValueRef *level1_out) { LLVMBuilderRef builder = bld->gallivm->builder; struct lp_build_context *int_bld = &bld->int_bld; struct lp_build_context *float_bld = &bld->float_bld; LLVMValueRef last_level; LLVMValueRef clamp_min; LLVMValueRef clamp_max; *level0_out = lod_ipart; *level1_out = lp_build_add(int_bld, lod_ipart, int_bld->one); last_level = bld->dynamic_state->last_level(bld->dynamic_state, bld->gallivm, unit); /* * Clamp both lod_ipart and lod_ipart + 1 to [0, last_level], with the * minimum number of comparisons, and zeroing lod_fpart in the extreme * ends in the process. */ /* lod_ipart < 0 */ clamp_min = LLVMBuildICmp(builder, LLVMIntSLT, lod_ipart, int_bld->zero, "clamp_lod_to_zero"); *level0_out = LLVMBuildSelect(builder, clamp_min, int_bld->zero, *level0_out, ""); *level1_out = LLVMBuildSelect(builder, clamp_min, int_bld->zero, *level1_out, ""); *lod_fpart_inout = LLVMBuildSelect(builder, clamp_min, float_bld->zero, *lod_fpart_inout, ""); /* lod_ipart >= last_level */ clamp_max = LLVMBuildICmp(builder, LLVMIntSGE, lod_ipart, last_level, "clamp_lod_to_last"); *level0_out = LLVMBuildSelect(builder, clamp_max, last_level, *level0_out, ""); *level1_out = LLVMBuildSelect(builder, clamp_max, last_level, *level1_out, ""); *lod_fpart_inout = LLVMBuildSelect(builder, clamp_max, float_bld->zero, *lod_fpart_inout, ""); lp_build_name(*level0_out, "sampler%u_miplevel0", unit); lp_build_name(*level1_out, "sampler%u_miplevel1", unit); lp_build_name(*lod_fpart_inout, "sampler%u_mipweight", unit); }
/* * Bri-linear lod computation * * Use a piece-wise linear approximation of log2 such that: * - round to nearest, for values in the neighborhood of -1, 0, 1, 2, etc. * - linear approximation for values in the neighborhood of 0.5, 1.5., etc, * with the steepness specified in 'factor' * - exact result for 0.5, 1.5, etc. * * * 1.0 - /----* * / * / * / * 0.5 - * * / * / * / * 0.0 - *----/ * * | | * 2^0 2^1 * * This is a technique also commonly used in hardware: * - http://ixbtlabs.com/articles2/gffx/nv40-rx800-3.html * * TODO: For correctness, this should only be applied when texture is known to * have regular mipmaps, i.e., mipmaps derived from the base level. * * TODO: This could be done in fixed point, where applicable. */ static void lp_build_brilinear_lod(struct lp_build_context *bld, LLVMValueRef lod, double factor, LLVMValueRef *out_lod_ipart, LLVMValueRef *out_lod_fpart) { LLVMValueRef lod_fpart; double pre_offset = (factor - 0.5)/factor - 0.5; double post_offset = 1 - factor; if (0) { lp_build_printf(bld->gallivm, "lod = %f\n", lod); } lod = lp_build_add(bld, lod, lp_build_const_vec(bld->gallivm, bld->type, pre_offset)); lp_build_ifloor_fract(bld, lod, out_lod_ipart, &lod_fpart); lod_fpart = lp_build_mul(bld, lod_fpart, lp_build_const_vec(bld->gallivm, bld->type, factor)); lod_fpart = lp_build_add(bld, lod_fpart, lp_build_const_vec(bld->gallivm, bld->type, post_offset)); /* * It's not necessary to clamp lod_fpart since: * - the above expression will never produce numbers greater than one. * - the mip filtering branch is only taken if lod_fpart is positive */ *out_lod_fpart = lod_fpart; if (0) { lp_build_printf(bld->gallivm, "lod_ipart = %i\n", *out_lod_ipart); lp_build_printf(bld->gallivm, "lod_fpart = %f\n\n", *out_lod_fpart); } }
/** * Small vector x scale multiplication optimization. */ LLVMValueRef lp_build_mul_imm(struct lp_build_context *bld, LLVMValueRef a, int b) { LLVMValueRef factor; if(b == 0) return bld->zero; if(b == 1) return a; if(b == -1) return LLVMBuildNeg(bld->builder, a, ""); if(b == 2 && bld->type.floating) return lp_build_add(bld, a, a); if(util_is_pot(b)) { unsigned shift = ffs(b) - 1; if(bld->type.floating) { #if 0 /* * Power of two multiplication by directly manipulating the mantissa. * * XXX: This might not be always faster, it will introduce a small error * for multiplication by zero, and it will produce wrong results * for Inf and NaN. */ unsigned mantissa = lp_mantissa(bld->type); factor = lp_build_int_const_scalar(bld->type, (unsigned long long)shift << mantissa); a = LLVMBuildBitCast(bld->builder, a, lp_build_int_vec_type(bld->type), ""); a = LLVMBuildAdd(bld->builder, a, factor, ""); a = LLVMBuildBitCast(bld->builder, a, lp_build_vec_type(bld->type), ""); return a; #endif } else { factor = lp_build_const_scalar(bld->type, shift); return LLVMBuildShl(bld->builder, a, factor, ""); } } factor = lp_build_const_scalar(bld->type, (double)b); return lp_build_mul(bld, a, factor); }
/* * Combined log2 and brilinear lod computation. * * It's in all identical to calling lp_build_fast_log2() and * lp_build_brilinear_lod() above, but by combining we can compute the integer * and fractional part independently. */ static void lp_build_brilinear_rho(struct lp_build_context *bld, LLVMValueRef rho, double factor, LLVMValueRef *out_lod_ipart, LLVMValueRef *out_lod_fpart) { LLVMValueRef lod_ipart; LLVMValueRef lod_fpart; const double pre_factor = (2*factor - 0.5)/(M_SQRT2*factor); const double post_offset = 1 - 2*factor; assert(bld->type.floating); assert(lp_check_value(bld->type, rho)); /* * The pre factor will make the intersections with the exact powers of two * happen precisely where we want then to be, which means that the integer * part will not need any post adjustments. */ rho = lp_build_mul(bld, rho, lp_build_const_vec(bld->gallivm, bld->type, pre_factor)); /* ipart = ifloor(log2(rho)) */ lod_ipart = lp_build_extract_exponent(bld, rho, 0); /* fpart = rho / 2**ipart */ lod_fpart = lp_build_extract_mantissa(bld, rho); lod_fpart = lp_build_mul(bld, lod_fpart, lp_build_const_vec(bld->gallivm, bld->type, factor)); lod_fpart = lp_build_add(bld, lod_fpart, lp_build_const_vec(bld->gallivm, bld->type, post_offset)); /* * Like lp_build_brilinear_lod, it's not necessary to clamp lod_fpart since: * - the above expression will never produce numbers greater than one. * - the mip filtering branch is only taken if lod_fpart is positive */ *out_lod_ipart = lod_ipart; *out_lod_fpart = lod_fpart; }
/** * For PIPE_TEX_MIPFILTER_NEAREST, convert float LOD to integer * mipmap level index. * Note: this is all scalar code. * \param lod scalar float texture level of detail * \param level_out returns integer */ void lp_build_nearest_mip_level(struct lp_build_sample_context *bld, unsigned unit, LLVMValueRef lod_ipart, LLVMValueRef *level_out) { struct lp_build_context *int_bld = &bld->int_bld; LLVMValueRef first_level, last_level, level; first_level = bld->dynamic_state->first_level(bld->dynamic_state, bld->gallivm, unit); last_level = bld->dynamic_state->last_level(bld->dynamic_state, bld->gallivm, unit); /* convert float lod to integer */ level = lp_build_add(int_bld, lod_ipart, first_level); /* clamp level to legal range of levels */ *level_out = lp_build_clamp(int_bld, level, first_level, last_level); }
static LLVMValueRef fetch_constant( struct lp_build_tgsi_context * bld_base, const struct tgsi_full_src_register *reg, enum tgsi_opcode_type type, unsigned swizzle) { struct si_shader_context *si_shader_ctx = si_shader_context(bld_base); struct lp_build_context * base = &bld_base->base; const struct tgsi_ind_register *ireg = ®->Indirect; unsigned idx; LLVMValueRef args[2]; LLVMValueRef addr; LLVMValueRef result; if (swizzle == LP_CHAN_ALL) { unsigned chan; LLVMValueRef values[4]; for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) values[chan] = fetch_constant(bld_base, reg, type, chan); return lp_build_gather_values(bld_base->base.gallivm, values, 4); } idx = reg->Register.Index * 4 + swizzle; if (!reg->Register.Indirect) return bitcast(bld_base, type, si_shader_ctx->constants[idx]); args[0] = si_shader_ctx->const_resource; args[1] = lp_build_const_int32(base->gallivm, idx * 4); addr = si_shader_ctx->radeon_bld.soa.addr[ireg->Index][ireg->Swizzle]; addr = LLVMBuildLoad(base->gallivm->builder, addr, "load addr reg"); addr = lp_build_mul_imm(&bld_base->uint_bld, addr, 16); args[1] = lp_build_add(&bld_base->uint_bld, addr, args[1]); result = build_intrinsic(base->gallivm->builder, "llvm.SI.load.const", base->elem_type, args, 2, LLVMReadNoneAttribute | LLVMNoUnwindAttribute); return bitcast(bld_base, type, result); }
/** * @sa http://www.opengl.org/sdk/docs/man/xhtml/glBlendEquationSeparate.xml */ LLVMValueRef lp_build_blend_func(struct lp_build_context *bld, unsigned func, LLVMValueRef term1, LLVMValueRef term2) { switch (func) { case PIPE_BLEND_ADD: return lp_build_add(bld, term1, term2); case PIPE_BLEND_SUBTRACT: return lp_build_sub(bld, term1, term2); case PIPE_BLEND_REVERSE_SUBTRACT: return lp_build_sub(bld, term2, term1); case PIPE_BLEND_MIN: return lp_build_min(bld, term1, term2); case PIPE_BLEND_MAX: return lp_build_max(bld, term1, term2); default: assert(0); return bld->zero; } }
/** * Linear interpolation. * * This also works for integer values with a few caveats. * * @sa http://www.stereopsis.com/doubleblend.html */ LLVMValueRef lp_build_lerp(struct lp_build_context *bld, LLVMValueRef x, LLVMValueRef v0, LLVMValueRef v1) { LLVMValueRef delta; LLVMValueRef res; delta = lp_build_sub(bld, v1, v0); res = lp_build_mul(bld, x, delta); res = lp_build_add(bld, v0, res); if(bld->type.fixed) /* XXX: This step is necessary for lerping 8bit colors stored on 16bits, * but it will be wrong for other uses. Basically we need a more * powerful lp_type, capable of further distinguishing the values * interpretation from the value storage. */ res = LLVMBuildAnd(bld->builder, res, lp_build_int_const_scalar(bld->type, (1 << bld->type.width/2) - 1), ""); return res; }
/** * Helper used by lp_build_cube_lookup() * \param sign scalar +1 or -1 * \param coord float vector * \param ima float vector */ static LLVMValueRef lp_build_cube_coord(struct lp_build_context *coord_bld, LLVMValueRef sign, int negate_coord, LLVMValueRef coord, LLVMValueRef ima) { /* return negate(coord) * ima * sign + 0.5; */ LLVMValueRef half = lp_build_const_vec(coord_bld->gallivm, coord_bld->type, 0.5); LLVMValueRef res; assert(negate_coord == +1 || negate_coord == -1); if (negate_coord == -1) { coord = lp_build_negate(coord_bld, coord); } res = lp_build_mul(coord_bld, coord, ima); if (sign) { sign = lp_build_broadcast_scalar(coord_bld, sign); res = lp_build_mul(coord_bld, res, sign); } res = lp_build_add(coord_bld, res, half); return res; }
/** * Build LLVM code for texture coord wrapping, for linear filtering, * for scaled integer texcoords. * \param block_length is the length of the pixel block along the * coordinate axis * \param coord0 the incoming texcoord (s,t,r or q) scaled to the texture size * \param length the texture size along one dimension * \param stride pixel stride along the coordinate axis (in bytes) * \param is_pot if TRUE, length is a power of two * \param wrap_mode one of PIPE_TEX_WRAP_x * \param offset0 resulting relative offset for coord0 * \param offset1 resulting relative offset for coord0 + 1 * \param i0 resulting sub-block pixel coordinate for coord0 * \param i1 resulting sub-block pixel coordinate for coord0 + 1 */ static void lp_build_sample_wrap_linear_int(struct lp_build_sample_context *bld, unsigned block_length, LLVMValueRef coord0, LLVMValueRef length, LLVMValueRef stride, boolean is_pot, unsigned wrap_mode, LLVMValueRef *offset0, LLVMValueRef *offset1, LLVMValueRef *i0, LLVMValueRef *i1) { struct lp_build_context *int_coord_bld = &bld->int_coord_bld; LLVMBuilderRef builder = bld->gallivm->builder; LLVMValueRef length_minus_one; LLVMValueRef lmask, umask, mask; if (block_length != 1) { /* * If the pixel block covers more than one pixel then there is no easy * way to calculate offset1 relative to offset0. Instead, compute them * independently. */ LLVMValueRef coord1; lp_build_sample_wrap_nearest_int(bld, block_length, coord0, length, stride, is_pot, wrap_mode, offset0, i0); coord1 = lp_build_add(int_coord_bld, coord0, int_coord_bld->one); lp_build_sample_wrap_nearest_int(bld, block_length, coord1, length, stride, is_pot, wrap_mode, offset1, i1); return; } /* * Scalar pixels -- try to compute offset0 and offset1 with a single stride * multiplication. */ *i0 = int_coord_bld->zero; *i1 = int_coord_bld->zero; length_minus_one = lp_build_sub(int_coord_bld, length, int_coord_bld->one); switch(wrap_mode) { case PIPE_TEX_WRAP_REPEAT: if (is_pot) { coord0 = LLVMBuildAnd(builder, coord0, length_minus_one, ""); } else { /* Add a bias to the texcoord to handle negative coords */ LLVMValueRef bias = lp_build_mul_imm(int_coord_bld, length, 1024); coord0 = LLVMBuildAdd(builder, coord0, bias, ""); coord0 = LLVMBuildURem(builder, coord0, length, ""); } mask = lp_build_compare(bld->gallivm, int_coord_bld->type, PIPE_FUNC_NOTEQUAL, coord0, length_minus_one); *offset0 = lp_build_mul(int_coord_bld, coord0, stride); *offset1 = LLVMBuildAnd(builder, lp_build_add(int_coord_bld, *offset0, stride), mask, ""); break; case PIPE_TEX_WRAP_CLAMP_TO_EDGE: lmask = lp_build_compare(int_coord_bld->gallivm, int_coord_bld->type, PIPE_FUNC_GEQUAL, coord0, int_coord_bld->zero); umask = lp_build_compare(int_coord_bld->gallivm, int_coord_bld->type, PIPE_FUNC_LESS, coord0, length_minus_one); coord0 = lp_build_select(int_coord_bld, lmask, coord0, int_coord_bld->zero); coord0 = lp_build_select(int_coord_bld, umask, coord0, length_minus_one); mask = LLVMBuildAnd(builder, lmask, umask, ""); *offset0 = lp_build_mul(int_coord_bld, coord0, stride); *offset1 = lp_build_add(int_coord_bld, *offset0, LLVMBuildAnd(builder, stride, mask, "")); break; case PIPE_TEX_WRAP_CLAMP: case PIPE_TEX_WRAP_CLAMP_TO_BORDER: case PIPE_TEX_WRAP_MIRROR_REPEAT: case PIPE_TEX_WRAP_MIRROR_CLAMP: case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_EDGE: case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_BORDER: default: assert(0); *offset0 = int_coord_bld->zero; *offset1 = int_coord_bld->zero; break; } }
/** * Convert linear float values to srgb int values. * Several possibilities how to do this, e.g. * - use table (based on exponent/highest order mantissa bits) and do * linear interpolation (https://gist.github.com/rygorous/2203834) * - Chebyshev polynomial * - Approximation using reciprocals * - using int-to-float and float-to-int tricks for pow() * (http://stackoverflow.com/questions/6475373/optimizations-for-pow-with-const-non-integer-exponent) * * @param src float (vector) value(s) to convert. */ static LLVMValueRef lp_build_linear_to_srgb(struct gallivm_state *gallivm, struct lp_type src_type, LLVMValueRef src) { LLVMBuilderRef builder = gallivm->builder; struct lp_build_context f32_bld; LLVMValueRef lin_thresh, lin, lin_const, is_linear, tmp, pow_final; lp_build_context_init(&f32_bld, gallivm, src_type); src = lp_build_clamp(&f32_bld, src, f32_bld.zero, f32_bld.one); if (0) { /* * using int-to-float and float-to-int trick for pow(). * This is much more accurate than necessary thanks to the correction, * but it most certainly makes no sense without rsqrt available. * Bonus points if you understand how this works... * All in all (including min/max clamp, conversion) 19 instructions. */ float exp_f = 2.0f / 3.0f; /* some compilers can't do exp2f, so this is exp2f(127.0f/exp_f - 127.0f) */ float exp2f_c = 1.30438178253e+19f; float coeff_f = 0.62996f; LLVMValueRef pow_approx, coeff, x2, exponent, pow_1, pow_2; struct lp_type int_type = lp_int_type(src_type); /* * First calculate approx x^8/12 */ exponent = lp_build_const_vec(gallivm, src_type, exp_f); coeff = lp_build_const_vec(gallivm, src_type, exp2f_c * powf(coeff_f, 1.0f / exp_f)); /* premultiply src */ tmp = lp_build_mul(&f32_bld, coeff, src); /* "log2" */ tmp = LLVMBuildBitCast(builder, tmp, lp_build_vec_type(gallivm, int_type), ""); tmp = lp_build_int_to_float(&f32_bld, tmp); /* multiply for pow */ tmp = lp_build_mul(&f32_bld, tmp, exponent); /* "exp2" */ pow_approx = lp_build_itrunc(&f32_bld, tmp); pow_approx = LLVMBuildBitCast(builder, pow_approx, lp_build_vec_type(gallivm, src_type), ""); /* * Since that pow was inaccurate (like 3 bits, though each sqrt step would * give another bit), compensate the error (which is why we chose another * exponent in the first place). */ /* x * x^(8/12) = x^(20/12) */ pow_1 = lp_build_mul(&f32_bld, pow_approx, src); /* x * x * x^(-4/12) = x^(20/12) */ /* Should avoid using rsqrt if it's not available, but * using x * x^(4/12) * x^(4/12) instead will change error weight */ tmp = lp_build_fast_rsqrt(&f32_bld, pow_approx); x2 = lp_build_mul(&f32_bld, src, src); pow_2 = lp_build_mul(&f32_bld, x2, tmp); /* average the values so the errors cancel out, compensate bias, * we also squeeze the 1.055 mul of the srgb conversion plus the 255.0 mul * for conversion to int in here */ tmp = lp_build_add(&f32_bld, pow_1, pow_2); coeff = lp_build_const_vec(gallivm, src_type, 1.0f / (3.0f * coeff_f) * 0.999852f * powf(1.055f * 255.0f, 4.0f)); pow_final = lp_build_mul(&f32_bld, tmp, coeff); /* x^(5/12) = rsqrt(rsqrt(x^20/12)) */ if (lp_build_fast_rsqrt_available(src_type)) { pow_final = lp_build_fast_rsqrt(&f32_bld, lp_build_fast_rsqrt(&f32_bld, pow_final)); } else { pow_final = lp_build_sqrt(&f32_bld, lp_build_sqrt(&f32_bld, pow_final)); } pow_final = lp_build_add(&f32_bld, pow_final, lp_build_const_vec(gallivm, src_type, -0.055f * 255.0f)); } else { /* * using "rational polynomial" approximation here. * Essentially y = a*x^0.375 + b*x^0.5 + c, with also * factoring in the 255.0 mul and the scaling mul. * (a is closer to actual value so has higher weight than b.) * Note: the constants are magic values. They were found empirically, * possibly could be improved but good enough (be VERY careful with * error metric if you'd want to tweak them, they also MUST fit with * the crappy polynomial above for srgb->linear since it is required * that each srgb value maps back to the same value). * This function has an error of max +-0.17 (and we'd only require +-0.6), * for the approximated srgb->linear values the error is naturally larger * (+-0.42) but still accurate enough (required +-0.5 essentially). * All in all (including min/max clamp, conversion) 15 instructions. * FMA would help (minus 2 instructions). */ LLVMValueRef x05, x0375, a_const, b_const, c_const, tmp2; if (lp_build_fast_rsqrt_available(src_type)) { tmp = lp_build_fast_rsqrt(&f32_bld, src); x05 = lp_build_mul(&f32_bld, src, tmp); } else { /* * I don't really expect this to be practical without rsqrt * but there's no reason for triple punishment so at least * save the otherwise resulting division and unnecessary mul... */ x05 = lp_build_sqrt(&f32_bld, src); } tmp = lp_build_mul(&f32_bld, x05, src); if (lp_build_fast_rsqrt_available(src_type)) { x0375 = lp_build_fast_rsqrt(&f32_bld, lp_build_fast_rsqrt(&f32_bld, tmp)); } else { x0375 = lp_build_sqrt(&f32_bld, lp_build_sqrt(&f32_bld, tmp)); } a_const = lp_build_const_vec(gallivm, src_type, 0.675f * 1.0622 * 255.0f); b_const = lp_build_const_vec(gallivm, src_type, 0.325f * 1.0622 * 255.0f); c_const = lp_build_const_vec(gallivm, src_type, -0.0620f * 255.0f); tmp = lp_build_mul(&f32_bld, a_const, x0375); tmp2 = lp_build_mul(&f32_bld, b_const, x05); tmp2 = lp_build_add(&f32_bld, tmp2, c_const); pow_final = lp_build_add(&f32_bld, tmp, tmp2); } /* linear part is easy */ lin_const = lp_build_const_vec(gallivm, src_type, 12.92f * 255.0f); lin = lp_build_mul(&f32_bld, src, lin_const); lin_thresh = lp_build_const_vec(gallivm, src_type, 0.0031308f); is_linear = lp_build_compare(gallivm, src_type, PIPE_FUNC_LEQUAL, src, lin_thresh); tmp = lp_build_select(&f32_bld, is_linear, lin, pow_final); f32_bld.type.sign = 0; return lp_build_iround(&f32_bld, tmp); }
/** * Increment the shader input attribute values. * This is called when we move from one quad to the next. */ static void attribs_update(struct lp_build_interp_soa_context *bld, struct gallivm_state *gallivm, LLVMValueRef loop_iter, int start, int end) { LLVMBuilderRef builder = gallivm->builder; struct lp_build_context *coeff_bld = &bld->coeff_bld; LLVMValueRef oow = NULL; unsigned attrib; unsigned chan; for(attrib = start; attrib < end; ++attrib) { const unsigned mask = bld->mask[attrib]; const unsigned interp = bld->interp[attrib]; for(chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) { if(mask & (1 << chan)) { LLVMValueRef a; if (interp == LP_INTERP_CONSTANT || interp == LP_INTERP_FACING) { a = LLVMBuildLoad(builder, bld->a[attrib][chan], ""); } else if (interp == LP_INTERP_POSITION) { assert(attrib > 0); a = bld->attribs[0][chan]; } else { LLVMValueRef dadq; a = bld->a[attrib][chan]; /* * Broadcast the attribute value for this quad into all elements */ { /* stored as vector load as float */ LLVMTypeRef ptr_type = LLVMPointerType(LLVMFloatTypeInContext( gallivm->context), 0); LLVMValueRef ptr; a = LLVMBuildBitCast(builder, a, ptr_type, ""); ptr = LLVMBuildGEP(builder, a, &loop_iter, 1, ""); a = LLVMBuildLoad(builder, ptr, ""); a = lp_build_broadcast_scalar(&bld->coeff_bld, a); } /* * Get the derivatives. */ dadq = bld->dadq[attrib][chan]; #if PERSPECTIVE_DIVIDE_PER_QUAD if (interp == LP_INTERP_PERSPECTIVE) { LLVMValueRef dwdq = bld->dadq[0][3]; if (oow == NULL) { assert(bld->oow); oow = LLVMBuildShuffleVector(coeff_bld->builder, bld->oow, coeff_bld->undef, shuffle, ""); } dadq = lp_build_sub(coeff_bld, dadq, lp_build_mul(coeff_bld, a, dwdq)); dadq = lp_build_mul(coeff_bld, dadq, oow); } #endif /* * Add the derivatives */ a = lp_build_add(coeff_bld, a, dadq); #if !PERSPECTIVE_DIVIDE_PER_QUAD if (interp == LP_INTERP_PERSPECTIVE) { if (oow == NULL) { LLVMValueRef w = bld->attribs[0][3]; assert(attrib != 0); assert(bld->mask[0] & TGSI_WRITEMASK_W); oow = lp_build_rcp(coeff_bld, w); } a = lp_build_mul(coeff_bld, a, oow); } #endif if (attrib == 0 && chan == 2) { /* FIXME: Depth values can exceed 1.0, due to the fact that * setup interpolation coefficients refer to (0,0) which causes * precision loss. So we must clamp to 1.0 here to avoid artifacts */ a = lp_build_min(coeff_bld, a, coeff_bld->one); } attrib_name(a, attrib, chan, ""); } bld->attribs[attrib][chan] = a; } } } }
/** * Sample a single texture image with nearest sampling. * If sampling a cube texture, r = cube face in [0,5]. * Return filtered color as two vectors of 16-bit fixed point values. */ static void lp_build_sample_image_nearest(struct lp_build_sample_context *bld, LLVMValueRef int_size, LLVMValueRef row_stride_vec, LLVMValueRef img_stride_vec, LLVMValueRef data_ptr, LLVMValueRef s, LLVMValueRef t, LLVMValueRef r, LLVMValueRef *colors_lo, LLVMValueRef *colors_hi) { const unsigned dims = bld->dims; LLVMBuilderRef builder = bld->gallivm->builder; struct lp_build_context i32, h16, u8n; LLVMTypeRef i32_vec_type, h16_vec_type, u8n_vec_type; LLVMValueRef i32_c8; LLVMValueRef width_vec, height_vec, depth_vec; LLVMValueRef s_ipart, t_ipart = NULL, r_ipart = NULL; LLVMValueRef x_stride; LLVMValueRef x_offset, offset; LLVMValueRef x_subcoord, y_subcoord, z_subcoord; lp_build_context_init(&i32, bld->gallivm, lp_type_int_vec(32)); lp_build_context_init(&h16, bld->gallivm, lp_type_ufixed(16)); lp_build_context_init(&u8n, bld->gallivm, lp_type_unorm(8)); i32_vec_type = lp_build_vec_type(bld->gallivm, i32.type); h16_vec_type = lp_build_vec_type(bld->gallivm, h16.type); u8n_vec_type = lp_build_vec_type(bld->gallivm, u8n.type); lp_build_extract_image_sizes(bld, bld->int_size_type, bld->int_coord_type, int_size, &width_vec, &height_vec, &depth_vec); if (bld->static_state->normalized_coords) { LLVMValueRef scaled_size; LLVMValueRef flt_size; /* scale size by 256 (8 fractional bits) */ scaled_size = lp_build_shl_imm(&bld->int_size_bld, int_size, 8); flt_size = lp_build_int_to_float(&bld->float_size_bld, scaled_size); lp_build_unnormalized_coords(bld, flt_size, &s, &t, &r); } else { /* scale coords by 256 (8 fractional bits) */ s = lp_build_mul_imm(&bld->coord_bld, s, 256); if (dims >= 2) t = lp_build_mul_imm(&bld->coord_bld, t, 256); if (dims >= 3) r = lp_build_mul_imm(&bld->coord_bld, r, 256); } /* convert float to int */ s = LLVMBuildFPToSI(builder, s, i32_vec_type, ""); if (dims >= 2) t = LLVMBuildFPToSI(builder, t, i32_vec_type, ""); if (dims >= 3) r = LLVMBuildFPToSI(builder, r, i32_vec_type, ""); /* compute floor (shift right 8) */ i32_c8 = lp_build_const_int_vec(bld->gallivm, i32.type, 8); s_ipart = LLVMBuildAShr(builder, s, i32_c8, ""); if (dims >= 2) t_ipart = LLVMBuildAShr(builder, t, i32_c8, ""); if (dims >= 3) r_ipart = LLVMBuildAShr(builder, r, i32_c8, ""); /* get pixel, row, image strides */ x_stride = lp_build_const_vec(bld->gallivm, bld->int_coord_bld.type, bld->format_desc->block.bits/8); /* Do texcoord wrapping, compute texel offset */ lp_build_sample_wrap_nearest_int(bld, bld->format_desc->block.width, s_ipart, width_vec, x_stride, bld->static_state->pot_width, bld->static_state->wrap_s, &x_offset, &x_subcoord); offset = x_offset; if (dims >= 2) { LLVMValueRef y_offset; lp_build_sample_wrap_nearest_int(bld, bld->format_desc->block.height, t_ipart, height_vec, row_stride_vec, bld->static_state->pot_height, bld->static_state->wrap_t, &y_offset, &y_subcoord); offset = lp_build_add(&bld->int_coord_bld, offset, y_offset); if (dims >= 3) { LLVMValueRef z_offset; lp_build_sample_wrap_nearest_int(bld, 1, /* block length (depth) */ r_ipart, depth_vec, img_stride_vec, bld->static_state->pot_height, bld->static_state->wrap_r, &z_offset, &z_subcoord); offset = lp_build_add(&bld->int_coord_bld, offset, z_offset); } else if (bld->static_state->target == PIPE_TEXTURE_CUBE) { LLVMValueRef z_offset; /* The r coord is the cube face in [0,5] */ z_offset = lp_build_mul(&bld->int_coord_bld, r, img_stride_vec); offset = lp_build_add(&bld->int_coord_bld, offset, z_offset); } } /* * Fetch the pixels as 4 x 32bit (rgba order might differ): * * rgba0 rgba1 rgba2 rgba3 * * bit cast them into 16 x u8 * * r0 g0 b0 a0 r1 g1 b1 a1 r2 g2 b2 a2 r3 g3 b3 a3 * * unpack them into two 8 x i16: * * r0 g0 b0 a0 r1 g1 b1 a1 * r2 g2 b2 a2 r3 g3 b3 a3 * * The higher 8 bits of the resulting elements will be zero. */ { LLVMValueRef rgba8; if (util_format_is_rgba8_variant(bld->format_desc)) { /* * Given the format is a rgba8, just read the pixels as is, * without any swizzling. Swizzling will be done later. */ rgba8 = lp_build_gather(bld->gallivm, bld->texel_type.length, bld->format_desc->block.bits, bld->texel_type.width, data_ptr, offset); rgba8 = LLVMBuildBitCast(builder, rgba8, u8n_vec_type, ""); } else { rgba8 = lp_build_fetch_rgba_aos(bld->gallivm, bld->format_desc, u8n.type, data_ptr, offset, x_subcoord, y_subcoord); } /* Expand one 4*rgba8 to two 2*rgba16 */ lp_build_unpack2(bld->gallivm, u8n.type, h16.type, rgba8, colors_lo, colors_hi); } }
/** * Emit LLVM for one TGSI instruction. * \param return TRUE for success, FALSE otherwise */ boolean lp_emit_instruction_aos( struct lp_build_tgsi_aos_context *bld, const struct tgsi_full_instruction *inst, const struct tgsi_opcode_info *info, int *pc) { LLVMValueRef src0, src1, src2; LLVMValueRef tmp0, tmp1; LLVMValueRef dst0 = NULL; /* * Stores and write masks are handled in a general fashion after the long * instruction opcode switch statement. * * Although not stricitly necessary, we avoid generating instructions for * channels which won't be stored, in cases where's that easy. For some * complex instructions, like texture sampling, it is more convenient to * assume a full writemask and then let LLVM optimization passes eliminate * redundant code. */ (*pc)++; assert(info->num_dst <= 1); if (info->num_dst) { dst0 = bld->bld_base.base.undef; } switch (inst->Instruction.Opcode) { case TGSI_OPCODE_ARL: src0 = lp_build_emit_fetch(&bld->bld_base, inst, 0, LP_CHAN_ALL); dst0 = lp_build_floor(&bld->bld_base.base, src0); break; case TGSI_OPCODE_MOV: dst0 = lp_build_emit_fetch(&bld->bld_base, inst, 0, LP_CHAN_ALL); break; case TGSI_OPCODE_LIT: return FALSE; case TGSI_OPCODE_RCP: /* TGSI_OPCODE_RECIP */ src0 = lp_build_emit_fetch(&bld->bld_base, inst, 0, LP_CHAN_ALL); dst0 = lp_build_rcp(&bld->bld_base.base, src0); break; case TGSI_OPCODE_RSQ: /* TGSI_OPCODE_RECIPSQRT */ src0 = lp_build_emit_fetch(&bld->bld_base, inst, 0, LP_CHAN_ALL); tmp0 = lp_build_emit_llvm_unary(&bld->bld_base, TGSI_OPCODE_ABS, src0); dst0 = lp_build_rsqrt(&bld->bld_base.base, tmp0); break; case TGSI_OPCODE_EXP: return FALSE; case TGSI_OPCODE_LOG: return FALSE; case TGSI_OPCODE_MUL: src0 = lp_build_emit_fetch(&bld->bld_base, inst, 0, LP_CHAN_ALL); src1 = lp_build_emit_fetch(&bld->bld_base, inst, 1, LP_CHAN_ALL); dst0 = lp_build_mul(&bld->bld_base.base, src0, src1); break; case TGSI_OPCODE_ADD: src0 = lp_build_emit_fetch(&bld->bld_base, inst, 0, LP_CHAN_ALL); src1 = lp_build_emit_fetch(&bld->bld_base, inst, 1, LP_CHAN_ALL); dst0 = lp_build_add(&bld->bld_base.base, src0, src1); break; case TGSI_OPCODE_DP3: /* TGSI_OPCODE_DOT3 */ return FALSE; case TGSI_OPCODE_DP4: /* TGSI_OPCODE_DOT4 */ return FALSE; case TGSI_OPCODE_DST: return FALSE; case TGSI_OPCODE_MIN: src0 = lp_build_emit_fetch(&bld->bld_base, inst, 0, LP_CHAN_ALL); src1 = lp_build_emit_fetch(&bld->bld_base, inst, 1, LP_CHAN_ALL); dst0 = lp_build_max(&bld->bld_base.base, src0, src1); break; case TGSI_OPCODE_MAX: src0 = lp_build_emit_fetch(&bld->bld_base, inst, 0, LP_CHAN_ALL); src1 = lp_build_emit_fetch(&bld->bld_base, inst, 1, LP_CHAN_ALL); dst0 = lp_build_max(&bld->bld_base.base, src0, src1); break; case TGSI_OPCODE_SLT: /* TGSI_OPCODE_SETLT */ src0 = lp_build_emit_fetch(&bld->bld_base, inst, 0, LP_CHAN_ALL); src1 = lp_build_emit_fetch(&bld->bld_base, inst, 1, LP_CHAN_ALL); tmp0 = lp_build_cmp(&bld->bld_base.base, PIPE_FUNC_LESS, src0, src1); dst0 = lp_build_select(&bld->bld_base.base, tmp0, bld->bld_base.base.one, bld->bld_base.base.zero); break; case TGSI_OPCODE_SGE: /* TGSI_OPCODE_SETGE */ src0 = lp_build_emit_fetch(&bld->bld_base, inst, 0, LP_CHAN_ALL); src1 = lp_build_emit_fetch(&bld->bld_base, inst, 1, LP_CHAN_ALL); tmp0 = lp_build_cmp(&bld->bld_base.base, PIPE_FUNC_GEQUAL, src0, src1); dst0 = lp_build_select(&bld->bld_base.base, tmp0, bld->bld_base.base.one, bld->bld_base.base.zero); break; case TGSI_OPCODE_MAD: /* TGSI_OPCODE_MADD */ src0 = lp_build_emit_fetch(&bld->bld_base, inst, 0, LP_CHAN_ALL); src1 = lp_build_emit_fetch(&bld->bld_base, inst, 1, LP_CHAN_ALL); src2 = lp_build_emit_fetch(&bld->bld_base, inst, 2, LP_CHAN_ALL); tmp0 = lp_build_mul(&bld->bld_base.base, src0, src1); dst0 = lp_build_add(&bld->bld_base.base, tmp0, src2); break; case TGSI_OPCODE_SUB: src0 = lp_build_emit_fetch(&bld->bld_base, inst, 0, LP_CHAN_ALL); src1 = lp_build_emit_fetch(&bld->bld_base, inst, 1, LP_CHAN_ALL); dst0 = lp_build_sub(&bld->bld_base.base, src0, src1); break; case TGSI_OPCODE_LRP: src0 = lp_build_emit_fetch(&bld->bld_base, inst, 0, LP_CHAN_ALL); src1 = lp_build_emit_fetch(&bld->bld_base, inst, 1, LP_CHAN_ALL); src2 = lp_build_emit_fetch(&bld->bld_base, inst, 2, LP_CHAN_ALL); tmp0 = lp_build_sub(&bld->bld_base.base, src1, src2); tmp0 = lp_build_mul(&bld->bld_base.base, src0, tmp0); dst0 = lp_build_add(&bld->bld_base.base, tmp0, src2); break; case TGSI_OPCODE_CND: src0 = lp_build_emit_fetch(&bld->bld_base, inst, 0, LP_CHAN_ALL); src1 = lp_build_emit_fetch(&bld->bld_base, inst, 1, LP_CHAN_ALL); src2 = lp_build_emit_fetch(&bld->bld_base, inst, 2, LP_CHAN_ALL); tmp1 = lp_build_const_vec(bld->bld_base.base.gallivm, bld->bld_base.base.type, 0.5); tmp0 = lp_build_cmp(&bld->bld_base.base, PIPE_FUNC_GREATER, src2, tmp1); dst0 = lp_build_select(&bld->bld_base.base, tmp0, src0, src1); break; case TGSI_OPCODE_DP2A: return FALSE; case TGSI_OPCODE_FRC: src0 = lp_build_emit_fetch(&bld->bld_base, inst, 0, LP_CHAN_ALL); tmp0 = lp_build_floor(&bld->bld_base.base, src0); dst0 = lp_build_sub(&bld->bld_base.base, src0, tmp0); break; case TGSI_OPCODE_CLAMP: src0 = lp_build_emit_fetch(&bld->bld_base, inst, 0, LP_CHAN_ALL); src1 = lp_build_emit_fetch(&bld->bld_base, inst, 1, LP_CHAN_ALL); src2 = lp_build_emit_fetch(&bld->bld_base, inst, 2, LP_CHAN_ALL); tmp0 = lp_build_max(&bld->bld_base.base, src0, src1); dst0 = lp_build_min(&bld->bld_base.base, tmp0, src2); break; case TGSI_OPCODE_FLR: src0 = lp_build_emit_fetch(&bld->bld_base, inst, 0, LP_CHAN_ALL); dst0 = lp_build_floor(&bld->bld_base.base, src0); break; case TGSI_OPCODE_ROUND: src0 = lp_build_emit_fetch(&bld->bld_base, inst, 0, LP_CHAN_ALL); dst0 = lp_build_round(&bld->bld_base.base, src0); break; case TGSI_OPCODE_EX2: src0 = lp_build_emit_fetch(&bld->bld_base, inst, 0, LP_CHAN_ALL); tmp0 = lp_build_swizzle_scalar_aos(&bld->bld_base.base, src0, TGSI_SWIZZLE_X, TGSI_NUM_CHANNELS); dst0 = lp_build_exp2(&bld->bld_base.base, tmp0); break; case TGSI_OPCODE_LG2: src0 = lp_build_emit_fetch(&bld->bld_base, inst, 0, LP_CHAN_ALL); tmp0 = swizzle_scalar_aos(bld, src0, TGSI_SWIZZLE_X); dst0 = lp_build_log2(&bld->bld_base.base, tmp0); break; case TGSI_OPCODE_POW: src0 = lp_build_emit_fetch(&bld->bld_base, inst, 0, LP_CHAN_ALL); src0 = swizzle_scalar_aos(bld, src0, TGSI_SWIZZLE_X); src1 = lp_build_emit_fetch(&bld->bld_base, inst, 1, LP_CHAN_ALL); src1 = swizzle_scalar_aos(bld, src1, TGSI_SWIZZLE_X); dst0 = lp_build_pow(&bld->bld_base.base, src0, src1); break; case TGSI_OPCODE_XPD: return FALSE; case TGSI_OPCODE_RCC: /* deprecated? */ assert(0); return FALSE; case TGSI_OPCODE_DPH: return FALSE; case TGSI_OPCODE_COS: src0 = lp_build_emit_fetch(&bld->bld_base, inst, 0, LP_CHAN_ALL); tmp0 = swizzle_scalar_aos(bld, src0, TGSI_SWIZZLE_X); dst0 = lp_build_cos(&bld->bld_base.base, tmp0); break; case TGSI_OPCODE_DDX: return FALSE; case TGSI_OPCODE_DDY: return FALSE; case TGSI_OPCODE_KILP: /* predicated kill */ return FALSE; case TGSI_OPCODE_KIL: /* conditional kill */ return FALSE; case TGSI_OPCODE_PK2H: return FALSE; break; case TGSI_OPCODE_PK2US: return FALSE; break; case TGSI_OPCODE_PK4B: return FALSE; break; case TGSI_OPCODE_PK4UB: return FALSE; case TGSI_OPCODE_RFL: return FALSE; case TGSI_OPCODE_SEQ: src0 = lp_build_emit_fetch(&bld->bld_base, inst, 0, LP_CHAN_ALL); src1 = lp_build_emit_fetch(&bld->bld_base, inst, 1, LP_CHAN_ALL); tmp0 = lp_build_cmp(&bld->bld_base.base, PIPE_FUNC_EQUAL, src0, src1); dst0 = lp_build_select(&bld->bld_base.base, tmp0, bld->bld_base.base.one, bld->bld_base.base.zero); break; case TGSI_OPCODE_SFL: dst0 = bld->bld_base.base.zero; break; case TGSI_OPCODE_SGT: src0 = lp_build_emit_fetch(&bld->bld_base, inst, 0, LP_CHAN_ALL); src1 = lp_build_emit_fetch(&bld->bld_base, inst, 1, LP_CHAN_ALL); tmp0 = lp_build_cmp(&bld->bld_base.base, PIPE_FUNC_GREATER, src0, src1); dst0 = lp_build_select(&bld->bld_base.base, tmp0, bld->bld_base.base.one, bld->bld_base.base.zero); break; case TGSI_OPCODE_SIN: src0 = lp_build_emit_fetch(&bld->bld_base, inst, 0, LP_CHAN_ALL); tmp0 = swizzle_scalar_aos(bld, src0, TGSI_SWIZZLE_X); dst0 = lp_build_sin(&bld->bld_base.base, tmp0); break; case TGSI_OPCODE_SLE: src0 = lp_build_emit_fetch(&bld->bld_base, inst, 0, LP_CHAN_ALL); src1 = lp_build_emit_fetch(&bld->bld_base, inst, 1, LP_CHAN_ALL); tmp0 = lp_build_cmp(&bld->bld_base.base, PIPE_FUNC_LEQUAL, src0, src1); dst0 = lp_build_select(&bld->bld_base.base, tmp0, bld->bld_base.base.one, bld->bld_base.base.zero); break; case TGSI_OPCODE_SNE: src0 = lp_build_emit_fetch(&bld->bld_base, inst, 0, LP_CHAN_ALL); src1 = lp_build_emit_fetch(&bld->bld_base, inst, 1, LP_CHAN_ALL); tmp0 = lp_build_cmp(&bld->bld_base.base, PIPE_FUNC_NOTEQUAL, src0, src1); dst0 = lp_build_select(&bld->bld_base.base, tmp0, bld->bld_base.base.one, bld->bld_base.base.zero); break; case TGSI_OPCODE_STR: dst0 = bld->bld_base.base.one; break; case TGSI_OPCODE_TEX: dst0 = emit_tex(bld, inst, LP_BLD_TEX_MODIFIER_NONE); break; case TGSI_OPCODE_TXD: dst0 = emit_tex(bld, inst, LP_BLD_TEX_MODIFIER_EXPLICIT_DERIV); break; case TGSI_OPCODE_UP2H: /* deprecated */ assert (0); return FALSE; break; case TGSI_OPCODE_UP2US: /* deprecated */ assert(0); return FALSE; break; case TGSI_OPCODE_UP4B: /* deprecated */ assert(0); return FALSE; break; case TGSI_OPCODE_UP4UB: /* deprecated */ assert(0); return FALSE; break; case TGSI_OPCODE_X2D: /* deprecated? */ assert(0); return FALSE; break; case TGSI_OPCODE_ARA: /* deprecated */ assert(0); return FALSE; break; case TGSI_OPCODE_ARR: src0 = lp_build_emit_fetch(&bld->bld_base, inst, 0, LP_CHAN_ALL); dst0 = lp_build_round(&bld->bld_base.base, src0); break; case TGSI_OPCODE_BRA: /* deprecated */ assert(0); return FALSE; break; case TGSI_OPCODE_CAL: return FALSE; case TGSI_OPCODE_RET: return FALSE; case TGSI_OPCODE_END: *pc = -1; break; case TGSI_OPCODE_SSG: /* TGSI_OPCODE_SGN */ tmp0 = lp_build_emit_fetch(&bld->bld_base, inst, 0, LP_CHAN_ALL); dst0 = lp_build_sgn(&bld->bld_base.base, tmp0); break; case TGSI_OPCODE_CMP: src0 = lp_build_emit_fetch(&bld->bld_base, inst, 0, LP_CHAN_ALL); src1 = lp_build_emit_fetch(&bld->bld_base, inst, 1, LP_CHAN_ALL); src2 = lp_build_emit_fetch(&bld->bld_base, inst, 2, LP_CHAN_ALL); tmp0 = lp_build_cmp(&bld->bld_base.base, PIPE_FUNC_LESS, src0, bld->bld_base.base.zero); dst0 = lp_build_select(&bld->bld_base.base, tmp0, src1, src2); break; case TGSI_OPCODE_SCS: return FALSE; case TGSI_OPCODE_TXB: dst0 = emit_tex(bld, inst, LP_BLD_TEX_MODIFIER_LOD_BIAS); break; case TGSI_OPCODE_NRM: /* fall-through */ case TGSI_OPCODE_NRM4: return FALSE; case TGSI_OPCODE_DIV: /* deprecated */ assert(0); return FALSE; break; case TGSI_OPCODE_DP2: return FALSE; case TGSI_OPCODE_TXL: dst0 = emit_tex(bld, inst, LP_BLD_TEX_MODIFIER_EXPLICIT_LOD); break; case TGSI_OPCODE_TXP: dst0 = emit_tex(bld, inst, LP_BLD_TEX_MODIFIER_PROJECTED); break; case TGSI_OPCODE_BRK: return FALSE; case TGSI_OPCODE_IF: return FALSE; case TGSI_OPCODE_BGNLOOP: return FALSE; case TGSI_OPCODE_BGNSUB: return FALSE; case TGSI_OPCODE_ELSE: return FALSE; case TGSI_OPCODE_ENDIF: return FALSE; case TGSI_OPCODE_ENDLOOP: return FALSE; case TGSI_OPCODE_ENDSUB: return FALSE; case TGSI_OPCODE_PUSHA: /* deprecated? */ assert(0); return FALSE; break; case TGSI_OPCODE_POPA: /* deprecated? */ assert(0); return FALSE; break; case TGSI_OPCODE_CEIL: src0 = lp_build_emit_fetch(&bld->bld_base, inst, 0, LP_CHAN_ALL); dst0 = lp_build_ceil(&bld->bld_base.base, src0); break; case TGSI_OPCODE_I2F: /* deprecated? */ assert(0); return FALSE; break; case TGSI_OPCODE_NOT: /* deprecated? */ assert(0); return FALSE; break; case TGSI_OPCODE_TRUNC: src0 = lp_build_emit_fetch(&bld->bld_base, inst, 0, LP_CHAN_ALL); dst0 = lp_build_trunc(&bld->bld_base.base, src0); break; case TGSI_OPCODE_SHL: /* deprecated? */ assert(0); return FALSE; break; case TGSI_OPCODE_ISHR: /* deprecated? */ assert(0); return FALSE; break; case TGSI_OPCODE_AND: /* deprecated? */ assert(0); return FALSE; break; case TGSI_OPCODE_OR: /* deprecated? */ assert(0); return FALSE; break; case TGSI_OPCODE_MOD: /* deprecated? */ assert(0); return FALSE; break; case TGSI_OPCODE_XOR: /* deprecated? */ assert(0); return FALSE; break; case TGSI_OPCODE_SAD: /* deprecated? */ assert(0); return FALSE; break; case TGSI_OPCODE_TXF: /* deprecated? */ assert(0); return FALSE; break; case TGSI_OPCODE_TXQ: /* deprecated? */ assert(0); return FALSE; break; case TGSI_OPCODE_CONT: return FALSE; case TGSI_OPCODE_EMIT: return FALSE; break; case TGSI_OPCODE_ENDPRIM: return FALSE; break; case TGSI_OPCODE_NOP: break; default: return FALSE; } if (info->num_dst) { lp_emit_store_aos(bld, inst, 0, dst0); } return TRUE; }
/** * Sample a single texture image with (bi-)(tri-)linear sampling. * Return filtered color as two vectors of 16-bit fixed point values. */ static void lp_build_sample_image_linear(struct lp_build_sample_context *bld, LLVMValueRef int_size, LLVMValueRef row_stride_vec, LLVMValueRef img_stride_vec, LLVMValueRef data_ptr, LLVMValueRef s, LLVMValueRef t, LLVMValueRef r, LLVMValueRef *colors_lo, LLVMValueRef *colors_hi) { const unsigned dims = bld->dims; LLVMBuilderRef builder = bld->gallivm->builder; struct lp_build_context i32, h16, u8n; LLVMTypeRef i32_vec_type, h16_vec_type, u8n_vec_type; LLVMValueRef i32_c8, i32_c128, i32_c255; LLVMValueRef width_vec, height_vec, depth_vec; LLVMValueRef s_ipart, s_fpart, s_fpart_lo, s_fpart_hi; LLVMValueRef t_ipart = NULL, t_fpart = NULL, t_fpart_lo = NULL, t_fpart_hi = NULL; LLVMValueRef r_ipart = NULL, r_fpart = NULL, r_fpart_lo = NULL, r_fpart_hi = NULL; LLVMValueRef x_stride, y_stride, z_stride; LLVMValueRef x_offset0, x_offset1; LLVMValueRef y_offset0, y_offset1; LLVMValueRef z_offset0, z_offset1; LLVMValueRef offset[2][2][2]; /* [z][y][x] */ LLVMValueRef x_subcoord[2], y_subcoord[2], z_subcoord[2]; LLVMValueRef neighbors_lo[2][2][2]; /* [z][y][x] */ LLVMValueRef neighbors_hi[2][2][2]; /* [z][y][x] */ LLVMValueRef packed_lo, packed_hi; unsigned x, y, z; unsigned i, j, k; unsigned numj, numk; lp_build_context_init(&i32, bld->gallivm, lp_type_int_vec(32)); lp_build_context_init(&h16, bld->gallivm, lp_type_ufixed(16)); lp_build_context_init(&u8n, bld->gallivm, lp_type_unorm(8)); i32_vec_type = lp_build_vec_type(bld->gallivm, i32.type); h16_vec_type = lp_build_vec_type(bld->gallivm, h16.type); u8n_vec_type = lp_build_vec_type(bld->gallivm, u8n.type); lp_build_extract_image_sizes(bld, bld->int_size_type, bld->int_coord_type, int_size, &width_vec, &height_vec, &depth_vec); if (bld->static_state->normalized_coords) { LLVMValueRef scaled_size; LLVMValueRef flt_size; /* scale size by 256 (8 fractional bits) */ scaled_size = lp_build_shl_imm(&bld->int_size_bld, int_size, 8); flt_size = lp_build_int_to_float(&bld->float_size_bld, scaled_size); lp_build_unnormalized_coords(bld, flt_size, &s, &t, &r); } else { /* scale coords by 256 (8 fractional bits) */ s = lp_build_mul_imm(&bld->coord_bld, s, 256); if (dims >= 2) t = lp_build_mul_imm(&bld->coord_bld, t, 256); if (dims >= 3) r = lp_build_mul_imm(&bld->coord_bld, r, 256); } /* convert float to int */ s = LLVMBuildFPToSI(builder, s, i32_vec_type, ""); if (dims >= 2) t = LLVMBuildFPToSI(builder, t, i32_vec_type, ""); if (dims >= 3) r = LLVMBuildFPToSI(builder, r, i32_vec_type, ""); /* subtract 0.5 (add -128) */ i32_c128 = lp_build_const_int_vec(bld->gallivm, i32.type, -128); s = LLVMBuildAdd(builder, s, i32_c128, ""); if (dims >= 2) { t = LLVMBuildAdd(builder, t, i32_c128, ""); } if (dims >= 3) { r = LLVMBuildAdd(builder, r, i32_c128, ""); } /* compute floor (shift right 8) */ i32_c8 = lp_build_const_int_vec(bld->gallivm, i32.type, 8); s_ipart = LLVMBuildAShr(builder, s, i32_c8, ""); if (dims >= 2) t_ipart = LLVMBuildAShr(builder, t, i32_c8, ""); if (dims >= 3) r_ipart = LLVMBuildAShr(builder, r, i32_c8, ""); /* compute fractional part (AND with 0xff) */ i32_c255 = lp_build_const_int_vec(bld->gallivm, i32.type, 255); s_fpart = LLVMBuildAnd(builder, s, i32_c255, ""); if (dims >= 2) t_fpart = LLVMBuildAnd(builder, t, i32_c255, ""); if (dims >= 3) r_fpart = LLVMBuildAnd(builder, r, i32_c255, ""); /* get pixel, row and image strides */ x_stride = lp_build_const_vec(bld->gallivm, bld->int_coord_bld.type, bld->format_desc->block.bits/8); y_stride = row_stride_vec; z_stride = img_stride_vec; /* do texcoord wrapping and compute texel offsets */ lp_build_sample_wrap_linear_int(bld, bld->format_desc->block.width, s_ipart, width_vec, x_stride, bld->static_state->pot_width, bld->static_state->wrap_s, &x_offset0, &x_offset1, &x_subcoord[0], &x_subcoord[1]); for (z = 0; z < 2; z++) { for (y = 0; y < 2; y++) { offset[z][y][0] = x_offset0; offset[z][y][1] = x_offset1; } } if (dims >= 2) { lp_build_sample_wrap_linear_int(bld, bld->format_desc->block.height, t_ipart, height_vec, y_stride, bld->static_state->pot_height, bld->static_state->wrap_t, &y_offset0, &y_offset1, &y_subcoord[0], &y_subcoord[1]); for (z = 0; z < 2; z++) { for (x = 0; x < 2; x++) { offset[z][0][x] = lp_build_add(&bld->int_coord_bld, offset[z][0][x], y_offset0); offset[z][1][x] = lp_build_add(&bld->int_coord_bld, offset[z][1][x], y_offset1); } } } if (dims >= 3) { lp_build_sample_wrap_linear_int(bld, bld->format_desc->block.height, r_ipart, depth_vec, z_stride, bld->static_state->pot_depth, bld->static_state->wrap_r, &z_offset0, &z_offset1, &z_subcoord[0], &z_subcoord[1]); for (y = 0; y < 2; y++) { for (x = 0; x < 2; x++) { offset[0][y][x] = lp_build_add(&bld->int_coord_bld, offset[0][y][x], z_offset0); offset[1][y][x] = lp_build_add(&bld->int_coord_bld, offset[1][y][x], z_offset1); } } } else if (bld->static_state->target == PIPE_TEXTURE_CUBE) { LLVMValueRef z_offset; z_offset = lp_build_mul(&bld->int_coord_bld, r, img_stride_vec); for (y = 0; y < 2; y++) { for (x = 0; x < 2; x++) { /* The r coord is the cube face in [0,5] */ offset[0][y][x] = lp_build_add(&bld->int_coord_bld, offset[0][y][x], z_offset); } } } /* * Transform 4 x i32 in * * s_fpart = {s0, s1, s2, s3} * * into 8 x i16 * * s_fpart = {00, s0, 00, s1, 00, s2, 00, s3} * * into two 8 x i16 * * s_fpart_lo = {s0, s0, s0, s0, s1, s1, s1, s1} * s_fpart_hi = {s2, s2, s2, s2, s3, s3, s3, s3} * * and likewise for t_fpart. There is no risk of loosing precision here * since the fractional parts only use the lower 8bits. */ s_fpart = LLVMBuildBitCast(builder, s_fpart, h16_vec_type, ""); if (dims >= 2) t_fpart = LLVMBuildBitCast(builder, t_fpart, h16_vec_type, ""); if (dims >= 3) r_fpart = LLVMBuildBitCast(builder, r_fpart, h16_vec_type, ""); { LLVMTypeRef elem_type = LLVMInt32TypeInContext(bld->gallivm->context); LLVMValueRef shuffles_lo[LP_MAX_VECTOR_LENGTH]; LLVMValueRef shuffles_hi[LP_MAX_VECTOR_LENGTH]; LLVMValueRef shuffle_lo; LLVMValueRef shuffle_hi; for (j = 0; j < h16.type.length; j += 4) { #ifdef PIPE_ARCH_LITTLE_ENDIAN unsigned subindex = 0; #else unsigned subindex = 1; #endif LLVMValueRef index; index = LLVMConstInt(elem_type, j/2 + subindex, 0); for (i = 0; i < 4; ++i) shuffles_lo[j + i] = index; index = LLVMConstInt(elem_type, h16.type.length/2 + j/2 + subindex, 0); for (i = 0; i < 4; ++i) shuffles_hi[j + i] = index; } shuffle_lo = LLVMConstVector(shuffles_lo, h16.type.length); shuffle_hi = LLVMConstVector(shuffles_hi, h16.type.length); s_fpart_lo = LLVMBuildShuffleVector(builder, s_fpart, h16.undef, shuffle_lo, ""); s_fpart_hi = LLVMBuildShuffleVector(builder, s_fpart, h16.undef, shuffle_hi, ""); if (dims >= 2) { t_fpart_lo = LLVMBuildShuffleVector(builder, t_fpart, h16.undef, shuffle_lo, ""); t_fpart_hi = LLVMBuildShuffleVector(builder, t_fpart, h16.undef, shuffle_hi, ""); } if (dims >= 3) { r_fpart_lo = LLVMBuildShuffleVector(builder, r_fpart, h16.undef, shuffle_lo, ""); r_fpart_hi = LLVMBuildShuffleVector(builder, r_fpart, h16.undef, shuffle_hi, ""); } } /* * Fetch the pixels as 4 x 32bit (rgba order might differ): * * rgba0 rgba1 rgba2 rgba3 * * bit cast them into 16 x u8 * * r0 g0 b0 a0 r1 g1 b1 a1 r2 g2 b2 a2 r3 g3 b3 a3 * * unpack them into two 8 x i16: * * r0 g0 b0 a0 r1 g1 b1 a1 * r2 g2 b2 a2 r3 g3 b3 a3 * * The higher 8 bits of the resulting elements will be zero. */ numj = 1 + (dims >= 2); numk = 1 + (dims >= 3); for (k = 0; k < numk; k++) { for (j = 0; j < numj; j++) { for (i = 0; i < 2; i++) { LLVMValueRef rgba8; if (util_format_is_rgba8_variant(bld->format_desc)) { /* * Given the format is a rgba8, just read the pixels as is, * without any swizzling. Swizzling will be done later. */ rgba8 = lp_build_gather(bld->gallivm, bld->texel_type.length, bld->format_desc->block.bits, bld->texel_type.width, data_ptr, offset[k][j][i]); rgba8 = LLVMBuildBitCast(builder, rgba8, u8n_vec_type, ""); } else { rgba8 = lp_build_fetch_rgba_aos(bld->gallivm, bld->format_desc, u8n.type, data_ptr, offset[k][j][i], x_subcoord[i], y_subcoord[j]); } /* Expand one 4*rgba8 to two 2*rgba16 */ lp_build_unpack2(bld->gallivm, u8n.type, h16.type, rgba8, &neighbors_lo[k][j][i], &neighbors_hi[k][j][i]); } } } /* * Linear interpolation with 8.8 fixed point. */ if (dims == 1) { /* 1-D lerp */ packed_lo = lp_build_lerp(&h16, s_fpart_lo, neighbors_lo[0][0][0], neighbors_lo[0][0][1]); packed_hi = lp_build_lerp(&h16, s_fpart_hi, neighbors_hi[0][0][0], neighbors_hi[0][0][1]); } else { /* 2-D lerp */ packed_lo = lp_build_lerp_2d(&h16, s_fpart_lo, t_fpart_lo, neighbors_lo[0][0][0], neighbors_lo[0][0][1], neighbors_lo[0][1][0], neighbors_lo[0][1][1]); packed_hi = lp_build_lerp_2d(&h16, s_fpart_hi, t_fpart_hi, neighbors_hi[0][0][0], neighbors_hi[0][0][1], neighbors_hi[0][1][0], neighbors_hi[0][1][1]); if (dims >= 3) { LLVMValueRef packed_lo2, packed_hi2; /* lerp in the second z slice */ packed_lo2 = lp_build_lerp_2d(&h16, s_fpart_lo, t_fpart_lo, neighbors_lo[1][0][0], neighbors_lo[1][0][1], neighbors_lo[1][1][0], neighbors_lo[1][1][1]); packed_hi2 = lp_build_lerp_2d(&h16, s_fpart_hi, t_fpart_hi, neighbors_hi[1][0][0], neighbors_hi[1][0][1], neighbors_hi[1][1][0], neighbors_hi[1][1][1]); /* interp between two z slices */ packed_lo = lp_build_lerp(&h16, r_fpart_lo, packed_lo, packed_lo2); packed_hi = lp_build_lerp(&h16, r_fpart_hi, packed_hi, packed_hi2); } } *colors_lo = packed_lo; *colors_hi = packed_hi; }
/** * Apply the stencil operator (add/sub/keep/etc) to the given vector * of stencil values. * \return new stencil values vector */ static LLVMValueRef lp_build_stencil_op_single(struct lp_build_context *bld, const struct pipe_stencil_state *stencil, enum stencil_op op, LLVMValueRef stencilRef, LLVMValueRef stencilVals) { LLVMBuilderRef builder = bld->gallivm->builder; struct lp_type type = bld->type; LLVMValueRef res; LLVMValueRef max = lp_build_const_int_vec(bld->gallivm, type, 0xff); unsigned stencil_op; assert(type.sign); switch (op) { case S_FAIL_OP: stencil_op = stencil->fail_op; break; case Z_FAIL_OP: stencil_op = stencil->zfail_op; break; case Z_PASS_OP: stencil_op = stencil->zpass_op; break; default: assert(0 && "Invalid stencil_op mode"); stencil_op = PIPE_STENCIL_OP_KEEP; } switch (stencil_op) { case PIPE_STENCIL_OP_KEEP: res = stencilVals; /* we can return early for this case */ return res; case PIPE_STENCIL_OP_ZERO: res = bld->zero; break; case PIPE_STENCIL_OP_REPLACE: res = stencilRef; break; case PIPE_STENCIL_OP_INCR: res = lp_build_add(bld, stencilVals, bld->one); res = lp_build_min(bld, res, max); break; case PIPE_STENCIL_OP_DECR: res = lp_build_sub(bld, stencilVals, bld->one); res = lp_build_max(bld, res, bld->zero); break; case PIPE_STENCIL_OP_INCR_WRAP: res = lp_build_add(bld, stencilVals, bld->one); res = LLVMBuildAnd(builder, res, max, ""); break; case PIPE_STENCIL_OP_DECR_WRAP: res = lp_build_sub(bld, stencilVals, bld->one); res = LLVMBuildAnd(builder, res, max, ""); break; case PIPE_STENCIL_OP_INVERT: res = LLVMBuildNot(builder, stencilVals, ""); res = LLVMBuildAnd(builder, res, max, ""); break; default: assert(0 && "bad stencil op mode"); res = bld->undef; } return res; }
/** * Apply the stencil operator (add/sub/keep/etc) to the given vector * of stencil values. * \return new stencil values vector */ static LLVMValueRef lp_build_stencil_op_single(struct lp_build_context *bld, const struct pipe_stencil_state *stencil, enum stencil_op op, LLVMValueRef stencilRef, LLVMValueRef stencilVals, LLVMValueRef mask) { const unsigned stencilMax = 255; /* XXX fix */ struct lp_type type = bld->type; LLVMValueRef res; LLVMValueRef max = lp_build_const_int_vec(type, stencilMax); unsigned stencil_op; assert(type.sign); switch (op) { case S_FAIL_OP: stencil_op = stencil->fail_op; break; case Z_FAIL_OP: stencil_op = stencil->zfail_op; break; case Z_PASS_OP: stencil_op = stencil->zpass_op; break; default: assert(0 && "Invalid stencil_op mode"); stencil_op = PIPE_STENCIL_OP_KEEP; } switch (stencil_op) { case PIPE_STENCIL_OP_KEEP: res = stencilVals; /* we can return early for this case */ return res; case PIPE_STENCIL_OP_ZERO: res = bld->zero; break; case PIPE_STENCIL_OP_REPLACE: res = stencilRef; break; case PIPE_STENCIL_OP_INCR: res = lp_build_add(bld, stencilVals, bld->one); res = lp_build_min(bld, res, max); break; case PIPE_STENCIL_OP_DECR: res = lp_build_sub(bld, stencilVals, bld->one); res = lp_build_max(bld, res, bld->zero); break; case PIPE_STENCIL_OP_INCR_WRAP: res = lp_build_add(bld, stencilVals, bld->one); res = LLVMBuildAnd(bld->builder, res, max, ""); break; case PIPE_STENCIL_OP_DECR_WRAP: res = lp_build_sub(bld, stencilVals, bld->one); res = LLVMBuildAnd(bld->builder, res, max, ""); break; case PIPE_STENCIL_OP_INVERT: res = LLVMBuildNot(bld->builder, stencilVals, ""); res = LLVMBuildAnd(bld->builder, res, max, ""); break; default: assert(0 && "bad stencil op mode"); res = NULL; } if (stencil->writemask != stencilMax) { /* mask &= stencil->writemask */ LLVMValueRef writemask = lp_build_const_int_vec(type, stencil->writemask); mask = LLVMBuildAnd(bld->builder, mask, writemask, ""); /* res = (res & mask) | (stencilVals & ~mask) */ res = lp_build_select_bitwise(bld, writemask, res, stencilVals); } else { /* res = mask ? res : stencilVals */ res = lp_build_select(bld, mask, res, stencilVals); } return res; }
/** * Performs optimisations and blending independent of SoA/AoS * * @param func the blend function * @param factor_src PIPE_BLENDFACTOR_xxx * @param factor_dst PIPE_BLENDFACTOR_xxx * @param src source rgba * @param dst dest rgba * @param src_factor src factor computed value * @param dst_factor dst factor computed value * @param not_alpha_dependent same factors accross all channels of src/dst * * not_alpha_dependent should be: * SoA: always true as it is only one channel at a time * AoS: rgb_src_factor == alpha_src_factor && rgb_dst_factor == alpha_dst_factor * * Note that pretty much every possible optimisation can only be done on non-unorm targets * due to unorm values not going above 1.0 meaning factorisation can change results. * e.g. (0.9 * 0.9) + (0.9 * 0.9) != 0.9 * (0.9 + 0.9) as result of + is always <= 1. */ LLVMValueRef lp_build_blend(struct lp_build_context *bld, unsigned func, unsigned factor_src, unsigned factor_dst, LLVMValueRef src, LLVMValueRef dst, LLVMValueRef src_factor, LLVMValueRef dst_factor, boolean not_alpha_dependent, boolean optimise_only) { LLVMValueRef result, src_term, dst_term; /* If we are not alpha dependent we can mess with the src/dst factors */ if (not_alpha_dependent) { if (lp_build_blend_factor_complementary(factor_src, factor_dst)) { if (func == PIPE_BLEND_ADD) { if (factor_src < factor_dst) { return lp_build_lerp(bld, src_factor, dst, src, 0); } else { return lp_build_lerp(bld, dst_factor, src, dst, 0); } } else if(bld->type.floating && func == PIPE_BLEND_SUBTRACT) { result = lp_build_add(bld, src, dst); if (factor_src < factor_dst) { result = lp_build_mul(bld, result, src_factor); return lp_build_sub(bld, result, dst); } else { result = lp_build_mul(bld, result, dst_factor); return lp_build_sub(bld, src, result); } } else if(bld->type.floating && func == PIPE_BLEND_REVERSE_SUBTRACT) { result = lp_build_add(bld, src, dst); if (factor_src < factor_dst) { result = lp_build_mul(bld, result, src_factor); return lp_build_sub(bld, dst, result); } else { result = lp_build_mul(bld, result, dst_factor); return lp_build_sub(bld, result, src); } } } if (bld->type.floating && factor_src == factor_dst) { if (func == PIPE_BLEND_ADD || func == PIPE_BLEND_SUBTRACT || func == PIPE_BLEND_REVERSE_SUBTRACT) { LLVMValueRef result; result = lp_build_blend_func(bld, func, src, dst); return lp_build_mul(bld, result, src_factor); } } } if (optimise_only) return NULL; src_term = lp_build_mul(bld, src, src_factor); dst_term = lp_build_mul(bld, dst, dst_factor); return lp_build_blend_func(bld, func, src_term, dst_term); }