static inline void
blend_unorm8_argb(struct reg *src, __m256i dst_argb)
{
if (gt.blend.enable) {
const __m256i mask = _mm256_set1_epi32(0xff);
const __m256 scale = _mm256_set1_ps(1.0f / 255.0f);
struct reg dst[4];
/* Convert to float */
dst[2].reg = _mm256_mul_ps(_mm256_cvtepi32_ps(_mm256_and_si256(dst_argb, mask)), scale);
dst_argb = _mm256_srli_epi32(dst_argb, 8);
dst[1].reg = _mm256_mul_ps(_mm256_cvtepi32_ps(_mm256_and_si256(dst_argb, mask)), scale);
dst_argb = _mm256_srli_epi32(dst_argb, 8);
dst[0].reg = _mm256_mul_ps(_mm256_cvtepi32_ps(_mm256_and_si256(dst_argb, mask)), scale);
dst_argb = _mm256_srli_epi32(dst_argb, 8);
dst[3].reg = _mm256_mul_ps(_mm256_cvtepi32_ps(_mm256_and_si256(dst_argb, mask)), scale);
/* Blend, assuming src BLENDFACTOR_SRC_ALPHA, dst
* BLENDFACTOR_INV_SRC_ALPHA, and BLENDFUNCTION_ADD. */
const __m256 inv_alpha = _mm256_sub_ps(_mm256_set1_ps(1.0f), src[3].reg);
src[0].reg = _mm256_add_ps(_mm256_mul_ps(src[3].reg, src[0].reg),
_mm256_mul_ps(inv_alpha, dst[0].reg));
src[1].reg = _mm256_add_ps(_mm256_mul_ps(src[3].reg, src[1].reg),
_mm256_mul_ps(inv_alpha, dst[1].reg));
src[2].reg = _mm256_add_ps(_mm256_mul_ps(src[3].reg, src[2].reg),
_mm256_mul_ps(inv_alpha, dst[2].reg));
src[3].reg = _mm256_add_ps(_mm256_mul_ps(src[3].reg, src[3].reg),
_mm256_mul_ps(inv_alpha, dst[3].reg));
}
}
inline void avx2_uv_to_xy_f(const __m256i u, const __m256i v, __m256& x, __m256& y)
{
y = _mm256_cvtepi32_ps(v);
x = _mm256_cvtepi32_ps(u);
x = _mm256_fmadd_ps(y, calin::math::simd::c_m256(_c_m256_vx), x);
y = _mm256_mul_ps(y, calin::math::simd::c_m256(_c_m256_vy));
}
inline void avx2_xy_to_uv_with_remainder_f(
__m256& x_in_dx_out, __m256& y_in_dy_out, __m256i& u, __m256i& v)
{
avx2_xy_to_uv_f(x_in_dx_out, y_in_dy_out, u, v);
x_in_dx_out = _mm256_sub_ps(x_in_dx_out, _mm256_cvtepi32_ps(u));
__m256 vf = _mm256_cvtepi32_ps(v);
x_in_dx_out = _mm256_fnmadd_ps(vf, calin::math::simd::c_m256(_c_m256_vx), x_in_dx_out);
y_in_dy_out = _mm256_fnmadd_ps(vf, calin::math::simd::c_m256(_c_m256_vy), y_in_dy_out);
}
inline void avx2_xy_to_uv_f(__m256 x, __m256 y, __m256i& u, __m256i& v)
{
// Convert X,Y first into U,V space then round to nearest
// integer. That gets us close to correct answer, mapping XY to a
// lozenge-shaped space rather than hexagonal. We then correct the
// four regions that lie outside the hexagonal cell assigning them
// to their correct neighboring cell.
// Writer's note: see ~/Google Drive/Work/calin
// double dv = y*c_vy_inv;
// double du = x-dv*c_vx;
// u = std::lround(du);
// v = std::lround(dv);
// du -= u;
// dv -= v;
y = _mm256_mul_ps(y, calin::math::simd::c_m256(_c_m256_vy_inv));
x = _mm256_fnmadd_ps(y, calin::math::simd::c_m256(_c_m256_vx), x);
u = _mm256_cvtps_epi32(x);
v = _mm256_cvtps_epi32(y);
x = _mm256_sub_ps(x, _mm256_cvtepi32_ps(u));
y = _mm256_sub_ps(y, _mm256_cvtepi32_ps(v));
// double c3 = dv-du;
const __m256i c3 = _mm256_castps_si256(_mm256_sub_ps(y, x));
__m256i uvshift;
__m256i mask;
// double c1 = du+0.5*dv;
// double c2 = dv+0.5*du;
// if(c3<0) {
// if(c1>=1) u++;
// else if(c2<-1) v--;
// } else {
// if(c2>=1) v++;
// else if(c1<-1) u--;
// }
uvshift = _mm256_cvtps_epi32(_mm256_fmadd_ps(y, calin::math::simd::c_m256(_c_m256_one_half), x));
mask = _mm256_srai_epi32(_mm256_xor_si256(uvshift, c3), 31);
u = _mm256_blendv_epi8(u, _mm256_add_epi32(u, uvshift), mask);
uvshift = _mm256_cvtps_epi32(_mm256_fmadd_ps(x, calin::math::simd::c_m256(_c_m256_one_half), y));
mask = _mm256_srai_epi32(_mm256_xor_si256(uvshift, c3), 31);
v = _mm256_blendv_epi8(_mm256_add_epi32(v, uvshift), v, mask);
}
__m256 mm256_exp_ps(__m256 x) {
__m256 tmp = _mm256_setzero_ps(), fx;
__m256i emm0;
__m256 one = *(__m256*)m256_ps_1;
x = _mm256_min_ps(x, *(__m256*)m256_ps_exp_hi);
x = _mm256_max_ps(x, *(__m256*)m256_ps_exp_lo);
/* express exp(x) as exp(g + n*log(2)) */
fx = _mm256_mul_ps(x, *(__m256*)m256_ps_cephes_LOG2EF);
fx = _mm256_add_ps(fx, *(__m256*)m256_ps_0p5);
/* how to perform a floorf with SSE: just below */
/* step 1 : cast to int */
emm0 = _mm256_cvttps_epi32(fx);
/* step 2 : cast back to float */
tmp = _mm256_cvtepi32_ps(emm0);
/* if greater, substract 1 */
__m256 mask = _mm256_cmp_ps( tmp, fx, _CMP_GT_OS );
mask = _mm256_and_ps(mask, one);
fx = _mm256_sub_ps(tmp, mask);
tmp = _mm256_mul_ps(fx, *(__m256*)m256_ps_cephes_exp_C1);
__m256 z = _mm256_mul_ps(fx, *(__m256*)m256_ps_cephes_exp_C2);
x = _mm256_sub_ps(x, tmp);
x = _mm256_sub_ps(x, z);
z = _mm256_mul_ps(x,x);
__m256 y = *(__m256*)m256_ps_cephes_exp_p0;
y = _mm256_mul_ps(y, x);
y = _mm256_add_ps(y, *(__m256*)m256_ps_cephes_exp_p1);
y = _mm256_mul_ps(y, x);
y = _mm256_add_ps(y, *(__m256*)m256_ps_cephes_exp_p2);
y = _mm256_mul_ps(y, x);
y = _mm256_add_ps(y, *(__m256*)m256_ps_cephes_exp_p3);
y = _mm256_mul_ps(y, x);
y = _mm256_add_ps(y, *(__m256*)m256_ps_cephes_exp_p4);
y = _mm256_mul_ps(y, x);
y = _mm256_add_ps(y, *(__m256*)m256_ps_cephes_exp_p5);
y = _mm256_mul_ps(y, z);
y = _mm256_add_ps(y, x);
y = _mm256_add_ps(y, one);
/* build 2^n */
emm0 = _mm256_cvttps_epi32(fx);
emm0 = _mm256_add_epi32(emm0, *(__m256i*)m256_pi32_0x7f);
emm0 = _mm256_slli_epi32(emm0, 23);
__m256 pow2n = _mm256_castsi256_ps(emm0);
y = _mm256_mul_ps(y, pow2n);
_mm256_zeroupper();
return y;
}
inline avx_m256_t newsin_ps(avx_m256_t x) {
avx_m256_t sign_bit = _mm256_and_ps(x, _ps_sign_mask);
x = _mm256_and_ps(x, _ps_inv_sign_mask);
avx_m256_t y = _mm256_mul_ps(x, _ps_cephes_FOPI);
avx_m256i_t emm2 = _mm256_cvttps_epi32(y);
emm2 = _mm256_add_epi32(emm2, _pi32_1);
emm2 = _mm256_and_si256(emm2, _pi32_inv1);
y = _mm256_cvtepi32_ps(emm2);
avx_m256i_t emm0 = _mm256_and_si256(emm2, _pi32_4);
emm0 = _mm256_slli_epi32(emm0, 29);
emm2 = _mm256_and_si256(emm2, _pi32_2);
emm2 = _mm256_cmpeq_epi32(emm2, _mm256_setzero_si256());
avx_m256_t swap_sign_bit = _mm256_castsi256_ps(emm0);
avx_m256_t poly_mask = _mm256_castsi256_ps(emm2);
sign_bit = _mm256_xor_ps(sign_bit, swap_sign_bit);
avx_m256_t temp = _ps_minus_cephes_DP123;
temp = _mm256_mul_ps(y, temp);
x = _mm256_add_ps(x, temp);
avx_m256_t x2 = _mm256_mul_ps(x, x);
avx_m256_t x3 = _mm256_mul_ps(x2, x);
avx_m256_t x4 = _mm256_mul_ps(x2, x2);
y = _ps_coscof_p0;
avx_m256_t y2 = _ps_sincof_p0;
y = _mm256_mul_ps(y, x2);
y2 = _mm256_mul_ps(y2, x2);
y = _mm256_add_ps(y, _ps_coscof_p1);
y2 = _mm256_add_ps(y2, _ps_sincof_p1);
y = _mm256_mul_ps(y, x2);
y2 = _mm256_mul_ps(y2, x2);
y = _mm256_add_ps(y, _ps_coscof_p2);
y2 = _mm256_add_ps(y2, _ps_sincof_p2);
y = _mm256_mul_ps(y, x4);
y2 = _mm256_mul_ps(y2, x3);
temp = _mm256_mul_ps(x2, _ps_0p5);
temp = _mm256_sub_ps(temp, _ps_1);
y = _mm256_sub_ps(y, temp);
y2 = _mm256_add_ps(y2, x);
y = _mm256_andnot_ps(poly_mask, y);
y2 = _mm256_and_ps(poly_mask, y2);
y = _mm256_add_ps(y, y2);
y = _mm256_xor_ps(y, sign_bit);
return y;
} // newsin_ps()
inline __m256i avx2_positive_hexid_to_ringid_root(const __m256i hexid)
{
// The following algorithm works until hexid=12,589,056
// const unsigned iarg = 1+4*(hexid-1)/3;
// return (unsigned(std::sqrt(float(iarg)))+1)/2;
__m256 arg = _mm256_cvtepi32_ps(hexid);
arg = _mm256_fmsub_ps(arg, calin::math::simd::c_m256(_c_m256_four_thirds),
calin::math::simd::c_m256(_c_m256_one_third));
arg = _mm256_sqrt_ps(arg);
arg = _mm256_fmadd_ps(arg, calin::math::simd::c_m256(_c_m256_one_half),
calin::math::simd::c_m256(_c_m256_one_half));
arg = _mm256_floor_ps(arg);
return _mm256_cvtps_epi32(arg);
}
void cvtScale_s16s32f32Line_AVX2(const short* src, int* dst, float scale, float shift, int width)
{
int x = 0;
__m256 scale256 = _mm256_set1_ps(scale);
__m256 shift256 = _mm256_set1_ps(shift);
const int shuffle = 0xD8;
for (; x <= width - 16; x += 16)
{
__m256i v_src = _mm256_loadu_si256((const __m256i *)(src + x));
v_src = _mm256_permute4x64_epi64(v_src, shuffle);
__m256i v_src_lo = _mm256_srai_epi32(_mm256_unpacklo_epi16(v_src, v_src), 16);
__m256i v_src_hi = _mm256_srai_epi32(_mm256_unpackhi_epi16(v_src, v_src), 16);
__m256 v_dst0 = _mm256_add_ps(_mm256_mul_ps(_mm256_cvtepi32_ps(v_src_lo), scale256), shift256);
__m256 v_dst1 = _mm256_add_ps(_mm256_mul_ps(_mm256_cvtepi32_ps(v_src_hi), scale256), shift256);
_mm256_storeu_si256((__m256i *)(dst + x), _mm256_cvtps_epi32(v_dst0));
_mm256_storeu_si256((__m256i *)(dst + x + 8), _mm256_cvtps_epi32(v_dst1));
}
for (; x < width; x++)
dst[x] = saturate_cast<int>(src[x] * scale + shift);
}
void convertCAVX(int num, uint8_t *in, float *out){
int i;
__m256 sub = _mm256_set1_ps(128.0);
__m256 mul = _mm256_set1_ps(1/128.0);
for(i=0; i<num; i+=8){
__m128i val = _mm_loadu_si128((__m128i *)(in + i));
__m256i ints = _mm256_cvtepu8_epi32(val);
__m256 cvtd = _mm256_cvtepi32_ps(ints);
__m256 res = _mm256_mul_ps(_mm256_sub_ps(cvtd, sub), mul);
_mm256_storeu_ps(out + i, res);
}
}
void TransLut_FindIndexAvx2 <TransLut::MapperLin>::find_index (const TransLut::FloatIntMix val_arr [8], __m256i &index, __m256 &frac)
{
assert (val_arr != 0);
const __m256 scale = _mm256_set1_ps (1 << LINLUT_RES_L2);
const __m256i offset =
_mm256_set1_epi32 (-LINLUT_MIN_F * (1 << LINLUT_RES_L2));
const __m256i val_min = _mm256_setzero_si256 ();
const __m256i val_max = _mm256_set1_epi32 (LINLUT_SIZE_F - 2);
const __m256 v =
_mm256_load_ps (reinterpret_cast <const float *> (val_arr));
const __m256 val_scl = _mm256_mul_ps (v, scale);
const __m256i index_raw = _mm256_cvtps_epi32 (val_scl);
__m256i index_tmp = _mm256_add_epi32 (index_raw, offset);
index_tmp = _mm256_min_epi32 (index_tmp, val_max);
index = _mm256_max_epi32 (index_tmp, val_min);
frac = _mm256_sub_ps (val_scl, _mm256_cvtepi32_ps (index_raw));
}
__m256 _inner_mm256_exp_ps1(__m256 arg)
{
arg = _mm256_mul_ps(arg, _mm256_set1_ps(1.4426950408889634073599246810018921374266459541529859f));
__m256i e = _mm256_add_epi32(
_mm256_castps_si256(_mm256_cmp_ps(arg, _mm256_set1_ps(0.0f), _CMP_LT_OQ)),
_mm256_cvttps_epi32(arg));
arg = _mm256_sub_ps(arg, _mm256_cvtepi32_ps(e));
__m256 intermediate_result;
intermediate_result = _mm256_fmadd_ps(_mm256_set1_ps(0.0136779459179717f), arg, _mm256_set1_ps(0.0517692205767896f));
intermediate_result = _mm256_fmadd_ps(intermediate_result, arg, _mm256_set1_ps(0.241554388295527f));
intermediate_result = _mm256_fmadd_ps(intermediate_result, arg, _mm256_set1_ps(0.692998430056128f));
intermediate_result = _mm256_fmadd_ps(intermediate_result, arg, _mm256_set1_ps(0.999999804292074f));
arg = intermediate_result;
__m256 res = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_add_epi32(e, _mm256_set1_epi32(127)), 23));
res = _mm256_mul_ps(res, arg);
return res;
}
void calculate_fma_float (unsigned char * out, double X0, double Y0, double scale, unsigned YSTART, unsigned SX, unsigned SY)
{
__m256 dd = _mm256_set1_ps ((float) scale);
__m256 XX0 = _mm256_set1_ps ((float) X0);
for (unsigned j = YSTART; j < SY; j++) {
__m256 y0 = _mm256_set1_ps (j*(float) scale + (float) Y0);
for (unsigned i = 0; i < SX; i += 8) {
__m256i ind = _mm256_setr_epi32 (i, i + 1, i + 2, i + 3, i + 4, i + 5, i + 6, i + 7);
__m256 x0 = _mm256_fmadd_ps (dd, _mm256_cvtepi32_ps (ind), XX0);
__m256 x = x0;
__m256 y = y0;
__m256i counts = _mm256_setzero_si256 ();
__m256i cmp_mask = _mm256_set1_epi32 (0xFFFFFFFFu);
for (unsigned n = 0; n < 255; n++) {
__m256 x2 = _mm256_mul_ps (x, x);
__m256 y2 = _mm256_mul_ps (y, y);
__m256 abs = _mm256_add_ps (x2, y2);
__m256i cmp = _mm256_castps_si256 (_mm256_cmp_ps (abs, _mm256_set1_ps (4), 1));
cmp_mask = _mm256_and_si256 (cmp_mask, cmp);
if (_mm256_testz_si256 (cmp_mask, cmp_mask)) {
break;
}
counts = _mm256_sub_epi32 (counts, cmp_mask);
__m256 t = _mm256_add_ps (x, x);
y = _mm256_fmadd_ps (t, y, y0);
x = _mm256_add_ps (_mm256_sub_ps (x2, y2), x0);
}
__m256i result = _mm256_shuffle_epi8 (counts, _mm256_setr_epi8 (0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12));
__m128i result128 = _128i_shuffle (_mm256_extractf128_si256 (result, 0), _mm256_extractf128_si256 (result, 1), 0, 0, 0, 0);
result128 = _mm_shuffle_epi32 (result128, combine_4_2bits (0, 2, 0, 2));
_mm_storel_epi64 ((__m128i*) out, result128);
out += 8;
}
}
}
INLINE const avxb operator > ( const avxi& a, const avxi& b ) { return _mm256_cmp_ps(_mm256_cvtepi32_ps(a.m256), _mm256_cvtepi32_ps(b.m256), _CMP_NLE_UQ); }
template <bool inversion, bool align> void Convert(const uint8_t * src, const __m256 &_1_255, float * dst)
{
__m128i _src = Invert<inversion>(_mm_loadl_epi64((__m128i*)src));
Avx::Store<align>(dst, _mm256_mul_ps(_mm256_cvtepi32_ps(_mm256_cvtepu8_epi32(_src)), _1_255));
}
/* since sin256_ps and cos256_ps are almost identical, sincos256_ps could replace both of them..
it is almost as fast, and gives you a free cosine with your sine */
void sincos256_ps(v8sf x, v8sf *s, v8sf *c) {
v8sf xmm1, xmm2, xmm3 = _mm256_setzero_ps(), sign_bit_sin, y;
v8si imm0, imm2, imm4;
#ifndef __AVX2__
v4si imm0_1, imm0_2;
v4si imm2_1, imm2_2;
v4si imm4_1, imm4_2;
#endif
sign_bit_sin = x;
/* take the absolute value */
x = _mm256_and_ps(x, *(v8sf*)_ps256_inv_sign_mask);
/* extract the sign bit (upper one) */
sign_bit_sin = _mm256_and_ps(sign_bit_sin, *(v8sf*)_ps256_sign_mask);
/* scale by 4/Pi */
y = _mm256_mul_ps(x, *(v8sf*)_ps256_cephes_FOPI);
#ifdef __AVX2__
/* store the integer part of y in imm2 */
imm2 = _mm256_cvttps_epi32(y);
/* j=(j+1) & (~1) (see the cephes sources) */
imm2 = _mm256_add_epi32(imm2, *(v8si*)_pi32_256_1);
imm2 = _mm256_and_si128(imm2, *(v8si*)_pi32_256_inv1);
y = _mm256_cvtepi32_ps(imm2);
imm4 = imm2;
/* get the swap sign flag for the sine */
imm0 = _mm256_and_si128(imm2, *(v8si*)_pi32_256_4);
imm0 = _mm256_slli_epi32(imm0, 29);
//v8sf swap_sign_bit_sin = _mm256_castsi256_ps(imm0);
/* get the polynom selection mask for the sine*/
imm2 = _mm256_and_si128(imm2, *(v8si*)_pi32_256_2);
imm2 = _mm256_cmpeq_epi32(imm2, *(v8si*)_pi32_256_0);
//v8sf poly_mask = _mm256_castsi256_ps(imm2);
#else
/* we use SSE2 routines to perform the integer ops */
COPY_IMM_TO_XMM(_mm256_cvttps_epi32(y),imm2_1,imm2_2);
imm2_1 = _mm_add_epi32(imm2_1, *(v4si*)_pi32avx_1);
imm2_2 = _mm_add_epi32(imm2_2, *(v4si*)_pi32avx_1);
imm2_1 = _mm_and_si128(imm2_1, *(v4si*)_pi32avx_inv1);
imm2_2 = _mm_and_si128(imm2_2, *(v4si*)_pi32avx_inv1);
COPY_XMM_TO_IMM(imm2_1,imm2_2,imm2);
y = _mm256_cvtepi32_ps(imm2);
imm4_1 = imm2_1;
imm4_2 = imm2_2;
imm0_1 = _mm_and_si128(imm2_1, *(v4si*)_pi32avx_4);
imm0_2 = _mm_and_si128(imm2_2, *(v4si*)_pi32avx_4);
imm0_1 = _mm_slli_epi32(imm0_1, 29);
imm0_2 = _mm_slli_epi32(imm0_2, 29);
COPY_XMM_TO_IMM(imm0_1, imm0_2, imm0);
imm2_1 = _mm_and_si128(imm2_1, *(v4si*)_pi32avx_2);
imm2_2 = _mm_and_si128(imm2_2, *(v4si*)_pi32avx_2);
imm2_1 = _mm_cmpeq_epi32(imm2_1, _mm_setzero_si128());
imm2_2 = _mm_cmpeq_epi32(imm2_2, _mm_setzero_si128());
COPY_XMM_TO_IMM(imm2_1, imm2_2, imm2);
#endif
v8sf swap_sign_bit_sin = _mm256_castsi256_ps(imm0);
v8sf poly_mask = _mm256_castsi256_ps(imm2);
/* The magic pass: "******"
x = ((x - y * DP1) - y * DP2) - y * DP3; */
xmm1 = *(v8sf*)_ps256_minus_cephes_DP1;
xmm2 = *(v8sf*)_ps256_minus_cephes_DP2;
xmm3 = *(v8sf*)_ps256_minus_cephes_DP3;
xmm1 = _mm256_mul_ps(y, xmm1);
xmm2 = _mm256_mul_ps(y, xmm2);
xmm3 = _mm256_mul_ps(y, xmm3);
x = _mm256_add_ps(x, xmm1);
x = _mm256_add_ps(x, xmm2);
x = _mm256_add_ps(x, xmm3);
#ifdef __AVX2__
imm4 = _mm256_sub_epi32(imm4, *(v8si*)_pi32_256_2);
imm4 = _mm256_andnot_si128(imm4, *(v8si*)_pi32_256_4);
imm4 = _mm256_slli_epi32(imm4, 29);
#else
imm4_1 = _mm_sub_epi32(imm4_1, *(v4si*)_pi32avx_2);
imm4_2 = _mm_sub_epi32(imm4_2, *(v4si*)_pi32avx_2);
imm4_1 = _mm_andnot_si128(imm4_1, *(v4si*)_pi32avx_4);
imm4_2 = _mm_andnot_si128(imm4_2, *(v4si*)_pi32avx_4);
imm4_1 = _mm_slli_epi32(imm4_1, 29);
imm4_2 = _mm_slli_epi32(imm4_2, 29);
COPY_XMM_TO_IMM(imm4_1, imm4_2, imm4);
#endif
v8sf sign_bit_cos = _mm256_castsi256_ps(imm4);
sign_bit_sin = _mm256_xor_ps(sign_bit_sin, swap_sign_bit_sin);
/* Evaluate the first polynom (0 <= x <= Pi/4) */
v8sf z = _mm256_mul_ps(x,x);
y = *(v8sf*)_ps256_coscof_p0;
y = _mm256_mul_ps(y, z);
y = _mm256_add_ps(y, *(v8sf*)_ps256_coscof_p1);
y = _mm256_mul_ps(y, z);
y = _mm256_add_ps(y, *(v8sf*)_ps256_coscof_p2);
y = _mm256_mul_ps(y, z);
y = _mm256_mul_ps(y, z);
v8sf tmp = _mm256_mul_ps(z, *(v8sf*)_ps256_0p5);
y = _mm256_sub_ps(y, tmp);
y = _mm256_add_ps(y, *(v8sf*)_ps256_1);
/* Evaluate the second polynom (Pi/4 <= x <= 0) */
v8sf y2 = *(v8sf*)_ps256_sincof_p0;
y2 = _mm256_mul_ps(y2, z);
y2 = _mm256_add_ps(y2, *(v8sf*)_ps256_sincof_p1);
y2 = _mm256_mul_ps(y2, z);
y2 = _mm256_add_ps(y2, *(v8sf*)_ps256_sincof_p2);
y2 = _mm256_mul_ps(y2, z);
y2 = _mm256_mul_ps(y2, x);
y2 = _mm256_add_ps(y2, x);
/* select the correct result from the two polynoms */
xmm3 = poly_mask;
v8sf ysin2 = _mm256_and_ps(xmm3, y2);
v8sf ysin1 = _mm256_andnot_ps(xmm3, y);
y2 = _mm256_sub_ps(y2,ysin2);
y = _mm256_sub_ps(y, ysin1);
xmm1 = _mm256_add_ps(ysin1,ysin2);
xmm2 = _mm256_add_ps(y,y2);
/* update the sign */
*s = _mm256_xor_ps(xmm1, sign_bit_sin);
*c = _mm256_xor_ps(xmm2, sign_bit_cos);
}
inline avx_m256_t newexp_ps(avx_m256_t x) {
avx_m256_t one = _ps_1;
avx_m256_t zero = _ps_0;
x = _mm256_min_ps(x, _ps_exp_hi);
x = _mm256_max_ps(x, _ps_exp_lo);
avx_m256_t temp_2 = _mm256_mul_ps(x, _ps_cephes_LOG2EF);
temp_2 = _mm256_add_ps(temp_2, _ps_0p5);
avx_m256i_t emm0 = _mm256_cvttps_epi32(temp_2);
avx_m256_t temp_1 = _mm256_cvtepi32_ps(emm0);
avx_m256_t temp_3 = _mm256_sub_ps(temp_1, temp_2);
avx_m256_t mask = _mm256_cmp_ps(temp_3, zero, _CMP_GT_OQ);
mask = _mm256_and_ps(mask, one);
temp_2 = _mm256_sub_ps(temp_1, mask);
emm0 = _mm256_cvttps_epi32(temp_2);
temp_1 = _mm256_mul_ps(temp_2, _ps_cephes_exp_C12);
x = _mm256_sub_ps(x, temp_1);
avx_m256_t x2 = _mm256_mul_ps(x, x);
avx_m256_t x3 = _mm256_mul_ps(x2, x);
avx_m256_t x4 = _mm256_mul_ps(x2, x2);
temp_1 = _mm256_add_ps(x, one);
temp_2 = _mm256_mul_ps(x2, _ps_cephes_exp_p5);
temp_3 = _mm256_mul_ps(x3, _ps_cephes_exp_p4);
temp_1 = _mm256_add_ps(temp_1, temp_2);
temp_2 = _mm256_mul_ps(x3, _ps_cephes_exp_p0);
temp_1 = _mm256_add_ps(temp_1, temp_3);
avx_m256_t temp_4 = _mm256_mul_ps(x, _ps_cephes_exp_p2);
temp_3 = _mm256_mul_ps(x2, _ps_cephes_exp_p1);
emm0 = _mm256_castps_si256(_mm256_add_ps(_mm256_castsi256_ps(emm0), _mm256_castsi256_ps(_pi32_0x7f)));
temp_2 = _mm256_add_ps(temp_2, temp_3);
temp_3 = _mm256_add_ps(temp_3, temp_4);
//emm0 = _mm256_slli_epi32(emm0, 23);
// convert emm0 into two 128-bit integer vectors
// perform shift on both vectors
// combine both vectors into 256-bit emm0
__m128i emm0hi = _mm256_extractf128_si256(emm0, 0);
__m128i emm0lo = _mm256_extractf128_si256(emm0, 1);
emm0hi = _mm_slli_epi32(emm0hi, 23);
emm0lo = _mm_slli_epi32(emm0lo, 23);
emm0 = _mm256_insertf128_si256(emm0, emm0hi, 0);
emm0 = _mm256_insertf128_si256(emm0, emm0lo, 1);
avx_m256_t pow2n = _mm256_castsi256_ps(emm0);
temp_2 = _mm256_add_ps(temp_2, temp_3);
temp_2 = _mm256_mul_ps(temp_2, x4);
avx_m256_t y = _mm256_add_ps(temp_1, temp_2);
y = _mm256_mul_ps(y, pow2n);
return y;
} // newexp_ps()
inline void newsincos_ps_dual(avx_m256_t x1, avx_m256_t x2, avx_m256_t *s1, avx_m256_t *s2,
avx_m256_t *c1, avx_m256_t *c2) {
avx_m256_t tempa = _ps_sign_mask;
avx_m256_t tempb = _ps_inv_sign_mask;
avx_m256_t sign_bit1 = _mm256_and_ps(x1, tempa);
avx_m256_t sign_bit2 = _mm256_and_ps(x2, tempa);
x1 = _mm256_and_ps(x1, tempb);
x2 = _mm256_and_ps(x2, tempb);
tempa = _ps_cephes_FOPI;
avx_m256_t y1 = _mm256_mul_ps(x1, tempa);
avx_m256_t y2 = _mm256_mul_ps(x2, tempa);
//avx_m256i_t emm21 = _mm256_cvttps_epi32(y1);
//avx_m256i_t emm22 = _mm256_cvttps_epi32(y2);
//emm21 = _mm256_add_epi32(emm21, _pi32_1);
//emm22 = _mm256_add_epi32(emm22, _pi32_1);
avx_m256i_t emm21 = _mm256_cvttps_epi32(_mm256_add_ps(y1, _ps_1));
avx_m256i_t emm22 = _mm256_cvttps_epi32(_mm256_add_ps(y2, _ps_1));
//emm21 = _mm256_and_si256(emm21, _pi32_inv1);
//emm22 = _mm256_and_si256(emm22, _pi32_inv1);
emm21 = _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(emm21), _mm256_castsi256_ps(_pi32_inv1)));
emm22 = _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(emm22), _mm256_castsi256_ps(_pi32_inv1)));
y1 = _mm256_cvtepi32_ps(emm21);
y2 = _mm256_cvtepi32_ps(emm22);
//avx_m256i_t tempia = _pi32_2;
//avx_m256i_t cos_emm21 = _mm256_sub_epi32(emm21, tempia);
//avx_m256i_t cos_emm22 = _mm256_sub_epi32(emm22, tempia);
avx_m256i_t cos_emm21 = _mm256_cvtps_epi32(_mm256_sub_ps(_mm256_cvtepi32_ps(emm21), _ps_2));
avx_m256i_t cos_emm22 = _mm256_cvtps_epi32(_mm256_sub_ps(_mm256_cvtepi32_ps(emm22), _ps_2));
//avx_m256i_t tempib = _pi32_4;
//avx_m256i_t emm01 = _mm256_and_si256(emm21, tempib);
//avx_m256i_t emm02 = _mm256_and_si256(emm22, tempib);
avx_m256i_t emm01 = _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(emm21),
_mm256_castsi256_ps(_pi32_4)));
avx_m256i_t emm02 = _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(emm22),
_mm256_castsi256_ps(_pi32_4)));
//avx_m256i_t cos_emm01 = _mm256_andnot_si256(cos_emm21, tempib);
//avx_m256i_t cos_emm02 = _mm256_andnot_si256(cos_emm22, tempib);
avx_m256i_t cos_emm01 = _mm256_castps_si256(_mm256_andnot_ps(_mm256_castsi256_ps(cos_emm21),
_mm256_castsi256_ps(_pi32_4)));
avx_m256i_t cos_emm02 = _mm256_castps_si256(_mm256_andnot_ps(_mm256_castsi256_ps(cos_emm22),
_mm256_castsi256_ps(_pi32_4)));
//emm01 = _mm256_slli_epi32(emm01, 29);
__m128i emm0hi1 = _mm256_extractf128_si256(emm01, 0);
__m128i emm0lo1 = _mm256_extractf128_si256(emm01, 1);
emm0hi1 = _mm_slli_epi32(emm0hi1, 29);
emm0lo1 = _mm_slli_epi32(emm0lo1, 29);
emm01 = _mm256_insertf128_si256(emm01, emm0hi1, 0);
emm01 = _mm256_insertf128_si256(emm01, emm0lo1, 1);
//emm02 = _mm256_slli_epi32(emm02, 29);
__m128i emm0hi2 = _mm256_extractf128_si256(emm02, 0);
__m128i emm0lo2 = _mm256_extractf128_si256(emm02, 1);
emm0hi2 = _mm_slli_epi32(emm0hi2, 29);
emm0lo2 = _mm_slli_epi32(emm0lo2, 29);
emm02 = _mm256_insertf128_si256(emm02, emm0hi1, 0);
emm02 = _mm256_insertf128_si256(emm02, emm0lo1, 1);
//cos_emm01 = _mm256_slli_epi32(cos_emm01, 29);
__m128i cos_emm0hi1 = _mm256_extractf128_si256(cos_emm01, 0);
__m128i cos_emm0lo1 = _mm256_extractf128_si256(cos_emm01, 1);
cos_emm0hi1 = _mm_slli_epi32(cos_emm0hi1, 29);
cos_emm0lo1 = _mm_slli_epi32(cos_emm0lo1, 29);
cos_emm01 = _mm256_insertf128_si256(cos_emm01, cos_emm0hi1, 0);
cos_emm01 = _mm256_insertf128_si256(cos_emm01, cos_emm0lo1, 1);
//cos_emm02 = _mm256_slli_epi32(cos_emm02, 29);
__m128i cos_emm0hi2 = _mm256_extractf128_si256(cos_emm02, 0);
__m128i cos_emm0lo2 = _mm256_extractf128_si256(cos_emm02, 1);
cos_emm0hi2 = _mm_slli_epi32(cos_emm0hi2, 29);
cos_emm0lo2 = _mm_slli_epi32(cos_emm0lo2, 29);
cos_emm02 = _mm256_insertf128_si256(cos_emm02, cos_emm0hi2, 0);
cos_emm02 = _mm256_insertf128_si256(cos_emm02, cos_emm0lo2, 1);
//tempia = _pi32_2;
//tempib = _mm256_setzero_si256();
//emm21 = _mm256_and_si256(emm21, tempia);
//emm22 = _mm256_and_si256(emm22, tempia);
emm21 = _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(emm21),
_mm256_castsi256_ps(_pi32_2)));
emm22 = _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(emm22),
_mm256_castsi256_ps(_pi32_2)));
//cos_emm21 = _mm256_and_si256(cos_emm21, tempia);
//cos_emm22 = _mm256_and_si256(cos_emm22, tempia);
cos_emm21 = _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(cos_emm21),
_mm256_castsi256_ps(_pi32_2)));
cos_emm22 = _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(cos_emm22),
_mm256_castsi256_ps(_pi32_2)));
//emm21 = _mm256_cmpeq_epi32(emm21, tempib);
//emm22 = _mm256_cmpeq_epi32(emm22, tempib);
emm21 = _mm256_castps_si256(_mm256_cmp_ps(_mm256_castsi256_ps(emm21), _mm256_setzero_ps(), _CMP_EQ_UQ));
emm22 = _mm256_castps_si256(_mm256_cmp_ps(_mm256_castsi256_ps(emm22), _mm256_setzero_ps(), _CMP_EQ_UQ));
//cos_emm21 = _mm256_cmpeq_epi32(cos_emm21, tempib);
//cos_emm22 = _mm256_cmpeq_epi32(cos_emm22, tempib);
cos_emm21 = _mm256_castps_si256(_mm256_cmp_ps(_mm256_castsi256_ps(cos_emm21), _mm256_setzero_ps(), _CMP_EQ_UQ));
cos_emm22 = _mm256_castps_si256(_mm256_cmp_ps(_mm256_castsi256_ps(cos_emm22), _mm256_setzero_ps(), _CMP_EQ_UQ));
avx_m256_t emm0f1 = _mm256_castsi256_ps(emm01);
avx_m256_t emm0f2 = _mm256_castsi256_ps(emm02);
avx_m256_t emm2f1 = _mm256_castsi256_ps(emm21);
avx_m256_t emm2f2 = _mm256_castsi256_ps(emm22);
avx_m256_t cos_emm0f1 = _mm256_castsi256_ps(cos_emm01);
avx_m256_t cos_emm0f2 = _mm256_castsi256_ps(cos_emm02);
avx_m256_t cos_emm2f1 = _mm256_castsi256_ps(cos_emm21);
avx_m256_t cos_emm2f2 = _mm256_castsi256_ps(cos_emm22);
sign_bit1 = _mm256_xor_ps(sign_bit1, emm0f1);
sign_bit2 = _mm256_xor_ps(sign_bit2, emm0f2);
tempa = _ps_minus_cephes_DP123;
tempb = _mm256_mul_ps(y2, tempa);
tempa = _mm256_mul_ps(y1, tempa);
x2 = _mm256_add_ps(x2, tempb);
x1 = _mm256_add_ps(x1, tempa);
avx_m256_t x21 = _mm256_mul_ps(x1, x1);
avx_m256_t x22 = _mm256_mul_ps(x2, x2);
avx_m256_t x31 = _mm256_mul_ps(x21, x1);
avx_m256_t x32 = _mm256_mul_ps(x22, x2);
avx_m256_t x41 = _mm256_mul_ps(x21, x21);
avx_m256_t x42 = _mm256_mul_ps(x22, x22);
tempa = _ps_coscof_p0;
tempb = _ps_sincof_p0;
y1 = _mm256_mul_ps(x21, tempa);
y2 = _mm256_mul_ps(x22, tempa);
avx_m256_t y21 = _mm256_mul_ps(x21, tempb);
avx_m256_t y22 = _mm256_mul_ps(x22, tempb);
tempa = _ps_coscof_p1;
tempb = _ps_sincof_p1;
y1 = _mm256_add_ps(y1, tempa);
y2 = _mm256_add_ps(y2, tempa);
y21 = _mm256_add_ps(y21, tempb);
y22 = _mm256_add_ps(y22, tempb);
y1 = _mm256_mul_ps(y1, x21);
y2 = _mm256_mul_ps(y2, x22);
y21 = _mm256_mul_ps(y21, x21);
y22 = _mm256_mul_ps(y22, x22);
tempa = _ps_coscof_p2;
tempb = _ps_sincof_p2;
y1 = _mm256_add_ps(y1, tempa);
y2 = _mm256_add_ps(y2, tempa);
y21 = _mm256_add_ps(y21, tempb);
y22 = _mm256_add_ps(y22, tempb);
y1 = _mm256_mul_ps(y1, x41);
y2 = _mm256_mul_ps(y2, x42);
y21 = _mm256_mul_ps(y21, x31);
y22 = _mm256_mul_ps(y22, x32);
tempa = _ps_0p5;
tempb = _ps_1;
avx_m256_t temp_21 = _mm256_mul_ps(x21, tempa);
avx_m256_t temp_22 = _mm256_mul_ps(x22, tempa);
y21 = _mm256_add_ps(y21, x1);
y22 = _mm256_add_ps(y22, x2);
temp_21 = _mm256_sub_ps(temp_21, tempb);
temp_22 = _mm256_sub_ps(temp_22, tempb);
y1 = _mm256_sub_ps(y1, temp_21);
y2 = _mm256_sub_ps(y2, temp_22);
avx_m256_t cos_y1 = y1;
avx_m256_t cos_y2 = y2;
avx_m256_t cos_y21 = y21;
avx_m256_t cos_y22 = y22;
y1 = _mm256_andnot_ps(emm2f1, y1);
y2 = _mm256_andnot_ps(emm2f2, y2);
cos_y1 = _mm256_andnot_ps(cos_emm2f1, cos_y1);
cos_y2 = _mm256_andnot_ps(cos_emm2f2, cos_y2);
y21 = _mm256_and_ps(emm2f1, y21);
y22 = _mm256_and_ps(emm2f2, y22);
cos_y21 = _mm256_and_ps(cos_emm2f1, cos_y21);
cos_y22 = _mm256_and_ps(cos_emm2f2, cos_y22);
y1 = _mm256_add_ps(y1, y21);
y2 = _mm256_add_ps(y2, y22);
cos_y1 = _mm256_add_ps(cos_y1, cos_y21);
cos_y2 = _mm256_add_ps(cos_y2, cos_y22);
*s1 = _mm256_xor_ps(y1, sign_bit1);
*s2 = _mm256_xor_ps(y2, sign_bit2);
*c1 = _mm256_xor_ps(cos_y1, cos_emm0f1);
*c2 = _mm256_xor_ps(cos_y2, cos_emm0f2);
} // newsincos_ps_dual()
inline void newsincos_ps(avx_m256_t x, avx_m256_t *s, avx_m256_t *c) {
avx_m256_t sign_bit = _mm256_and_ps(x, _ps_sign_mask);
x = _mm256_and_ps(x, _ps_inv_sign_mask);
avx_m256_t y = _mm256_mul_ps(x, _ps_cephes_FOPI);
//avx_m256i_t emm2 = _mm256_cvttps_epi32(y);
//emm2 = _mm256_add_epi32(emm2, _pi32_1);
avx_m256i_t emm2 = _mm256_cvttps_epi32(_mm256_add_ps(y, _ps_1));
//emm2 = _mm256_and_si256(emm2, _pi32_inv1);
emm2 = _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(emm2), _mm256_castsi256_ps(_pi32_inv1)));
y = _mm256_cvtepi32_ps(emm2);
//avx_m256i_t cos_emm2 = _mm256_sub_epi32(emm2, _pi32_2);
avx_m256i_t cos_emm2 = _mm256_cvtps_epi32(_mm256_sub_ps(_mm256_cvtepi32_ps(emm2), _ps_2));
//avx_m256i_t emm0 = _mm256_and_si256(emm2, _pi32_4);
avx_m256i_t emm0 = _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(emm2),
_mm256_castsi256_ps(_pi32_4)));
//avx_m256i_t cos_emm0 = _mm256_andnot_si256(cos_emm2, _pi32_4);
avx_m256i_t cos_emm0 = _mm256_castps_si256(_mm256_andnot_ps(_mm256_castsi256_ps(cos_emm2),
_mm256_castsi256_ps(_pi32_4)));
//emm0 = _mm256_slli_epi32(emm0, 29);
__m128i emm0hi = _mm256_extractf128_si256(emm0, 0);
__m128i emm0lo = _mm256_extractf128_si256(emm0, 1);
emm0hi = _mm_slli_epi32(emm0hi, 29);
emm0lo = _mm_slli_epi32(emm0lo, 29);
emm0 = _mm256_insertf128_si256(emm0, emm0hi, 0);
emm0 = _mm256_insertf128_si256(emm0, emm0lo, 1);
//cos_emm0 = _mm256_slli_epi32(cos_emm0, 29);
__m128i cos_emm0hi = _mm256_extractf128_si256(cos_emm0, 0);
__m128i cos_emm0lo = _mm256_extractf128_si256(cos_emm0, 1);
cos_emm0hi = _mm_slli_epi32(cos_emm0hi, 29);
cos_emm0lo = _mm_slli_epi32(cos_emm0lo, 29);
cos_emm0 = _mm256_insertf128_si256(cos_emm0, cos_emm0hi, 0);
cos_emm0 = _mm256_insertf128_si256(cos_emm0, cos_emm0lo, 1);
//emm2 = _mm256_and_si256(emm2, _pi32_2);
emm2 = _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(emm2),
_mm256_castsi256_ps(_pi32_2)));
//cos_emm2 = _mm256_and_si256(cos_emm2, _pi32_2);
cos_emm2 = _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(cos_emm2),
_mm256_castsi256_ps(_pi32_2)));
//emm2 = _mm256_cmpeq_epi32(emm2, _mm256_setzero_si256());
emm2 = _mm256_castps_si256(_mm256_cmp_ps(_mm256_castsi256_ps(emm2), _mm256_setzero_ps(), _CMP_EQ_UQ));
//cos_emm2 = _mm256_cmpeq_epi32(cos_emm2, _mm256_setzero_si256());
cos_emm2 = _mm256_castps_si256(_mm256_cmp_ps(_mm256_castsi256_ps(cos_emm2), _mm256_setzero_ps(), _CMP_EQ_UQ));
avx_m256_t emm0f = _mm256_castsi256_ps(emm0);
avx_m256_t emm2f = _mm256_castsi256_ps(emm2);
avx_m256_t cos_emm0f = _mm256_castsi256_ps(cos_emm0);
avx_m256_t cos_emm2f = _mm256_castsi256_ps(cos_emm2);
sign_bit = _mm256_xor_ps(sign_bit, emm0f);
avx_m256_t temp_2 = _ps_minus_cephes_DP123;
temp_2 = _mm256_mul_ps(y, temp_2);
x = _mm256_add_ps(x, temp_2);
avx_m256_t x2 = _mm256_mul_ps(x, x);
avx_m256_t x3 = _mm256_mul_ps(x2, x);
avx_m256_t x4 = _mm256_mul_ps(x2, x2);
y = _ps_coscof_p0;
avx_m256_t y2 = _ps_sincof_p0;
y = _mm256_mul_ps(y, x2);
y2 = _mm256_mul_ps(y2, x2);
y = _mm256_add_ps(y, _ps_coscof_p1);
y2 = _mm256_add_ps(y2, _ps_sincof_p1);
y = _mm256_mul_ps(y, x2);
y2 = _mm256_mul_ps(y2, x2);
y = _mm256_add_ps(y, _ps_coscof_p2);
y2 = _mm256_add_ps(y2, _ps_sincof_p2);
y = _mm256_mul_ps(y, x4);
y2 = _mm256_mul_ps(y2, x3);
temp_2 = _mm256_mul_ps(x2, _ps_0p5);
y2 = _mm256_add_ps(y2, x);
temp_2 = _mm256_sub_ps(temp_2, _ps_1);
y = _mm256_sub_ps(y, temp_2);
avx_m256_t cos_y = y;
avx_m256_t cos_y2 = y2;
y = _mm256_andnot_ps(emm2f, y);
cos_y = _mm256_andnot_ps(cos_emm2f, cos_y);
y2 = _mm256_and_ps(emm2f, y2);
cos_y2 = _mm256_and_ps(cos_emm2f, cos_y2);
y = _mm256_add_ps(y, y2);
cos_y = _mm256_add_ps(cos_y, cos_y2);
*s = _mm256_xor_ps(y, sign_bit);
*c = _mm256_xor_ps(cos_y, cos_emm0f);
} // newsincos_ps()
__m256 convert_to_float(void) const
{
return _mm256_cvtepi32_ps(data_);
}
inline void newexp_ps_dual(avx_m256_t x1, avx_m256_t x2, avx_m256_t* exp1, avx_m256_t* exp2) {
avx_m256_t one = _ps_1;
avx_m256_t zero = _ps_0;
x1 = _mm256_min_ps(x1, _ps_exp_hi);
x2 = _mm256_min_ps(x2, _ps_exp_hi);
x1 = _mm256_max_ps(x1, _ps_exp_lo);
x2 = _mm256_max_ps(x2, _ps_exp_lo);
avx_m256_t temp_21 = _mm256_mul_ps(x1, _ps_cephes_LOG2EF);
avx_m256_t temp_22 = _mm256_mul_ps(x2, _ps_cephes_LOG2EF);
temp_21 = _mm256_add_ps(temp_21, _ps_0p5);
temp_22 = _mm256_add_ps(temp_22, _ps_0p5);
avx_m256i_t emm01 = _mm256_cvttps_epi32(temp_21);
avx_m256i_t emm02 = _mm256_cvttps_epi32(temp_22);
avx_m256_t temp_11 = _mm256_cvtepi32_ps(emm01);
avx_m256_t temp_12 = _mm256_cvtepi32_ps(emm02);
avx_m256_t temp_31 = _mm256_sub_ps(temp_11, temp_21);
avx_m256_t temp_32 = _mm256_sub_ps(temp_12, temp_22);
avx_m256_t mask1 = _mm256_cmp_ps(temp_31, zero, _CMP_GT_OQ);
avx_m256_t mask2 = _mm256_cmp_ps(temp_32, zero, _CMP_GT_OQ);
mask1 = _mm256_and_ps(mask1, one);
mask2 = _mm256_and_ps(mask2, one);
temp_21 = _mm256_sub_ps(temp_11, mask1);
temp_22 = _mm256_sub_ps(temp_12, mask2);
emm01 = _mm256_cvttps_epi32(temp_21);
emm02 = _mm256_cvttps_epi32(temp_22);
temp_11 = _mm256_mul_ps(temp_21, _ps_cephes_exp_C12);
temp_12 = _mm256_mul_ps(temp_22, _ps_cephes_exp_C12);
x1 = _mm256_sub_ps(x1, temp_11);
x2 = _mm256_sub_ps(x2, temp_12);
avx_m256_t x21 = _mm256_mul_ps(x1, x1);
avx_m256_t x22 = _mm256_mul_ps(x2, x2);
avx_m256_t x31 = _mm256_mul_ps(x21, x1);
avx_m256_t x32 = _mm256_mul_ps(x22, x2);
avx_m256_t x41 = _mm256_mul_ps(x21, x21);
avx_m256_t x42 = _mm256_mul_ps(x22, x22);
temp_11 = _mm256_add_ps(x1, one);
temp_12 = _mm256_add_ps(x2, one);
temp_21 = _mm256_mul_ps(x21, _ps_cephes_exp_p5);
temp_22 = _mm256_mul_ps(x22, _ps_cephes_exp_p5);
temp_31 = _mm256_mul_ps(x31, _ps_cephes_exp_p4);
temp_32 = _mm256_mul_ps(x32, _ps_cephes_exp_p4);
temp_11 = _mm256_add_ps(temp_11, temp_21);
temp_12 = _mm256_add_ps(temp_12, temp_22);
temp_21 = _mm256_mul_ps(x31, _ps_cephes_exp_p0);
temp_22 = _mm256_mul_ps(x32, _ps_cephes_exp_p0);
temp_11 = _mm256_add_ps(temp_11, temp_31);
temp_12 = _mm256_add_ps(temp_12, temp_32);
avx_m256_t temp_41 = _mm256_mul_ps(x1, _ps_cephes_exp_p2);
avx_m256_t temp_42 = _mm256_mul_ps(x2, _ps_cephes_exp_p2);
temp_31 = _mm256_mul_ps(x21, _ps_cephes_exp_p1);
temp_32 = _mm256_mul_ps(x22, _ps_cephes_exp_p1);
//emm01 = _mm256_add_epi32(emm01, _pi32_0x7f);
//emm02 = _mm256_add_epi32(emm02, _pi32_0x7f);
emm01 = _mm256_castps_si256(_mm256_add_ps(_mm256_castsi256_ps(emm01), _mm256_castsi256_ps(_pi32_0x7f)));
emm02 = _mm256_castps_si256(_mm256_add_ps(_mm256_castsi256_ps(emm02), _mm256_castsi256_ps(_pi32_0x7f)));
temp_21 = _mm256_add_ps(temp_21, temp_31);
temp_22 = _mm256_add_ps(temp_22, temp_32);
temp_31 = _mm256_add_ps(temp_31, temp_41);
temp_32 = _mm256_add_ps(temp_32, temp_42);
//emm01 = _mm256_slli_epi32(emm01, 23);
__m128i emm0hi1 = _mm256_extractf128_si256(emm01, 0);
__m128i emm0lo1 = _mm256_extractf128_si256(emm01, 1);
emm0hi1 = _mm_slli_epi32(emm0hi1, 23);
emm0lo1 = _mm_slli_epi32(emm0lo1, 23);
emm01 = _mm256_insertf128_si256(emm01, emm0hi1, 0);
emm01 = _mm256_insertf128_si256(emm01, emm0lo1, 1);
//emm02 = _mm256_slli_epi32(emm02, 23);
__m128i emm0hi2 = _mm256_extractf128_si256(emm02, 0);
__m128i emm0lo2 = _mm256_extractf128_si256(emm02, 1);
emm0hi2 = _mm_slli_epi32(emm0hi2, 23);
emm0lo2 = _mm_slli_epi32(emm0lo2, 23);
emm02 = _mm256_insertf128_si256(emm02, emm0hi2, 0);
emm02 = _mm256_insertf128_si256(emm02, emm0lo2, 1);
avx_m256_t pow2n1 = _mm256_castsi256_ps(emm01);
avx_m256_t pow2n2 = _mm256_castsi256_ps(emm02);
temp_21 = _mm256_add_ps(temp_21, temp_31);
temp_22 = _mm256_add_ps(temp_22, temp_32);
temp_21 = _mm256_mul_ps(temp_21, x41);
temp_22 = _mm256_mul_ps(temp_22, x42);
avx_m256_t y1 = _mm256_add_ps(temp_11, temp_21);
avx_m256_t y2 = _mm256_add_ps(temp_12, temp_22);
*exp1 = _mm256_mul_ps(y1, pow2n1);
*exp2 = _mm256_mul_ps(y2, pow2n2);
} // newexp_ps_dual()
void
dump_surface(const char *filename, uint32_t binding_table_offset, int i)
{
struct surface s;
char *linear;
__m256i alpha;
get_surface(binding_table_offset, i, &s);
int png_format;
switch (s.format) {
case SF_R8G8B8X8_UNORM:
case SF_R8G8B8A8_UNORM:
case SF_R8G8B8X8_UNORM_SRGB:
case SF_R8G8B8A8_UNORM_SRGB:
png_format = PNG_FORMAT_RGBA;
break;
case SF_B8G8R8A8_UNORM:
case SF_B8G8R8X8_UNORM:
case SF_B8G8R8A8_UNORM_SRGB:
case SF_B8G8R8X8_UNORM_SRGB:
png_format = PNG_FORMAT_BGRA;
break;
default:
stub("image format");
return;
}
switch (s.format) {
case SF_R8G8B8X8_UNORM:
case SF_B8G8R8X8_UNORM:
case SF_R8G8B8X8_UNORM_SRGB:
case SF_B8G8R8X8_UNORM_SRGB:
alpha = _mm256_set1_epi32(0xff000000);
break;
default:
alpha = _mm256_set1_epi32(0);
break;
}
switch (s.tile_mode) {
case LINEAR:
linear = s.pixels;
break;
case XMAJOR:
linear = detile_xmajor(&s, alpha);
break;
case YMAJOR:
linear = detile_ymajor(&s, alpha);
break;
default:
linear = s.pixels;
stub("detile wmajor");
break;
}
FILE *f = fopen(filename, "wb");
ksim_assert(f != NULL);
png_image pi = {
.version = PNG_IMAGE_VERSION,
.width = s.width,
.height = s.height,
.format = png_format
};
ksim_assert(png_image_write_to_stdio(&pi, f, 0, linear, s.stride, NULL));
fclose(f);
if (linear != s.pixels)
free(linear);
}
static void
depth_test(struct primitive *p, struct dispatch *d)
{
uint32_t cpp = depth_format_size(gt.depth.format);
struct reg w_unorm;
struct reg d24x8, cmp, d_f;
void *base = ymajor_offset(p->depth.buffer, d->x, d->y, gt.depth.stride, cpp);
if (gt.depth.test_enable) {
const __m256 inv_scale = _mm256_set1_ps(1.0f / 16777215.0f);
switch (gt.depth.format) {
case D32_FLOAT:
d_f.reg = _mm256_load_ps(base);
break;
case D24_UNORM_X8_UINT:
d24x8.ireg = _mm256_load_si256(base);
d_f.reg = _mm256_mul_ps(_mm256_cvtepi32_ps(d24x8.ireg),
inv_scale);
break;
case D16_UNORM:
stub("D16_UNORM");
default:
ksim_unreachable("invalid depth format");
}
/* Swizzle two middle pixel pairs so that dword 0-3 and 4-7
* match the shader dispatch subspan orderingg. */
d_f.ireg = _mm256_permute4x64_epi64(d_f.ireg, SWIZZLE(0, 2, 1, 3));
switch (gt.depth.test_function) {
case COMPAREFUNCTION_ALWAYS:
cmp.reg = _mm256_cmp_ps(d_f.reg, d->w.reg, _CMP_TRUE_US);
break;
case COMPAREFUNCTION_NEVER:
cmp.reg = _mm256_cmp_ps(d_f.reg, d->w.reg, _CMP_FALSE_OS);
break;
case COMPAREFUNCTION_LESS:
cmp.reg = _mm256_cmp_ps(d_f.reg, d->w.reg, _CMP_LT_OS);
break;
case COMPAREFUNCTION_EQUAL:
cmp.reg = _mm256_cmp_ps(d_f.reg, d->w.reg, _CMP_EQ_OS);
break;
case COMPAREFUNCTION_LEQUAL:
cmp.reg = _mm256_cmp_ps(d_f.reg, d->w.reg, _CMP_LE_OS);
break;
case COMPAREFUNCTION_GREATER:
cmp.reg = _mm256_cmp_ps(d_f.reg, d->w.reg, _CMP_GT_OS);
break;
case COMPAREFUNCTION_NOTEQUAL:
cmp.reg = _mm256_cmp_ps(d_f.reg, d->w.reg, _CMP_NEQ_OS);
break;
case COMPAREFUNCTION_GEQUAL:
cmp.reg = _mm256_cmp_ps(d_f.reg, d->w.reg, _CMP_GE_OS);
break;
}
d->mask.ireg = _mm256_and_si256(cmp.ireg, d->mask.ireg);
}
if (gt.depth.write_enable) {
const __m256 scale = _mm256_set1_ps(16777215.0f);
const __m256 half = _mm256_set1_ps(0.5f);
struct reg w;
w.ireg = _mm256_permute4x64_epi64(d->w.ireg, SWIZZLE(0, 2, 1, 3));
__m256i m = _mm256_permute4x64_epi64(d->mask.ireg,
SWIZZLE(0, 2, 1, 3));
switch (gt.depth.format) {
case D32_FLOAT:
_mm256_maskstore_ps(base, m, w.reg);
break;
case D24_UNORM_X8_UINT:
w_unorm.ireg = _mm256_cvtps_epi32(_mm256_add_ps(_mm256_mul_ps(w.reg, scale), half));
_mm256_maskstore_epi32(base, m, w_unorm.ireg);
break;
case D16_UNORM:
stub("D16_UNORM");
default:
ksim_unreachable("invalid depth format");
}
}
}
INLINE const avxi min( const avxi& a, const avxi& b ) { return _mm256_cvtps_epi32(_mm256_min_ps(_mm256_cvtepi32_ps(a), _mm256_cvtepi32_ps(b))); }
/* evaluation of 8 sines at onces using AVX intrisics
The code is the exact rewriting of the cephes sinf function.
Precision is excellent as long as x < 8192 (I did not bother to
take into account the special handling they have for greater values
-- it does not return garbage for arguments over 8192, though, but
the extra precision is missing).
Note that it is such that sinf((float)M_PI) = 8.74e-8, which is the
surprising but correct result.
*/
v8sf sin256_ps(v8sf x) { // any x
v8sf xmm1, xmm2 = _mm256_setzero_ps(), xmm3, sign_bit, y;
v8si imm0, imm2;
#ifndef __AVX2__
v4si imm0_1, imm0_2;
v4si imm2_1, imm2_2;
#endif
sign_bit = x;
/* take the absolute value */
x = _mm256_and_ps(x, *(v8sf*)_ps256_inv_sign_mask);
/* extract the sign bit (upper one) */
sign_bit = _mm256_and_ps(sign_bit, *(v8sf*)_ps256_sign_mask);
/* scale by 4/Pi */
y = _mm256_mul_ps(x, *(v8sf*)_ps256_cephes_FOPI);
/*
Here we start a series of integer operations, which are in the
realm of AVX2.
If we don't have AVX, let's perform them using SSE2 directives
*/
#ifdef __AVX2__
/* store the integer part of y in mm0 */
imm2 = _mm256_cvttps_epi32(y);
/* j=(j+1) & (~1) (see the cephes sources) */
// another two AVX2 instruction
imm2 = _mm256_add_epi32(imm2, *(v8si*)_pi32_256_1);
imm2 = _mm256_and_si128(imm2, *(v8si*)_pi32_256_inv1);
y = _mm256_cvtepi32_ps(imm2);
/* get the swap sign flag */
imm0 = _mm256_and_si128(imm2, *(v8si*)_pi32_256_4);
imm0 = _mm256_slli_epi32(imm0, 29);
/* get the polynom selection mask
there is one polynom for 0 <= x <= Pi/4
and another one for Pi/4<x<=Pi/2
Both branches will be computed.
*/
imm2 = _mm256_and_si128(imm2, *(v8si*)_pi32_256_2);
imm2 = _mm256_cmpeq_epi32(imm2,*(v8si*)_pi32_256_0);
#else
/* we use SSE2 routines to perform the integer ops */
COPY_IMM_TO_XMM(_mm256_cvttps_epi32(y),imm2_1,imm2_2);
imm2_1 = _mm_add_epi32(imm2_1, *(v4si*)_pi32avx_1);
imm2_2 = _mm_add_epi32(imm2_2, *(v4si*)_pi32avx_1);
imm2_1 = _mm_and_si128(imm2_1, *(v4si*)_pi32avx_inv1);
imm2_2 = _mm_and_si128(imm2_2, *(v4si*)_pi32avx_inv1);
COPY_XMM_TO_IMM(imm2_1,imm2_2,imm2);
y = _mm256_cvtepi32_ps(imm2);
imm0_1 = _mm_and_si128(imm2_1, *(v4si*)_pi32avx_4);
imm0_2 = _mm_and_si128(imm2_2, *(v4si*)_pi32avx_4);
imm0_1 = _mm_slli_epi32(imm0_1, 29);
imm0_2 = _mm_slli_epi32(imm0_2, 29);
COPY_XMM_TO_IMM(imm0_1, imm0_2, imm0);
imm2_1 = _mm_and_si128(imm2_1, *(v4si*)_pi32avx_2);
imm2_2 = _mm_and_si128(imm2_2, *(v4si*)_pi32avx_2);
imm2_1 = _mm_cmpeq_epi32(imm2_1, _mm_setzero_si128());
imm2_2 = _mm_cmpeq_epi32(imm2_2, _mm_setzero_si128());
COPY_XMM_TO_IMM(imm2_1, imm2_2, imm2);
#endif
v8sf swap_sign_bit = _mm256_castsi256_ps(imm0);
v8sf poly_mask = _mm256_castsi256_ps(imm2);
sign_bit = _mm256_xor_ps(sign_bit, swap_sign_bit);
/* The magic pass: "******"
x = ((x - y * DP1) - y * DP2) - y * DP3; */
xmm1 = *(v8sf*)_ps256_minus_cephes_DP1;
xmm2 = *(v8sf*)_ps256_minus_cephes_DP2;
xmm3 = *(v8sf*)_ps256_minus_cephes_DP3;
xmm1 = _mm256_mul_ps(y, xmm1);
xmm2 = _mm256_mul_ps(y, xmm2);
xmm3 = _mm256_mul_ps(y, xmm3);
x = _mm256_add_ps(x, xmm1);
x = _mm256_add_ps(x, xmm2);
x = _mm256_add_ps(x, xmm3);
/* Evaluate the first polynom (0 <= x <= Pi/4) */
y = *(v8sf*)_ps256_coscof_p0;
v8sf z = _mm256_mul_ps(x,x);
y = _mm256_mul_ps(y, z);
y = _mm256_add_ps(y, *(v8sf*)_ps256_coscof_p1);
y = _mm256_mul_ps(y, z);
y = _mm256_add_ps(y, *(v8sf*)_ps256_coscof_p2);
y = _mm256_mul_ps(y, z);
y = _mm256_mul_ps(y, z);
v8sf tmp = _mm256_mul_ps(z, *(v8sf*)_ps256_0p5);
y = _mm256_sub_ps(y, tmp);
y = _mm256_add_ps(y, *(v8sf*)_ps256_1);
/* Evaluate the second polynom (Pi/4 <= x <= 0) */
v8sf y2 = *(v8sf*)_ps256_sincof_p0;
y2 = _mm256_mul_ps(y2, z);
y2 = _mm256_add_ps(y2, *(v8sf*)_ps256_sincof_p1);
y2 = _mm256_mul_ps(y2, z);
y2 = _mm256_add_ps(y2, *(v8sf*)_ps256_sincof_p2);
y2 = _mm256_mul_ps(y2, z);
y2 = _mm256_mul_ps(y2, x);
y2 = _mm256_add_ps(y2, x);
/* select the correct result from the two polynoms */
xmm3 = poly_mask;
y2 = _mm256_and_ps(xmm3, y2); //, xmm3);
y = _mm256_andnot_ps(xmm3, y);
y = _mm256_add_ps(y,y2);
/* update the sign */
y = _mm256_xor_ps(y, sign_bit);
return y;
}
void TransLut_FindIndexAvx2 <TransLut::MapperLog>::find_index (const TransLut::FloatIntMix val_arr [8], __m256i &index, __m256 &frac)
{
assert (val_arr != 0);
// Constants
static const int mant_size = 23;
static const int exp_bias = 127;
static const uint32_t base = (exp_bias + LOGLUT_MIN_L2) << mant_size;
static const float val_min = 1.0f / (int64_t (1) << -LOGLUT_MIN_L2);
// static const float val_max = float (int64_t (1) << LOGLUT_MAX_L2);
static const int frac_size = mant_size - LOGLUT_RES_L2;
static const uint32_t frac_mask = (1 << frac_size) - 1;
const __m256 zero_f = _mm256_setzero_ps ();
const __m256 one_f = _mm256_set1_ps (1);
const __m256 frac_mul = _mm256_set1_ps (1.0f / (1 << frac_size));
const __m256 mul_eps = _mm256_set1_ps (1.0f / val_min);
const __m256 mask_abs_f = _mm256_load_ps (
reinterpret_cast <const float *> (fstb::ToolsAvx2::_mask_abs)
);
const __m256i zero_i = _mm256_setzero_si256 ();
const __m256i mask_abs_epi32 = _mm256_set1_epi32 (0x7FFFFFFF);
const __m256i one_epi32 = _mm256_set1_epi32 (1);
const __m256i base_epi32 = _mm256_set1_epi32 (int (base));
const __m256i frac_mask_epi32 = _mm256_set1_epi32 (frac_mask);
const __m256i val_min_epi32 =
_mm256_set1_epi32 ((LOGLUT_MIN_L2 + exp_bias) << mant_size);
const __m256i val_max_epi32 =
_mm256_set1_epi32 ((LOGLUT_MAX_L2 + exp_bias) << mant_size);
const __m256i index_max_epi32 =
_mm256_set1_epi32 ((LOGLUT_MAX_L2 - LOGLUT_MIN_L2) << LOGLUT_RES_L2);
const __m256i hsize_epi32 = _mm256_set1_epi32 (LOGLUT_HSIZE);
const __m256i mirror_epi32 = _mm256_set1_epi32 (LOGLUT_HSIZE - 1);
// It really starts here
const __m256 val_f = _mm256_load_ps (reinterpret_cast <const float *> (val_arr));
const __m256 val_a = _mm256_and_ps (val_f, mask_abs_f);
const __m256i val_i = _mm256_load_si256 (reinterpret_cast <const __m256i *> (val_arr));
const __m256i val_u = _mm256_and_si256 (val_i, mask_abs_epi32);
// Standard path
__m256i index_std = _mm256_sub_epi32 (val_u, base_epi32);
index_std = _mm256_srli_epi32 (index_std, frac_size);
index_std = _mm256_add_epi32 (index_std, one_epi32);
__m256i frac_stdi = _mm256_and_si256 (val_u, frac_mask_epi32);
__m256 frac_std = _mm256_cvtepi32_ps (frac_stdi);
frac_std = _mm256_mul_ps (frac_std, frac_mul);
// Epsilon path
__m256 frac_eps = _mm256_max_ps (val_a, zero_f);
frac_eps = _mm256_mul_ps (frac_eps, mul_eps);
// Range cases
const __m256i eps_flag_i = _mm256_cmpgt_epi32 (val_min_epi32, val_u);
const __m256i std_flag_i = _mm256_cmpgt_epi32 (val_max_epi32, val_u);
const __m256 eps_flag_f = _mm256_castsi256_ps (eps_flag_i);
const __m256 std_flag_f = _mm256_castsi256_ps (std_flag_i);
__m256i index_tmp =
fstb::ToolsAvx2::select (std_flag_i, index_std, index_max_epi32);
__m256 frac_tmp =
fstb::ToolsAvx2::select (std_flag_f, frac_std, one_f);
index_tmp = fstb::ToolsAvx2::select (eps_flag_i, zero_i, index_tmp);
frac_tmp = fstb::ToolsAvx2::select (eps_flag_f, frac_eps, frac_tmp);
// Sign cases
const __m256i neg_flag_i = _mm256_srai_epi32 (val_i, 31);
const __m256 neg_flag_f = _mm256_castsi256_ps (neg_flag_i);
const __m256i index_neg = _mm256_sub_epi32 (mirror_epi32, index_tmp);
const __m256i index_pos = _mm256_add_epi32 (hsize_epi32, index_tmp);
const __m256 frac_neg = _mm256_sub_ps (one_f, frac_tmp);
index = fstb::ToolsAvx2::select (neg_flag_i, index_neg, index_pos);
frac = fstb::ToolsAvx2::select (neg_flag_f, frac_neg, frac_tmp);
}
//-----------------------------------------------------------------------------------------
// Rasterize the occludee AABB and depth test it against the CPU rasterized depth buffer
// If any of the rasterized AABB pixels passes the depth test exit early and mark the occludee
// as visible. If all rasterized AABB pixels are occluded then the occludee is culled
//-----------------------------------------------------------------------------------------
bool TransformedAABBoxAVX::RasterizeAndDepthTestAABBox(UINT *pRenderTargetPixels, const __m128 pXformedPos[], UINT idx)
{
// Set DAZ and FZ MXCSR bits to flush denormals to zero (i.e., make it faster)
// Denormal are zero (DAZ) is bit 6 and Flush to zero (FZ) is bit 15.
// so to enable the two to have to set bits 6 and 15 which 1000 0000 0100 0000 = 0x8040
_mm_setcsr( _mm_getcsr() | 0x8040 );
__m256i colOffset = _mm256_setr_epi32(0, 1, 2, 3, 0, 1, 2, 3);
__m256i rowOffset = _mm256_setr_epi32(0, 0, 0, 0, 1, 1, 1, 1);
float* pDepthBuffer = (float*)pRenderTargetPixels;
// Rasterize the AABB triangles 4 at a time
for(UINT i = 0; i < AABB_TRIANGLES; i += SSE)
{
vFloat4 xformedPos[3];
Gather(xformedPos, i, pXformedPos, idx);
// use fixed-point only for X and Y. Avoid work for Z and W.
__m128i fxPtX[3], fxPtY[3];
for(int m = 0; m < 3; m++)
{
fxPtX[m] = _mm_cvtps_epi32(xformedPos[m].X);
fxPtY[m] = _mm_cvtps_epi32(xformedPos[m].Y);
}
// Fab(x, y) = Ax + By + C = 0
// Fab(x, y) = (ya - yb)x + (xb - xa)y + (xa * yb - xb * ya) = 0
// Compute A = (ya - yb) for the 3 line segments that make up each triangle
__m128i A0 = _mm_sub_epi32(fxPtY[1], fxPtY[2]);
__m128i A1 = _mm_sub_epi32(fxPtY[2], fxPtY[0]);
__m128i A2 = _mm_sub_epi32(fxPtY[0], fxPtY[1]);
// Compute B = (xb - xa) for the 3 line segments that make up each triangle
__m128i B0 = _mm_sub_epi32(fxPtX[2], fxPtX[1]);
__m128i B1 = _mm_sub_epi32(fxPtX[0], fxPtX[2]);
__m128i B2 = _mm_sub_epi32(fxPtX[1], fxPtX[0]);
// Compute C = (xa * yb - xb * ya) for the 3 line segments that make up each triangle
__m128i C0 = _mm_sub_epi32(_mm_mullo_epi32(fxPtX[1], fxPtY[2]), _mm_mullo_epi32(fxPtX[2], fxPtY[1]));
__m128i C1 = _mm_sub_epi32(_mm_mullo_epi32(fxPtX[2], fxPtY[0]), _mm_mullo_epi32(fxPtX[0], fxPtY[2]));
__m128i C2 = _mm_sub_epi32(_mm_mullo_epi32(fxPtX[0], fxPtY[1]), _mm_mullo_epi32(fxPtX[1], fxPtY[0]));
// Compute triangle area
__m128i triArea = _mm_mullo_epi32(B2, A1);
triArea = _mm_sub_epi32(triArea, _mm_mullo_epi32(B1, A2));
__m128 oneOverTriArea = _mm_rcp_ps(_mm_cvtepi32_ps(triArea));
__m128 Z[3];
Z[0] = xformedPos[0].Z;
Z[1] = _mm_mul_ps(_mm_sub_ps(xformedPos[1].Z, Z[0]), oneOverTriArea);
Z[2] = _mm_mul_ps(_mm_sub_ps(xformedPos[2].Z, Z[0]), oneOverTriArea);
// Use bounding box traversal strategy to determine which pixels to rasterize
//__m128i startX = _mm_and_si128(HelperSSE::Max(HelperSSE::Min(HelperSSE::Min(fxPtX[0], fxPtX[1]), fxPtX[2]), _mm_set1_epi32(0)), _mm_set1_epi32(~1));
__m128i startX = _mm_and_si128(HelperSSE::Max(HelperSSE::Min(HelperSSE::Min(fxPtX[0], fxPtX[1]), fxPtX[2]), _mm_set1_epi32(0)), _mm_set1_epi32(~3));
__m128i endX = HelperSSE::Min(HelperSSE::Max(HelperSSE::Max(fxPtX[0], fxPtX[1]), fxPtX[2]), _mm_set1_epi32(SCREENW - 1));
__m128i startY = _mm_and_si128(HelperSSE::Max(HelperSSE::Min(HelperSSE::Min(fxPtY[0], fxPtY[1]), fxPtY[2]), _mm_set1_epi32(0)), _mm_set1_epi32(~1));
__m128i endY = HelperSSE::Min(HelperSSE::Max(HelperSSE::Max(fxPtY[0], fxPtY[1]), fxPtY[2]), _mm_set1_epi32(SCREENH - 1));
// Now we have 4 triangles set up. Rasterize them each individually.
for(int lane=0; lane < SSE; lane++)
{
// Skip triangle if area is zero
if(triArea.m128i_i32[lane] <= 0)
{
continue;
}
// Extract this triangle's properties from the SIMD versions
__m256 zz[3];
for (int vv = 0; vv < 3; vv++)
{
zz[vv] = _mm256_set1_ps(Z[vv].m128_f32[lane]);
}
int startXx = startX.m128i_i32[lane];
int endXx = endX.m128i_i32[lane];
int startYy = startY.m128i_i32[lane];
int endYy = endY.m128i_i32[lane];
__m256i aa0 = _mm256_set1_epi32(A0.m128i_i32[lane]);
__m256i aa1 = _mm256_set1_epi32(A1.m128i_i32[lane]);
__m256i aa2 = _mm256_set1_epi32(A2.m128i_i32[lane]);
__m256i bb0 = _mm256_set1_epi32(B0.m128i_i32[lane]);
__m256i bb1 = _mm256_set1_epi32(B1.m128i_i32[lane]);
__m256i bb2 = _mm256_set1_epi32(B2.m128i_i32[lane]);
__m256i aa0Inc = _mm256_slli_epi32(aa0, 2);
__m256i aa1Inc = _mm256_slli_epi32(aa1, 2);
__m256i aa2Inc = _mm256_slli_epi32(aa2, 2);
__m256i bb0Inc = _mm256_slli_epi32(bb0, 1);
__m256i bb1Inc = _mm256_slli_epi32(bb1, 1);
__m256i bb2Inc = _mm256_slli_epi32(bb2, 1);
__m256i row, col;
// Traverse pixels in 2x4 blocks and store 2x4 pixel quad depths contiguously in memory ==> 2*X
// This method provides better performance
int rowIdx = (startYy * SCREENW + 2 * startXx);
col = _mm256_add_epi32(colOffset, _mm256_set1_epi32(startXx));
__m256i aa0Col = _mm256_mullo_epi32(aa0, col);
__m256i aa1Col = _mm256_mullo_epi32(aa1, col);
__m256i aa2Col = _mm256_mullo_epi32(aa2, col);
row = _mm256_add_epi32(rowOffset, _mm256_set1_epi32(startYy));
__m256i bb0Row = _mm256_add_epi32(_mm256_mullo_epi32(bb0, row), _mm256_set1_epi32(C0.m128i_i32[lane]));
__m256i bb1Row = _mm256_add_epi32(_mm256_mullo_epi32(bb1, row), _mm256_set1_epi32(C1.m128i_i32[lane]));
__m256i bb2Row = _mm256_add_epi32(_mm256_mullo_epi32(bb2, row), _mm256_set1_epi32(C2.m128i_i32[lane]));
__m256i sum0Row = _mm256_add_epi32(aa0Col, bb0Row);
__m256i sum1Row = _mm256_add_epi32(aa1Col, bb1Row);
__m256i sum2Row = _mm256_add_epi32(aa2Col, bb2Row);
__m256 zx = _mm256_mul_ps(_mm256_cvtepi32_ps(aa1Inc), zz[1]);
zx = _mm256_add_ps(zx, _mm256_mul_ps(_mm256_cvtepi32_ps(aa2Inc), zz[2]));
// Incrementally compute Fab(x, y) for all the pixels inside the bounding box formed by (startX, endX) and (startY, endY)
for (int r = startYy; r < endYy; r += 2,
rowIdx += 2 * SCREENW,
sum0Row = _mm256_add_epi32(sum0Row, bb0Inc),
sum1Row = _mm256_add_epi32(sum1Row, bb1Inc),
sum2Row = _mm256_add_epi32(sum2Row, bb2Inc))
{
// Compute barycentric coordinates
int index = rowIdx;
__m256i alpha = sum0Row;
__m256i beta = sum1Row;
__m256i gama = sum2Row;
//Compute barycentric-interpolated depth
__m256 depth = zz[0];
depth = _mm256_add_ps(depth, _mm256_mul_ps(_mm256_cvtepi32_ps(beta), zz[1]));
depth = _mm256_add_ps(depth, _mm256_mul_ps(_mm256_cvtepi32_ps(gama), zz[2]));
__m256i anyOut = _mm256_setzero_si256();
for (int c = startXx; c < endXx; c += 4,
index += 8,
alpha = _mm256_add_epi32(alpha, aa0Inc),
beta = _mm256_add_epi32(beta, aa1Inc),
gama = _mm256_add_epi32(gama, aa2Inc),
depth = _mm256_add_ps(depth, zx))
{
//Test Pixel inside triangle
__m256i mask = _mm256_or_si256(_mm256_or_si256(alpha, beta), gama);
__m256 previousDepthValue = _mm256_loadu_ps(&pDepthBuffer[index]);
__m256 depthMask = _mm256_cmp_ps(depth, previousDepthValue, 0x1D);
__m256i finalMask = _mm256_andnot_si256(mask, _mm256_castps_si256(depthMask));
anyOut = _mm256_or_si256(anyOut, finalMask);
}//for each column
if (!_mm256_testz_si256(anyOut, _mm256_set1_epi32(0x80000000)))
{
return true; //early exit
}
}// for each row
}// for each triangle
}// for each set of SIMD# triangles
return false;
}
__m256 mm256_cos_ps(__m256 x) {
__m256 xmm1, xmm2 = _mm256_setzero_ps(), xmm3, y;
__m256i emm0, emm2;
/* take the absolute value */
x = _mm256_and_ps(x, *(__m256*)m256_ps_inv_sign_mask);
/* scale by 4/Pi */
y = _mm256_mul_ps(x, *(__m256*)m256_ps_cephes_FOPI);
/* store the integer part of y in mm0 */
emm2 = _mm256_cvttps_epi32(y);
/* j=(j+1) & (~1) (see the cephes sources) */
emm2 = _mm256_add_epi32(emm2, *(__m256i*)m256_pi32_1);
emm2 = _mm256_and_si256(emm2, *(__m256i*)m256_pi32_inv1);
y = _mm256_cvtepi32_ps(emm2);
emm2 = _mm256_sub_epi32(emm2, *(__m256i*)m256_pi32_2);
/* get the swap sign flag */
emm0 = _mm256_andnot_si256(emm2, *(__m256i*)m256_pi32_4);
emm0 = _mm256_slli_epi32(emm0, 29);
/* get the polynom selection mask */
emm2 = _mm256_and_si256(emm2, *(__m256i*)m256_pi32_2);
emm2 = _mm256_cmpeq_epi32(emm2, _mm256_setzero_si256());
__m256 sign_bit = _mm256_castsi256_ps(emm0);
__m256 poly_mask = _mm256_castsi256_ps(emm2);
/* The magic pass: "******"
x = ((x - y * DP1) - y * DP2) - y * DP3; */
xmm1 = *(__m256*)m256_ps_minus_cephes_DP1;
xmm2 = *(__m256*)m256_ps_minus_cephes_DP2;
xmm3 = *(__m256*)m256_ps_minus_cephes_DP3;
xmm1 = _mm256_mul_ps(y, xmm1);
xmm2 = _mm256_mul_ps(y, xmm2);
xmm3 = _mm256_mul_ps(y, xmm3);
x = _mm256_add_ps(x, xmm1);
x = _mm256_add_ps(x, xmm2);
x = _mm256_add_ps(x, xmm3);
/* Evaluate the first polynom (0 <= x <= Pi/4) */
y = *(__m256*)m256_ps_coscof_p0;
__m256 z = _mm256_mul_ps(x,x);
y = _mm256_mul_ps(y, z);
y = _mm256_add_ps(y, *(__m256*)m256_ps_coscof_p1);
y = _mm256_mul_ps(y, z);
y = _mm256_add_ps(y, *(__m256*)m256_ps_coscof_p2);
y = _mm256_mul_ps(y, z);
y = _mm256_mul_ps(y, z);
__m256 tmp = _mm256_mul_ps(z, *(__m256*)m256_ps_0p5);
y = _mm256_sub_ps(y, tmp);
y = _mm256_add_ps(y, *(__m256*)m256_ps_1);
/* Evaluate the second polynom (Pi/4 <= x <= 0) */
__m256 y2 = *(__m256*)m256_ps_sincof_p0;
y2 = _mm256_mul_ps(y2, z);
y2 = _mm256_add_ps(y2, *(__m256*)m256_ps_sincof_p1);
y2 = _mm256_mul_ps(y2, z);
y2 = _mm256_add_ps(y2, *(__m256*)m256_ps_sincof_p2);
y2 = _mm256_mul_ps(y2, z);
y2 = _mm256_mul_ps(y2, x);
y2 = _mm256_add_ps(y2, x);
/* select the correct result from the two polynoms */
xmm3 = poly_mask;
y2 = _mm256_and_ps(xmm3, y2); //, xmm3);
y = _mm256_andnot_ps(xmm3, y);
y = _mm256_add_ps(y,y2);
/* update the sign */
y = _mm256_xor_ps(y, sign_bit);
_mm256_zeroupper();
return y;
}
INLINE const avxi operator *( const avxi& a, const avxi& b ) { return _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(a), _mm256_cvtepi32_ps(b))); }
/* natural logarithm computed for 8 simultaneous float
return NaN for x <= 0
*/
v8sf log256_ps(v8sf x) {
v8si imm0;
v8sf one = *(v8sf*)_ps256_1;
//v8sf invalid_mask = _mm256_cmple_ps(x, _mm256_setzero_ps());
v8sf invalid_mask = _mm256_cmp_ps(x, _mm256_setzero_ps(), _CMP_LE_OS);
x = _mm256_max_ps(x, *(v8sf*)_ps256_min_norm_pos); /* cut off denormalized stuff */
// can be done with AVX2
imm0 = _mm256_srli_epi32(_mm256_castps_si256(x), 23);
/* keep only the fractional part */
x = _mm256_and_ps(x, *(v8sf*)_ps256_inv_mant_mask);
x = _mm256_or_ps(x, *(v8sf*)_ps256_0p5);
// this is again another AVX2 instruction
imm0 = _mm256_sub_epi32(imm0, *(v8si*)_pi32_256_0x7f);
v8sf e = _mm256_cvtepi32_ps(imm0);
e = _mm256_add_ps(e, one);
/* part2:
if( x < SQRTHF ) {
e -= 1;
x = x + x - 1.0;
} else { x = x - 1.0; }
*/
//v8sf mask = _mm256_cmplt_ps(x, *(v8sf*)_ps256_cephes_SQRTHF);
v8sf mask = _mm256_cmp_ps(x, *(v8sf*)_ps256_cephes_SQRTHF, _CMP_LT_OS);
v8sf tmp = _mm256_and_ps(x, mask);
x = _mm256_sub_ps(x, one);
e = _mm256_sub_ps(e, _mm256_and_ps(one, mask));
x = _mm256_add_ps(x, tmp);
v8sf z = _mm256_mul_ps(x,x);
v8sf y = *(v8sf*)_ps256_cephes_log_p0;
y = _mm256_mul_ps(y, x);
y = _mm256_add_ps(y, *(v8sf*)_ps256_cephes_log_p1);
y = _mm256_mul_ps(y, x);
y = _mm256_add_ps(y, *(v8sf*)_ps256_cephes_log_p2);
y = _mm256_mul_ps(y, x);
y = _mm256_add_ps(y, *(v8sf*)_ps256_cephes_log_p3);
y = _mm256_mul_ps(y, x);
y = _mm256_add_ps(y, *(v8sf*)_ps256_cephes_log_p4);
y = _mm256_mul_ps(y, x);
y = _mm256_add_ps(y, *(v8sf*)_ps256_cephes_log_p5);
y = _mm256_mul_ps(y, x);
y = _mm256_add_ps(y, *(v8sf*)_ps256_cephes_log_p6);
y = _mm256_mul_ps(y, x);
y = _mm256_add_ps(y, *(v8sf*)_ps256_cephes_log_p7);
y = _mm256_mul_ps(y, x);
y = _mm256_add_ps(y, *(v8sf*)_ps256_cephes_log_p8);
y = _mm256_mul_ps(y, x);
y = _mm256_mul_ps(y, z);
tmp = _mm256_mul_ps(e, *(v8sf*)_ps256_cephes_log_q1);
y = _mm256_add_ps(y, tmp);
tmp = _mm256_mul_ps(z, *(v8sf*)_ps256_0p5);
y = _mm256_sub_ps(y, tmp);
tmp = _mm256_mul_ps(e, *(v8sf*)_ps256_cephes_log_q2);
x = _mm256_add_ps(x, y);
x = _mm256_add_ps(x, tmp);
x = _mm256_or_ps(x, invalid_mask); // negative arg will be NAN
return x;
}
