inline void avx2_xy_to_uv_f(__m256 x, __m256 y, __m256i& u, __m256i& v)
{
// Convert X,Y first into U,V space then round to nearest
// integer. That gets us close to correct answer, mapping XY to a
// lozenge-shaped space rather than hexagonal. We then correct the
// four regions that lie outside the hexagonal cell assigning them
// to their correct neighboring cell.
// Writer's note: see ~/Google Drive/Work/calin
// double dv = y*c_vy_inv;
// double du = x-dv*c_vx;
// u = std::lround(du);
// v = std::lround(dv);
// du -= u;
// dv -= v;
y = _mm256_mul_ps(y, calin::math::simd::c_m256(_c_m256_vy_inv));
x = _mm256_fnmadd_ps(y, calin::math::simd::c_m256(_c_m256_vx), x);
u = _mm256_cvtps_epi32(x);
v = _mm256_cvtps_epi32(y);
x = _mm256_sub_ps(x, _mm256_cvtepi32_ps(u));
y = _mm256_sub_ps(y, _mm256_cvtepi32_ps(v));
// double c3 = dv-du;
const __m256i c3 = _mm256_castps_si256(_mm256_sub_ps(y, x));
__m256i uvshift;
__m256i mask;
// double c1 = du+0.5*dv;
// double c2 = dv+0.5*du;
// if(c3<0) {
// if(c1>=1) u++;
// else if(c2<-1) v--;
// } else {
// if(c2>=1) v++;
// else if(c1<-1) u--;
// }
uvshift = _mm256_cvtps_epi32(_mm256_fmadd_ps(y, calin::math::simd::c_m256(_c_m256_one_half), x));
mask = _mm256_srai_epi32(_mm256_xor_si256(uvshift, c3), 31);
u = _mm256_blendv_epi8(u, _mm256_add_epi32(u, uvshift), mask);
uvshift = _mm256_cvtps_epi32(_mm256_fmadd_ps(x, calin::math::simd::c_m256(_c_m256_one_half), y));
mask = _mm256_srai_epi32(_mm256_xor_si256(uvshift, c3), 31);
v = _mm256_blendv_epi8(_mm256_add_epi32(v, uvshift), v, mask);
}
__m256 _inner_mm256_exp_ps1(__m256 arg)
{
arg = _mm256_mul_ps(arg, _mm256_set1_ps(1.4426950408889634073599246810018921374266459541529859f));
__m256i e = _mm256_add_epi32(
_mm256_castps_si256(_mm256_cmp_ps(arg, _mm256_set1_ps(0.0f), _CMP_LT_OQ)),
_mm256_cvttps_epi32(arg));
arg = _mm256_sub_ps(arg, _mm256_cvtepi32_ps(e));
__m256 intermediate_result;
intermediate_result = _mm256_fmadd_ps(_mm256_set1_ps(0.0136779459179717f), arg, _mm256_set1_ps(0.0517692205767896f));
intermediate_result = _mm256_fmadd_ps(intermediate_result, arg, _mm256_set1_ps(0.241554388295527f));
intermediate_result = _mm256_fmadd_ps(intermediate_result, arg, _mm256_set1_ps(0.692998430056128f));
intermediate_result = _mm256_fmadd_ps(intermediate_result, arg, _mm256_set1_ps(0.999999804292074f));
arg = intermediate_result;
__m256 res = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_add_epi32(e, _mm256_set1_epi32(127)), 23));
res = _mm256_mul_ps(res, arg);
return res;
}
void calculate_fma_float (unsigned char * out, double X0, double Y0, double scale, unsigned YSTART, unsigned SX, unsigned SY)
{
__m256 dd = _mm256_set1_ps ((float) scale);
__m256 XX0 = _mm256_set1_ps ((float) X0);
for (unsigned j = YSTART; j < SY; j++) {
__m256 y0 = _mm256_set1_ps (j*(float) scale + (float) Y0);
for (unsigned i = 0; i < SX; i += 8) {
__m256i ind = _mm256_setr_epi32 (i, i + 1, i + 2, i + 3, i + 4, i + 5, i + 6, i + 7);
__m256 x0 = _mm256_fmadd_ps (dd, _mm256_cvtepi32_ps (ind), XX0);
__m256 x = x0;
__m256 y = y0;
__m256i counts = _mm256_setzero_si256 ();
__m256i cmp_mask = _mm256_set1_epi32 (0xFFFFFFFFu);
for (unsigned n = 0; n < 255; n++) {
__m256 x2 = _mm256_mul_ps (x, x);
__m256 y2 = _mm256_mul_ps (y, y);
__m256 abs = _mm256_add_ps (x2, y2);
__m256i cmp = _mm256_castps_si256 (_mm256_cmp_ps (abs, _mm256_set1_ps (4), 1));
cmp_mask = _mm256_and_si256 (cmp_mask, cmp);
if (_mm256_testz_si256 (cmp_mask, cmp_mask)) {
break;
}
counts = _mm256_sub_epi32 (counts, cmp_mask);
__m256 t = _mm256_add_ps (x, x);
y = _mm256_fmadd_ps (t, y, y0);
x = _mm256_add_ps (_mm256_sub_ps (x2, y2), x0);
}
__m256i result = _mm256_shuffle_epi8 (counts, _mm256_setr_epi8 (0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12));
__m128i result128 = _128i_shuffle (_mm256_extractf128_si256 (result, 0), _mm256_extractf128_si256 (result, 1), 0, 0, 0, 0);
result128 = _mm_shuffle_epi32 (result128, combine_4_2bits (0, 2, 0, 2));
_mm_storel_epi64 ((__m128i*) out, result128);
out += 8;
}
}
}
template<index_t index> INLINE const avxi shuffle( const avxi& a ) {
return _mm256_castps_si256(_mm256_permute_ps(_mm256_castsi256_ps(a), _MM_SHUFFLE(index, index, index, index)));
}
INLINE avxi( const int& a ) : m256(_mm256_castps_si256(_mm256_broadcast_ss((const float*)&a))) {}
INLINE const avxi operator ^( const avxi& a, const avxi& b ) { return _mm256_castps_si256(_mm256_xor_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))); }
template<> INLINE const avxi shuffle<0, 1, 0, 1>( const avxi& b ) { return _mm256_castps_si256(_mm256_castpd_ps(_mm256_movedup_pd(_mm256_castps_pd(_mm256_castsi256_ps(b))))); }
INLINE explicit avxi( const int* const a ) : m256(_mm256_castps_si256(_mm256_loadu_ps((const float*)a))) {}
double bst_compute_129_m256_maskstore_root_aligned( void*_bst_obj, double* p, double* q, size_t nn ) {
segments_t* mem = (segments_t*) _bst_obj;
int n, i, r, l_end, j, l_end_pre;
double t, e_tmp;
double* e = mem->e, *w = mem->w;
int* root = mem->r;
__m256d v_tmp;
__m256d v00, v01, v02, v03;
__m256d v10, v11, v12, v13;
__m256d v20, v21, v22, v23;
__m256d v30, v31, v32, v33;
__m256i v_cur_roots;
__m256 v_rootmask0, v_rootmask1;
// initialization
// mem->n = nn;
n = nn; // subtractions with n potentially negative. say hello to all the bugs
int idx1, idx1_root;
int idx2;
int idx3, idx3_root;
int pad_root, pad, pad_r;
idx1 = ((int) mem->e_sz) - 1;
idx1_root = ((int) mem->r_sz);
// the conventio is that iteration i, idx1 points to the first element of line i+1
e[idx1++] = q[n];
// pad contains the padding for row i+1
// for row n it's always 3
pad = 3;
pad_root = 7;
for (i = n-1; i >= 0; --i) {
idx1 -= 2*(n-i)+1 + pad;
idx1_root -= 2*(n-i)+1 + pad_root;
idx2 = idx1 + 1;
e[idx1] = q[i];
w[idx1] = q[i];
for (j = i+1; j < n+1; ++j,++idx2) {
e[idx2] = INFINITY;
w[idx2] = w[idx2-1] + p[j-1] + q[j];
}
idx2 += pad; // padding of line i+1
// idx2 now points to the first element of the next line
idx3 = idx1;
idx3_root = idx1_root;
pad_r = pad;
for (r = i; r < n; ++r) {
pad_r = (pad_r+1)&3; // padding of line r+1
idx1 = idx3;
idx1_root = idx3_root;
l_end = idx2 + (n-r);
// l_end points to the first entry after the current row
e_tmp = e[idx1++];
idx1_root++;
// calculate until a multiple of 8 doubles is left
// 8 = 4 * 2 128-bit vectors
l_end_pre = idx2 + ((n-r)&15);
for( ; (idx2 < l_end_pre) && (idx2 < l_end); ++idx2 ) {
t = e_tmp + e[idx2] + w[idx1];
if (t < e[idx1]) {
e[idx1] = t;
root[idx1_root] = r;
}
idx1++;
idx1_root++;
}
v_tmp = _mm256_set_pd( e_tmp, e_tmp, e_tmp, e_tmp );
// execute the shit for 4 vectors of size 2
v_cur_roots = _mm256_set_epi32(r, r, r, r, r, r, r, r);
for( ; idx2 < l_end; idx2 += 16 ) {
v01 = _mm256_load_pd( &w[idx1 ] );
v11 = _mm256_load_pd( &w[idx1+ 4] );
v21 = _mm256_load_pd( &w[idx1+ 8] );
v31 = _mm256_load_pd( &w[idx1+12] );
v00 = _mm256_load_pd( &e[idx2 ] );
v01 = _mm256_add_pd( v01, v_tmp );
v10 = _mm256_load_pd( &e[idx2+ 4] );
v11 = _mm256_add_pd( v11, v_tmp );
v20 = _mm256_load_pd( &e[idx2+ 8] );
v21 = _mm256_add_pd( v21, v_tmp );
v30 = _mm256_load_pd( &e[idx2+12] );
v31 = _mm256_add_pd( v31, v_tmp );
v01 = _mm256_add_pd( v01, v00 );
v03 = _mm256_load_pd( &e[idx1 ] );
v11 = _mm256_add_pd( v11, v10 );
v13 = _mm256_load_pd( &e[idx1+ 4] );
v21 = _mm256_add_pd( v21, v20 );
v23 = _mm256_load_pd( &e[idx1+ 8] );
v31 = _mm256_add_pd( v31, v30 );
v33 = _mm256_load_pd( &e[idx1+12] );
v02 = _mm256_cmp_pd( v01, v03, _CMP_LT_OQ );
v12 = _mm256_cmp_pd( v11, v13, _CMP_LT_OQ );
v22 = _mm256_cmp_pd( v21, v23, _CMP_LT_OQ );
v32 = _mm256_cmp_pd( v31, v33, _CMP_LT_OQ );
_mm256_maskstore_pd( &e[idx1 ],
_mm256_castpd_si256( v02 ), v01 );
_mm256_maskstore_pd( &e[idx1+ 4],
_mm256_castpd_si256( v12 ), v11 );
v_rootmask0 = _mm256_insertf128_ps(
_mm256_castps128_ps256(
_mm256_cvtpd_ps(v02)),
_mm256_cvtpd_ps(v12) , 1
);
_mm256_maskstore_pd( &e[idx1+ 8],
_mm256_castpd_si256( v22 ), v21 );
_mm256_maskstore_pd( &e[idx1+12],
_mm256_castpd_si256( v32 ), v31 );
v_rootmask1 = _mm256_insertf128_ps(
_mm256_castps128_ps256(
_mm256_cvtpd_ps(v22)),
_mm256_cvtpd_ps(v32) , 1
);
_mm256_maskstore_ps( &root[idx1_root ],
_mm256_castps_si256( v_rootmask0 ),
_mm256_castsi256_ps( v_cur_roots ) );
_mm256_maskstore_ps( &root[idx1_root + 8],
_mm256_castps_si256( v_rootmask1 ),
_mm256_castsi256_ps( v_cur_roots ) );
idx1 += 16;
idx1_root += 16;
}
idx2 += pad_r;
idx3++;
idx3_root++;
}
pad = (pad -1)&3;
pad_root = (pad_root-1)&7;
}
// the index of the last item of the first row is ((n/4)+1)*4-1, due to the padding
// if n is even, the total number of entries in the first
// row of the table is odd, so we need padding
return e[ ((n/4)+1)*4 - 1 ];
}
INLINE __m256 _mm256_maskload_ps (float const *ptr, __m256 mask) {
return _mm256_maskload_ps(ptr, _mm256_castps_si256(mask));
}
inline void newexp_ps_dual(avx_m256_t x1, avx_m256_t x2, avx_m256_t* exp1, avx_m256_t* exp2) {
avx_m256_t one = _ps_1;
avx_m256_t zero = _ps_0;
x1 = _mm256_min_ps(x1, _ps_exp_hi);
x2 = _mm256_min_ps(x2, _ps_exp_hi);
x1 = _mm256_max_ps(x1, _ps_exp_lo);
x2 = _mm256_max_ps(x2, _ps_exp_lo);
avx_m256_t temp_21 = _mm256_mul_ps(x1, _ps_cephes_LOG2EF);
avx_m256_t temp_22 = _mm256_mul_ps(x2, _ps_cephes_LOG2EF);
temp_21 = _mm256_add_ps(temp_21, _ps_0p5);
temp_22 = _mm256_add_ps(temp_22, _ps_0p5);
avx_m256i_t emm01 = _mm256_cvttps_epi32(temp_21);
avx_m256i_t emm02 = _mm256_cvttps_epi32(temp_22);
avx_m256_t temp_11 = _mm256_cvtepi32_ps(emm01);
avx_m256_t temp_12 = _mm256_cvtepi32_ps(emm02);
avx_m256_t temp_31 = _mm256_sub_ps(temp_11, temp_21);
avx_m256_t temp_32 = _mm256_sub_ps(temp_12, temp_22);
avx_m256_t mask1 = _mm256_cmp_ps(temp_31, zero, _CMP_GT_OQ);
avx_m256_t mask2 = _mm256_cmp_ps(temp_32, zero, _CMP_GT_OQ);
mask1 = _mm256_and_ps(mask1, one);
mask2 = _mm256_and_ps(mask2, one);
temp_21 = _mm256_sub_ps(temp_11, mask1);
temp_22 = _mm256_sub_ps(temp_12, mask2);
emm01 = _mm256_cvttps_epi32(temp_21);
emm02 = _mm256_cvttps_epi32(temp_22);
temp_11 = _mm256_mul_ps(temp_21, _ps_cephes_exp_C12);
temp_12 = _mm256_mul_ps(temp_22, _ps_cephes_exp_C12);
x1 = _mm256_sub_ps(x1, temp_11);
x2 = _mm256_sub_ps(x2, temp_12);
avx_m256_t x21 = _mm256_mul_ps(x1, x1);
avx_m256_t x22 = _mm256_mul_ps(x2, x2);
avx_m256_t x31 = _mm256_mul_ps(x21, x1);
avx_m256_t x32 = _mm256_mul_ps(x22, x2);
avx_m256_t x41 = _mm256_mul_ps(x21, x21);
avx_m256_t x42 = _mm256_mul_ps(x22, x22);
temp_11 = _mm256_add_ps(x1, one);
temp_12 = _mm256_add_ps(x2, one);
temp_21 = _mm256_mul_ps(x21, _ps_cephes_exp_p5);
temp_22 = _mm256_mul_ps(x22, _ps_cephes_exp_p5);
temp_31 = _mm256_mul_ps(x31, _ps_cephes_exp_p4);
temp_32 = _mm256_mul_ps(x32, _ps_cephes_exp_p4);
temp_11 = _mm256_add_ps(temp_11, temp_21);
temp_12 = _mm256_add_ps(temp_12, temp_22);
temp_21 = _mm256_mul_ps(x31, _ps_cephes_exp_p0);
temp_22 = _mm256_mul_ps(x32, _ps_cephes_exp_p0);
temp_11 = _mm256_add_ps(temp_11, temp_31);
temp_12 = _mm256_add_ps(temp_12, temp_32);
avx_m256_t temp_41 = _mm256_mul_ps(x1, _ps_cephes_exp_p2);
avx_m256_t temp_42 = _mm256_mul_ps(x2, _ps_cephes_exp_p2);
temp_31 = _mm256_mul_ps(x21, _ps_cephes_exp_p1);
temp_32 = _mm256_mul_ps(x22, _ps_cephes_exp_p1);
//emm01 = _mm256_add_epi32(emm01, _pi32_0x7f);
//emm02 = _mm256_add_epi32(emm02, _pi32_0x7f);
emm01 = _mm256_castps_si256(_mm256_add_ps(_mm256_castsi256_ps(emm01), _mm256_castsi256_ps(_pi32_0x7f)));
emm02 = _mm256_castps_si256(_mm256_add_ps(_mm256_castsi256_ps(emm02), _mm256_castsi256_ps(_pi32_0x7f)));
temp_21 = _mm256_add_ps(temp_21, temp_31);
temp_22 = _mm256_add_ps(temp_22, temp_32);
temp_31 = _mm256_add_ps(temp_31, temp_41);
temp_32 = _mm256_add_ps(temp_32, temp_42);
//emm01 = _mm256_slli_epi32(emm01, 23);
__m128i emm0hi1 = _mm256_extractf128_si256(emm01, 0);
__m128i emm0lo1 = _mm256_extractf128_si256(emm01, 1);
emm0hi1 = _mm_slli_epi32(emm0hi1, 23);
emm0lo1 = _mm_slli_epi32(emm0lo1, 23);
emm01 = _mm256_insertf128_si256(emm01, emm0hi1, 0);
emm01 = _mm256_insertf128_si256(emm01, emm0lo1, 1);
//emm02 = _mm256_slli_epi32(emm02, 23);
__m128i emm0hi2 = _mm256_extractf128_si256(emm02, 0);
__m128i emm0lo2 = _mm256_extractf128_si256(emm02, 1);
emm0hi2 = _mm_slli_epi32(emm0hi2, 23);
emm0lo2 = _mm_slli_epi32(emm0lo2, 23);
emm02 = _mm256_insertf128_si256(emm02, emm0hi2, 0);
emm02 = _mm256_insertf128_si256(emm02, emm0lo2, 1);
avx_m256_t pow2n1 = _mm256_castsi256_ps(emm01);
avx_m256_t pow2n2 = _mm256_castsi256_ps(emm02);
temp_21 = _mm256_add_ps(temp_21, temp_31);
temp_22 = _mm256_add_ps(temp_22, temp_32);
temp_21 = _mm256_mul_ps(temp_21, x41);
temp_22 = _mm256_mul_ps(temp_22, x42);
avx_m256_t y1 = _mm256_add_ps(temp_11, temp_21);
avx_m256_t y2 = _mm256_add_ps(temp_12, temp_22);
*exp1 = _mm256_mul_ps(y1, pow2n1);
*exp2 = _mm256_mul_ps(y2, pow2n2);
} // newexp_ps_dual()
inline void newsincos_ps_dual(avx_m256_t x1, avx_m256_t x2, avx_m256_t *s1, avx_m256_t *s2,
avx_m256_t *c1, avx_m256_t *c2) {
avx_m256_t tempa = _ps_sign_mask;
avx_m256_t tempb = _ps_inv_sign_mask;
avx_m256_t sign_bit1 = _mm256_and_ps(x1, tempa);
avx_m256_t sign_bit2 = _mm256_and_ps(x2, tempa);
x1 = _mm256_and_ps(x1, tempb);
x2 = _mm256_and_ps(x2, tempb);
tempa = _ps_cephes_FOPI;
avx_m256_t y1 = _mm256_mul_ps(x1, tempa);
avx_m256_t y2 = _mm256_mul_ps(x2, tempa);
//avx_m256i_t emm21 = _mm256_cvttps_epi32(y1);
//avx_m256i_t emm22 = _mm256_cvttps_epi32(y2);
//emm21 = _mm256_add_epi32(emm21, _pi32_1);
//emm22 = _mm256_add_epi32(emm22, _pi32_1);
avx_m256i_t emm21 = _mm256_cvttps_epi32(_mm256_add_ps(y1, _ps_1));
avx_m256i_t emm22 = _mm256_cvttps_epi32(_mm256_add_ps(y2, _ps_1));
//emm21 = _mm256_and_si256(emm21, _pi32_inv1);
//emm22 = _mm256_and_si256(emm22, _pi32_inv1);
emm21 = _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(emm21), _mm256_castsi256_ps(_pi32_inv1)));
emm22 = _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(emm22), _mm256_castsi256_ps(_pi32_inv1)));
y1 = _mm256_cvtepi32_ps(emm21);
y2 = _mm256_cvtepi32_ps(emm22);
//avx_m256i_t tempia = _pi32_2;
//avx_m256i_t cos_emm21 = _mm256_sub_epi32(emm21, tempia);
//avx_m256i_t cos_emm22 = _mm256_sub_epi32(emm22, tempia);
avx_m256i_t cos_emm21 = _mm256_cvtps_epi32(_mm256_sub_ps(_mm256_cvtepi32_ps(emm21), _ps_2));
avx_m256i_t cos_emm22 = _mm256_cvtps_epi32(_mm256_sub_ps(_mm256_cvtepi32_ps(emm22), _ps_2));
//avx_m256i_t tempib = _pi32_4;
//avx_m256i_t emm01 = _mm256_and_si256(emm21, tempib);
//avx_m256i_t emm02 = _mm256_and_si256(emm22, tempib);
avx_m256i_t emm01 = _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(emm21),
_mm256_castsi256_ps(_pi32_4)));
avx_m256i_t emm02 = _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(emm22),
_mm256_castsi256_ps(_pi32_4)));
//avx_m256i_t cos_emm01 = _mm256_andnot_si256(cos_emm21, tempib);
//avx_m256i_t cos_emm02 = _mm256_andnot_si256(cos_emm22, tempib);
avx_m256i_t cos_emm01 = _mm256_castps_si256(_mm256_andnot_ps(_mm256_castsi256_ps(cos_emm21),
_mm256_castsi256_ps(_pi32_4)));
avx_m256i_t cos_emm02 = _mm256_castps_si256(_mm256_andnot_ps(_mm256_castsi256_ps(cos_emm22),
_mm256_castsi256_ps(_pi32_4)));
//emm01 = _mm256_slli_epi32(emm01, 29);
__m128i emm0hi1 = _mm256_extractf128_si256(emm01, 0);
__m128i emm0lo1 = _mm256_extractf128_si256(emm01, 1);
emm0hi1 = _mm_slli_epi32(emm0hi1, 29);
emm0lo1 = _mm_slli_epi32(emm0lo1, 29);
emm01 = _mm256_insertf128_si256(emm01, emm0hi1, 0);
emm01 = _mm256_insertf128_si256(emm01, emm0lo1, 1);
//emm02 = _mm256_slli_epi32(emm02, 29);
__m128i emm0hi2 = _mm256_extractf128_si256(emm02, 0);
__m128i emm0lo2 = _mm256_extractf128_si256(emm02, 1);
emm0hi2 = _mm_slli_epi32(emm0hi2, 29);
emm0lo2 = _mm_slli_epi32(emm0lo2, 29);
emm02 = _mm256_insertf128_si256(emm02, emm0hi1, 0);
emm02 = _mm256_insertf128_si256(emm02, emm0lo1, 1);
//cos_emm01 = _mm256_slli_epi32(cos_emm01, 29);
__m128i cos_emm0hi1 = _mm256_extractf128_si256(cos_emm01, 0);
__m128i cos_emm0lo1 = _mm256_extractf128_si256(cos_emm01, 1);
cos_emm0hi1 = _mm_slli_epi32(cos_emm0hi1, 29);
cos_emm0lo1 = _mm_slli_epi32(cos_emm0lo1, 29);
cos_emm01 = _mm256_insertf128_si256(cos_emm01, cos_emm0hi1, 0);
cos_emm01 = _mm256_insertf128_si256(cos_emm01, cos_emm0lo1, 1);
//cos_emm02 = _mm256_slli_epi32(cos_emm02, 29);
__m128i cos_emm0hi2 = _mm256_extractf128_si256(cos_emm02, 0);
__m128i cos_emm0lo2 = _mm256_extractf128_si256(cos_emm02, 1);
cos_emm0hi2 = _mm_slli_epi32(cos_emm0hi2, 29);
cos_emm0lo2 = _mm_slli_epi32(cos_emm0lo2, 29);
cos_emm02 = _mm256_insertf128_si256(cos_emm02, cos_emm0hi2, 0);
cos_emm02 = _mm256_insertf128_si256(cos_emm02, cos_emm0lo2, 1);
//tempia = _pi32_2;
//tempib = _mm256_setzero_si256();
//emm21 = _mm256_and_si256(emm21, tempia);
//emm22 = _mm256_and_si256(emm22, tempia);
emm21 = _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(emm21),
_mm256_castsi256_ps(_pi32_2)));
emm22 = _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(emm22),
_mm256_castsi256_ps(_pi32_2)));
//cos_emm21 = _mm256_and_si256(cos_emm21, tempia);
//cos_emm22 = _mm256_and_si256(cos_emm22, tempia);
cos_emm21 = _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(cos_emm21),
_mm256_castsi256_ps(_pi32_2)));
cos_emm22 = _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(cos_emm22),
_mm256_castsi256_ps(_pi32_2)));
//emm21 = _mm256_cmpeq_epi32(emm21, tempib);
//emm22 = _mm256_cmpeq_epi32(emm22, tempib);
emm21 = _mm256_castps_si256(_mm256_cmp_ps(_mm256_castsi256_ps(emm21), _mm256_setzero_ps(), _CMP_EQ_UQ));
emm22 = _mm256_castps_si256(_mm256_cmp_ps(_mm256_castsi256_ps(emm22), _mm256_setzero_ps(), _CMP_EQ_UQ));
//cos_emm21 = _mm256_cmpeq_epi32(cos_emm21, tempib);
//cos_emm22 = _mm256_cmpeq_epi32(cos_emm22, tempib);
cos_emm21 = _mm256_castps_si256(_mm256_cmp_ps(_mm256_castsi256_ps(cos_emm21), _mm256_setzero_ps(), _CMP_EQ_UQ));
cos_emm22 = _mm256_castps_si256(_mm256_cmp_ps(_mm256_castsi256_ps(cos_emm22), _mm256_setzero_ps(), _CMP_EQ_UQ));
avx_m256_t emm0f1 = _mm256_castsi256_ps(emm01);
avx_m256_t emm0f2 = _mm256_castsi256_ps(emm02);
avx_m256_t emm2f1 = _mm256_castsi256_ps(emm21);
avx_m256_t emm2f2 = _mm256_castsi256_ps(emm22);
avx_m256_t cos_emm0f1 = _mm256_castsi256_ps(cos_emm01);
avx_m256_t cos_emm0f2 = _mm256_castsi256_ps(cos_emm02);
avx_m256_t cos_emm2f1 = _mm256_castsi256_ps(cos_emm21);
avx_m256_t cos_emm2f2 = _mm256_castsi256_ps(cos_emm22);
sign_bit1 = _mm256_xor_ps(sign_bit1, emm0f1);
sign_bit2 = _mm256_xor_ps(sign_bit2, emm0f2);
tempa = _ps_minus_cephes_DP123;
tempb = _mm256_mul_ps(y2, tempa);
tempa = _mm256_mul_ps(y1, tempa);
x2 = _mm256_add_ps(x2, tempb);
x1 = _mm256_add_ps(x1, tempa);
avx_m256_t x21 = _mm256_mul_ps(x1, x1);
avx_m256_t x22 = _mm256_mul_ps(x2, x2);
avx_m256_t x31 = _mm256_mul_ps(x21, x1);
avx_m256_t x32 = _mm256_mul_ps(x22, x2);
avx_m256_t x41 = _mm256_mul_ps(x21, x21);
avx_m256_t x42 = _mm256_mul_ps(x22, x22);
tempa = _ps_coscof_p0;
tempb = _ps_sincof_p0;
y1 = _mm256_mul_ps(x21, tempa);
y2 = _mm256_mul_ps(x22, tempa);
avx_m256_t y21 = _mm256_mul_ps(x21, tempb);
avx_m256_t y22 = _mm256_mul_ps(x22, tempb);
tempa = _ps_coscof_p1;
tempb = _ps_sincof_p1;
y1 = _mm256_add_ps(y1, tempa);
y2 = _mm256_add_ps(y2, tempa);
y21 = _mm256_add_ps(y21, tempb);
y22 = _mm256_add_ps(y22, tempb);
y1 = _mm256_mul_ps(y1, x21);
y2 = _mm256_mul_ps(y2, x22);
y21 = _mm256_mul_ps(y21, x21);
y22 = _mm256_mul_ps(y22, x22);
tempa = _ps_coscof_p2;
tempb = _ps_sincof_p2;
y1 = _mm256_add_ps(y1, tempa);
y2 = _mm256_add_ps(y2, tempa);
y21 = _mm256_add_ps(y21, tempb);
y22 = _mm256_add_ps(y22, tempb);
y1 = _mm256_mul_ps(y1, x41);
y2 = _mm256_mul_ps(y2, x42);
y21 = _mm256_mul_ps(y21, x31);
y22 = _mm256_mul_ps(y22, x32);
tempa = _ps_0p5;
tempb = _ps_1;
avx_m256_t temp_21 = _mm256_mul_ps(x21, tempa);
avx_m256_t temp_22 = _mm256_mul_ps(x22, tempa);
y21 = _mm256_add_ps(y21, x1);
y22 = _mm256_add_ps(y22, x2);
temp_21 = _mm256_sub_ps(temp_21, tempb);
temp_22 = _mm256_sub_ps(temp_22, tempb);
y1 = _mm256_sub_ps(y1, temp_21);
y2 = _mm256_sub_ps(y2, temp_22);
avx_m256_t cos_y1 = y1;
avx_m256_t cos_y2 = y2;
avx_m256_t cos_y21 = y21;
avx_m256_t cos_y22 = y22;
y1 = _mm256_andnot_ps(emm2f1, y1);
y2 = _mm256_andnot_ps(emm2f2, y2);
cos_y1 = _mm256_andnot_ps(cos_emm2f1, cos_y1);
cos_y2 = _mm256_andnot_ps(cos_emm2f2, cos_y2);
y21 = _mm256_and_ps(emm2f1, y21);
y22 = _mm256_and_ps(emm2f2, y22);
cos_y21 = _mm256_and_ps(cos_emm2f1, cos_y21);
cos_y22 = _mm256_and_ps(cos_emm2f2, cos_y22);
y1 = _mm256_add_ps(y1, y21);
y2 = _mm256_add_ps(y2, y22);
cos_y1 = _mm256_add_ps(cos_y1, cos_y21);
cos_y2 = _mm256_add_ps(cos_y2, cos_y22);
*s1 = _mm256_xor_ps(y1, sign_bit1);
*s2 = _mm256_xor_ps(y2, sign_bit2);
*c1 = _mm256_xor_ps(cos_y1, cos_emm0f1);
*c2 = _mm256_xor_ps(cos_y2, cos_emm0f2);
} // newsincos_ps_dual()
INLINE avxi unpackhi( const avxi& a, const avxi& b ) { return _mm256_castps_si256(_mm256_unpackhi_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))); }
INLINE const avxi broadcast(const int* ptr) { return _mm256_castps_si256(_mm256_broadcast_ss((const float*)ptr)); }
INLINE const avxi select( const avxb& mask, const avxi& t, const avxi& f ) { return _mm256_castps_si256(_mm256_blendv_ps(_mm256_castsi256_ps(f), _mm256_castsi256_ps(t), mask)); }
INLINE void _mm256_maskstore_ps (float *ptr, __m256 mask, __m256 data) {
_mm256_maskstore_ps(ptr, _mm256_castps_si256(mask), data);
}
//-----------------------------------------------------------------------------------------
// Rasterize the occludee AABB and depth test it against the CPU rasterized depth buffer
// If any of the rasterized AABB pixels passes the depth test exit early and mark the occludee
// as visible. If all rasterized AABB pixels are occluded then the occludee is culled
//-----------------------------------------------------------------------------------------
bool TransformedAABBoxAVX::RasterizeAndDepthTestAABBox(UINT *pRenderTargetPixels, const __m128 pXformedPos[], UINT idx)
{
// Set DAZ and FZ MXCSR bits to flush denormals to zero (i.e., make it faster)
// Denormal are zero (DAZ) is bit 6 and Flush to zero (FZ) is bit 15.
// so to enable the two to have to set bits 6 and 15 which 1000 0000 0100 0000 = 0x8040
_mm_setcsr( _mm_getcsr() | 0x8040 );
__m256i colOffset = _mm256_setr_epi32(0, 1, 2, 3, 0, 1, 2, 3);
__m256i rowOffset = _mm256_setr_epi32(0, 0, 0, 0, 1, 1, 1, 1);
float* pDepthBuffer = (float*)pRenderTargetPixels;
// Rasterize the AABB triangles 4 at a time
for(UINT i = 0; i < AABB_TRIANGLES; i += SSE)
{
vFloat4 xformedPos[3];
Gather(xformedPos, i, pXformedPos, idx);
// use fixed-point only for X and Y. Avoid work for Z and W.
__m128i fxPtX[3], fxPtY[3];
for(int m = 0; m < 3; m++)
{
fxPtX[m] = _mm_cvtps_epi32(xformedPos[m].X);
fxPtY[m] = _mm_cvtps_epi32(xformedPos[m].Y);
}
// Fab(x, y) = Ax + By + C = 0
// Fab(x, y) = (ya - yb)x + (xb - xa)y + (xa * yb - xb * ya) = 0
// Compute A = (ya - yb) for the 3 line segments that make up each triangle
__m128i A0 = _mm_sub_epi32(fxPtY[1], fxPtY[2]);
__m128i A1 = _mm_sub_epi32(fxPtY[2], fxPtY[0]);
__m128i A2 = _mm_sub_epi32(fxPtY[0], fxPtY[1]);
// Compute B = (xb - xa) for the 3 line segments that make up each triangle
__m128i B0 = _mm_sub_epi32(fxPtX[2], fxPtX[1]);
__m128i B1 = _mm_sub_epi32(fxPtX[0], fxPtX[2]);
__m128i B2 = _mm_sub_epi32(fxPtX[1], fxPtX[0]);
// Compute C = (xa * yb - xb * ya) for the 3 line segments that make up each triangle
__m128i C0 = _mm_sub_epi32(_mm_mullo_epi32(fxPtX[1], fxPtY[2]), _mm_mullo_epi32(fxPtX[2], fxPtY[1]));
__m128i C1 = _mm_sub_epi32(_mm_mullo_epi32(fxPtX[2], fxPtY[0]), _mm_mullo_epi32(fxPtX[0], fxPtY[2]));
__m128i C2 = _mm_sub_epi32(_mm_mullo_epi32(fxPtX[0], fxPtY[1]), _mm_mullo_epi32(fxPtX[1], fxPtY[0]));
// Compute triangle area
__m128i triArea = _mm_mullo_epi32(B2, A1);
triArea = _mm_sub_epi32(triArea, _mm_mullo_epi32(B1, A2));
__m128 oneOverTriArea = _mm_rcp_ps(_mm_cvtepi32_ps(triArea));
__m128 Z[3];
Z[0] = xformedPos[0].Z;
Z[1] = _mm_mul_ps(_mm_sub_ps(xformedPos[1].Z, Z[0]), oneOverTriArea);
Z[2] = _mm_mul_ps(_mm_sub_ps(xformedPos[2].Z, Z[0]), oneOverTriArea);
// Use bounding box traversal strategy to determine which pixels to rasterize
//__m128i startX = _mm_and_si128(HelperSSE::Max(HelperSSE::Min(HelperSSE::Min(fxPtX[0], fxPtX[1]), fxPtX[2]), _mm_set1_epi32(0)), _mm_set1_epi32(~1));
__m128i startX = _mm_and_si128(HelperSSE::Max(HelperSSE::Min(HelperSSE::Min(fxPtX[0], fxPtX[1]), fxPtX[2]), _mm_set1_epi32(0)), _mm_set1_epi32(~3));
__m128i endX = HelperSSE::Min(HelperSSE::Max(HelperSSE::Max(fxPtX[0], fxPtX[1]), fxPtX[2]), _mm_set1_epi32(SCREENW - 1));
__m128i startY = _mm_and_si128(HelperSSE::Max(HelperSSE::Min(HelperSSE::Min(fxPtY[0], fxPtY[1]), fxPtY[2]), _mm_set1_epi32(0)), _mm_set1_epi32(~1));
__m128i endY = HelperSSE::Min(HelperSSE::Max(HelperSSE::Max(fxPtY[0], fxPtY[1]), fxPtY[2]), _mm_set1_epi32(SCREENH - 1));
// Now we have 4 triangles set up. Rasterize them each individually.
for(int lane=0; lane < SSE; lane++)
{
// Skip triangle if area is zero
if(triArea.m128i_i32[lane] <= 0)
{
continue;
}
// Extract this triangle's properties from the SIMD versions
__m256 zz[3];
for (int vv = 0; vv < 3; vv++)
{
zz[vv] = _mm256_set1_ps(Z[vv].m128_f32[lane]);
}
int startXx = startX.m128i_i32[lane];
int endXx = endX.m128i_i32[lane];
int startYy = startY.m128i_i32[lane];
int endYy = endY.m128i_i32[lane];
__m256i aa0 = _mm256_set1_epi32(A0.m128i_i32[lane]);
__m256i aa1 = _mm256_set1_epi32(A1.m128i_i32[lane]);
__m256i aa2 = _mm256_set1_epi32(A2.m128i_i32[lane]);
__m256i bb0 = _mm256_set1_epi32(B0.m128i_i32[lane]);
__m256i bb1 = _mm256_set1_epi32(B1.m128i_i32[lane]);
__m256i bb2 = _mm256_set1_epi32(B2.m128i_i32[lane]);
__m256i aa0Inc = _mm256_slli_epi32(aa0, 2);
__m256i aa1Inc = _mm256_slli_epi32(aa1, 2);
__m256i aa2Inc = _mm256_slli_epi32(aa2, 2);
__m256i bb0Inc = _mm256_slli_epi32(bb0, 1);
__m256i bb1Inc = _mm256_slli_epi32(bb1, 1);
__m256i bb2Inc = _mm256_slli_epi32(bb2, 1);
__m256i row, col;
// Traverse pixels in 2x4 blocks and store 2x4 pixel quad depths contiguously in memory ==> 2*X
// This method provides better performance
int rowIdx = (startYy * SCREENW + 2 * startXx);
col = _mm256_add_epi32(colOffset, _mm256_set1_epi32(startXx));
__m256i aa0Col = _mm256_mullo_epi32(aa0, col);
__m256i aa1Col = _mm256_mullo_epi32(aa1, col);
__m256i aa2Col = _mm256_mullo_epi32(aa2, col);
row = _mm256_add_epi32(rowOffset, _mm256_set1_epi32(startYy));
__m256i bb0Row = _mm256_add_epi32(_mm256_mullo_epi32(bb0, row), _mm256_set1_epi32(C0.m128i_i32[lane]));
__m256i bb1Row = _mm256_add_epi32(_mm256_mullo_epi32(bb1, row), _mm256_set1_epi32(C1.m128i_i32[lane]));
__m256i bb2Row = _mm256_add_epi32(_mm256_mullo_epi32(bb2, row), _mm256_set1_epi32(C2.m128i_i32[lane]));
__m256i sum0Row = _mm256_add_epi32(aa0Col, bb0Row);
__m256i sum1Row = _mm256_add_epi32(aa1Col, bb1Row);
__m256i sum2Row = _mm256_add_epi32(aa2Col, bb2Row);
__m256 zx = _mm256_mul_ps(_mm256_cvtepi32_ps(aa1Inc), zz[1]);
zx = _mm256_add_ps(zx, _mm256_mul_ps(_mm256_cvtepi32_ps(aa2Inc), zz[2]));
// Incrementally compute Fab(x, y) for all the pixels inside the bounding box formed by (startX, endX) and (startY, endY)
for (int r = startYy; r < endYy; r += 2,
rowIdx += 2 * SCREENW,
sum0Row = _mm256_add_epi32(sum0Row, bb0Inc),
sum1Row = _mm256_add_epi32(sum1Row, bb1Inc),
sum2Row = _mm256_add_epi32(sum2Row, bb2Inc))
{
// Compute barycentric coordinates
int index = rowIdx;
__m256i alpha = sum0Row;
__m256i beta = sum1Row;
__m256i gama = sum2Row;
//Compute barycentric-interpolated depth
__m256 depth = zz[0];
depth = _mm256_add_ps(depth, _mm256_mul_ps(_mm256_cvtepi32_ps(beta), zz[1]));
depth = _mm256_add_ps(depth, _mm256_mul_ps(_mm256_cvtepi32_ps(gama), zz[2]));
__m256i anyOut = _mm256_setzero_si256();
for (int c = startXx; c < endXx; c += 4,
index += 8,
alpha = _mm256_add_epi32(alpha, aa0Inc),
beta = _mm256_add_epi32(beta, aa1Inc),
gama = _mm256_add_epi32(gama, aa2Inc),
depth = _mm256_add_ps(depth, zx))
{
//Test Pixel inside triangle
__m256i mask = _mm256_or_si256(_mm256_or_si256(alpha, beta), gama);
__m256 previousDepthValue = _mm256_loadu_ps(&pDepthBuffer[index]);
__m256 depthMask = _mm256_cmp_ps(depth, previousDepthValue, 0x1D);
__m256i finalMask = _mm256_andnot_si256(mask, _mm256_castps_si256(depthMask));
anyOut = _mm256_or_si256(anyOut, finalMask);
}//for each column
if (!_mm256_testz_si256(anyOut, _mm256_set1_epi32(0x80000000)))
{
return true; //early exit
}
}// for each row
}// for each triangle
}// for each set of SIMD# triangles
return false;
}
inline void newsincos_ps(avx_m256_t x, avx_m256_t *s, avx_m256_t *c) {
avx_m256_t sign_bit = _mm256_and_ps(x, _ps_sign_mask);
x = _mm256_and_ps(x, _ps_inv_sign_mask);
avx_m256_t y = _mm256_mul_ps(x, _ps_cephes_FOPI);
//avx_m256i_t emm2 = _mm256_cvttps_epi32(y);
//emm2 = _mm256_add_epi32(emm2, _pi32_1);
avx_m256i_t emm2 = _mm256_cvttps_epi32(_mm256_add_ps(y, _ps_1));
//emm2 = _mm256_and_si256(emm2, _pi32_inv1);
emm2 = _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(emm2), _mm256_castsi256_ps(_pi32_inv1)));
y = _mm256_cvtepi32_ps(emm2);
//avx_m256i_t cos_emm2 = _mm256_sub_epi32(emm2, _pi32_2);
avx_m256i_t cos_emm2 = _mm256_cvtps_epi32(_mm256_sub_ps(_mm256_cvtepi32_ps(emm2), _ps_2));
//avx_m256i_t emm0 = _mm256_and_si256(emm2, _pi32_4);
avx_m256i_t emm0 = _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(emm2),
_mm256_castsi256_ps(_pi32_4)));
//avx_m256i_t cos_emm0 = _mm256_andnot_si256(cos_emm2, _pi32_4);
avx_m256i_t cos_emm0 = _mm256_castps_si256(_mm256_andnot_ps(_mm256_castsi256_ps(cos_emm2),
_mm256_castsi256_ps(_pi32_4)));
//emm0 = _mm256_slli_epi32(emm0, 29);
__m128i emm0hi = _mm256_extractf128_si256(emm0, 0);
__m128i emm0lo = _mm256_extractf128_si256(emm0, 1);
emm0hi = _mm_slli_epi32(emm0hi, 29);
emm0lo = _mm_slli_epi32(emm0lo, 29);
emm0 = _mm256_insertf128_si256(emm0, emm0hi, 0);
emm0 = _mm256_insertf128_si256(emm0, emm0lo, 1);
//cos_emm0 = _mm256_slli_epi32(cos_emm0, 29);
__m128i cos_emm0hi = _mm256_extractf128_si256(cos_emm0, 0);
__m128i cos_emm0lo = _mm256_extractf128_si256(cos_emm0, 1);
cos_emm0hi = _mm_slli_epi32(cos_emm0hi, 29);
cos_emm0lo = _mm_slli_epi32(cos_emm0lo, 29);
cos_emm0 = _mm256_insertf128_si256(cos_emm0, cos_emm0hi, 0);
cos_emm0 = _mm256_insertf128_si256(cos_emm0, cos_emm0lo, 1);
//emm2 = _mm256_and_si256(emm2, _pi32_2);
emm2 = _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(emm2),
_mm256_castsi256_ps(_pi32_2)));
//cos_emm2 = _mm256_and_si256(cos_emm2, _pi32_2);
cos_emm2 = _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(cos_emm2),
_mm256_castsi256_ps(_pi32_2)));
//emm2 = _mm256_cmpeq_epi32(emm2, _mm256_setzero_si256());
emm2 = _mm256_castps_si256(_mm256_cmp_ps(_mm256_castsi256_ps(emm2), _mm256_setzero_ps(), _CMP_EQ_UQ));
//cos_emm2 = _mm256_cmpeq_epi32(cos_emm2, _mm256_setzero_si256());
cos_emm2 = _mm256_castps_si256(_mm256_cmp_ps(_mm256_castsi256_ps(cos_emm2), _mm256_setzero_ps(), _CMP_EQ_UQ));
avx_m256_t emm0f = _mm256_castsi256_ps(emm0);
avx_m256_t emm2f = _mm256_castsi256_ps(emm2);
avx_m256_t cos_emm0f = _mm256_castsi256_ps(cos_emm0);
avx_m256_t cos_emm2f = _mm256_castsi256_ps(cos_emm2);
sign_bit = _mm256_xor_ps(sign_bit, emm0f);
avx_m256_t temp_2 = _ps_minus_cephes_DP123;
temp_2 = _mm256_mul_ps(y, temp_2);
x = _mm256_add_ps(x, temp_2);
avx_m256_t x2 = _mm256_mul_ps(x, x);
avx_m256_t x3 = _mm256_mul_ps(x2, x);
avx_m256_t x4 = _mm256_mul_ps(x2, x2);
y = _ps_coscof_p0;
avx_m256_t y2 = _ps_sincof_p0;
y = _mm256_mul_ps(y, x2);
y2 = _mm256_mul_ps(y2, x2);
y = _mm256_add_ps(y, _ps_coscof_p1);
y2 = _mm256_add_ps(y2, _ps_sincof_p1);
y = _mm256_mul_ps(y, x2);
y2 = _mm256_mul_ps(y2, x2);
y = _mm256_add_ps(y, _ps_coscof_p2);
y2 = _mm256_add_ps(y2, _ps_sincof_p2);
y = _mm256_mul_ps(y, x4);
y2 = _mm256_mul_ps(y2, x3);
temp_2 = _mm256_mul_ps(x2, _ps_0p5);
y2 = _mm256_add_ps(y2, x);
temp_2 = _mm256_sub_ps(temp_2, _ps_1);
y = _mm256_sub_ps(y, temp_2);
avx_m256_t cos_y = y;
avx_m256_t cos_y2 = y2;
y = _mm256_andnot_ps(emm2f, y);
cos_y = _mm256_andnot_ps(cos_emm2f, cos_y);
y2 = _mm256_and_ps(emm2f, y2);
cos_y2 = _mm256_and_ps(cos_emm2f, cos_y2);
y = _mm256_add_ps(y, y2);
cos_y = _mm256_add_ps(cos_y, cos_y2);
*s = _mm256_xor_ps(y, sign_bit);
*c = _mm256_xor_ps(cos_y, cos_emm0f);
} // newsincos_ps()
template<index_t index_0, index_t index_1, index_t index_2, index_t index_3> INLINE const avxi shuffle( const avxi& a, const avxi& b ) {
return _mm256_castps_si256(_mm256_shuffle_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b), _MM_SHUFFLE(index_3, index_2, index_1, index_0)));
}
inline avx_m256_t newexp_ps(avx_m256_t x) {
avx_m256_t one = _ps_1;
avx_m256_t zero = _ps_0;
x = _mm256_min_ps(x, _ps_exp_hi);
x = _mm256_max_ps(x, _ps_exp_lo);
avx_m256_t temp_2 = _mm256_mul_ps(x, _ps_cephes_LOG2EF);
temp_2 = _mm256_add_ps(temp_2, _ps_0p5);
avx_m256i_t emm0 = _mm256_cvttps_epi32(temp_2);
avx_m256_t temp_1 = _mm256_cvtepi32_ps(emm0);
avx_m256_t temp_3 = _mm256_sub_ps(temp_1, temp_2);
avx_m256_t mask = _mm256_cmp_ps(temp_3, zero, _CMP_GT_OQ);
mask = _mm256_and_ps(mask, one);
temp_2 = _mm256_sub_ps(temp_1, mask);
emm0 = _mm256_cvttps_epi32(temp_2);
temp_1 = _mm256_mul_ps(temp_2, _ps_cephes_exp_C12);
x = _mm256_sub_ps(x, temp_1);
avx_m256_t x2 = _mm256_mul_ps(x, x);
avx_m256_t x3 = _mm256_mul_ps(x2, x);
avx_m256_t x4 = _mm256_mul_ps(x2, x2);
temp_1 = _mm256_add_ps(x, one);
temp_2 = _mm256_mul_ps(x2, _ps_cephes_exp_p5);
temp_3 = _mm256_mul_ps(x3, _ps_cephes_exp_p4);
temp_1 = _mm256_add_ps(temp_1, temp_2);
temp_2 = _mm256_mul_ps(x3, _ps_cephes_exp_p0);
temp_1 = _mm256_add_ps(temp_1, temp_3);
avx_m256_t temp_4 = _mm256_mul_ps(x, _ps_cephes_exp_p2);
temp_3 = _mm256_mul_ps(x2, _ps_cephes_exp_p1);
emm0 = _mm256_castps_si256(_mm256_add_ps(_mm256_castsi256_ps(emm0), _mm256_castsi256_ps(_pi32_0x7f)));
temp_2 = _mm256_add_ps(temp_2, temp_3);
temp_3 = _mm256_add_ps(temp_3, temp_4);
//emm0 = _mm256_slli_epi32(emm0, 23);
// convert emm0 into two 128-bit integer vectors
// perform shift on both vectors
// combine both vectors into 256-bit emm0
__m128i emm0hi = _mm256_extractf128_si256(emm0, 0);
__m128i emm0lo = _mm256_extractf128_si256(emm0, 1);
emm0hi = _mm_slli_epi32(emm0hi, 23);
emm0lo = _mm_slli_epi32(emm0lo, 23);
emm0 = _mm256_insertf128_si256(emm0, emm0hi, 0);
emm0 = _mm256_insertf128_si256(emm0, emm0lo, 1);
avx_m256_t pow2n = _mm256_castsi256_ps(emm0);
temp_2 = _mm256_add_ps(temp_2, temp_3);
temp_2 = _mm256_mul_ps(temp_2, x4);
avx_m256_t y = _mm256_add_ps(temp_1, temp_2);
y = _mm256_mul_ps(y, pow2n);
return y;
} // newexp_ps()
template<> INLINE const avxi shuffle<0, 0, 2, 2>( const avxi& b ) { return _mm256_castps_si256(_mm256_moveldup_ps(_mm256_castsi256_ps(b))); }
template<> INLINE const avxi shuffle<1, 1, 3, 3>( const avxi& b ) { return _mm256_castps_si256(_mm256_movehdup_ps(_mm256_castsi256_ps(b))); }
/* natural logarithm computed for 8 simultaneous float
return NaN for x <= 0
*/
v8sf log256_ps(v8sf x) {
v8si imm0;
v8sf one = *(v8sf*)_ps256_1;
//v8sf invalid_mask = _mm256_cmple_ps(x, _mm256_setzero_ps());
v8sf invalid_mask = _mm256_cmp_ps(x, _mm256_setzero_ps(), _CMP_LE_OS);
x = _mm256_max_ps(x, *(v8sf*)_ps256_min_norm_pos); /* cut off denormalized stuff */
// can be done with AVX2
imm0 = _mm256_srli_epi32(_mm256_castps_si256(x), 23);
/* keep only the fractional part */
x = _mm256_and_ps(x, *(v8sf*)_ps256_inv_mant_mask);
x = _mm256_or_ps(x, *(v8sf*)_ps256_0p5);
// this is again another AVX2 instruction
imm0 = _mm256_sub_epi32(imm0, *(v8si*)_pi32_256_0x7f);
v8sf e = _mm256_cvtepi32_ps(imm0);
e = _mm256_add_ps(e, one);
/* part2:
if( x < SQRTHF ) {
e -= 1;
x = x + x - 1.0;
} else { x = x - 1.0; }
*/
//v8sf mask = _mm256_cmplt_ps(x, *(v8sf*)_ps256_cephes_SQRTHF);
v8sf mask = _mm256_cmp_ps(x, *(v8sf*)_ps256_cephes_SQRTHF, _CMP_LT_OS);
v8sf tmp = _mm256_and_ps(x, mask);
x = _mm256_sub_ps(x, one);
e = _mm256_sub_ps(e, _mm256_and_ps(one, mask));
x = _mm256_add_ps(x, tmp);
v8sf z = _mm256_mul_ps(x,x);
v8sf y = *(v8sf*)_ps256_cephes_log_p0;
y = _mm256_mul_ps(y, x);
y = _mm256_add_ps(y, *(v8sf*)_ps256_cephes_log_p1);
y = _mm256_mul_ps(y, x);
y = _mm256_add_ps(y, *(v8sf*)_ps256_cephes_log_p2);
y = _mm256_mul_ps(y, x);
y = _mm256_add_ps(y, *(v8sf*)_ps256_cephes_log_p3);
y = _mm256_mul_ps(y, x);
y = _mm256_add_ps(y, *(v8sf*)_ps256_cephes_log_p4);
y = _mm256_mul_ps(y, x);
y = _mm256_add_ps(y, *(v8sf*)_ps256_cephes_log_p5);
y = _mm256_mul_ps(y, x);
y = _mm256_add_ps(y, *(v8sf*)_ps256_cephes_log_p6);
y = _mm256_mul_ps(y, x);
y = _mm256_add_ps(y, *(v8sf*)_ps256_cephes_log_p7);
y = _mm256_mul_ps(y, x);
y = _mm256_add_ps(y, *(v8sf*)_ps256_cephes_log_p8);
y = _mm256_mul_ps(y, x);
y = _mm256_mul_ps(y, z);
tmp = _mm256_mul_ps(e, *(v8sf*)_ps256_cephes_log_q1);
y = _mm256_add_ps(y, tmp);
tmp = _mm256_mul_ps(z, *(v8sf*)_ps256_0p5);
y = _mm256_sub_ps(y, tmp);
tmp = _mm256_mul_ps(e, *(v8sf*)_ps256_cephes_log_q2);
x = _mm256_add_ps(x, y);
x = _mm256_add_ps(x, tmp);
x = _mm256_or_ps(x, invalid_mask); // negative arg will be NAN
return x;
}