void DoubleToComplex(double *srcI, double *srcQ, Complex *dst, const unsigned int len)
{
__m256d avxR_D, avxI_D, avxX_D, avxY_D, avxA_D, avxB_D;
__m128 avxA, avxB;
#if 1
__m256 avxD;
#endif
for (unsigned int i=0; i+4<=len; i+=4) {
avxR_D = _mm256_loadu_pd(srcI + i);
avxI_D = _mm256_loadu_pd(srcQ + i);
avxX_D = _mm256_unpacklo_pd(avxR_D, avxI_D); //swizzle
avxY_D = _mm256_unpackhi_pd(avxR_D, avxI_D);
avxA_D = _mm256_permute2f128_pd(avxX_D, avxY_D, 0x20);
avxB_D = _mm256_permute2f128_pd(avxX_D, avxY_D, 0x31);
avxA = _mm256_cvtpd_ps(avxA_D); //double to float
avxB = _mm256_cvtpd_ps(avxB_D);
#if 0
avxD = _mm256_castps128_ps256(avxA);
avxD = _mm256_insertf128_ps(avxD, avxB, 1);
_mm256_storeu_ps((float*)(dst+i), avxD);
#else
_mm_maskstore_ps((float*)(dst+i), _mm_set_epi32(SET_1, SET_1, SET_1, SET_1), avxA);
_mm_maskstore_ps((float*)(dst+i+2), _mm_set_epi32(SET_1, SET_1, SET_1, SET_1), avxB);
#endif
}
for (unsigned int i=len-(len & 0x03); i<len; ++i) {
dst[i].m_real = static_cast<float>(srcI[i]);
dst[i].m_imag = static_cast<float>(srcQ[i]);
}
}
double bst_compute_129_m256_maskstore_root_aligned( void*_bst_obj, double* p, double* q, size_t nn ) {
segments_t* mem = (segments_t*) _bst_obj;
int n, i, r, l_end, j, l_end_pre;
double t, e_tmp;
double* e = mem->e, *w = mem->w;
int* root = mem->r;
__m256d v_tmp;
__m256d v00, v01, v02, v03;
__m256d v10, v11, v12, v13;
__m256d v20, v21, v22, v23;
__m256d v30, v31, v32, v33;
__m256i v_cur_roots;
__m256 v_rootmask0, v_rootmask1;
// initialization
// mem->n = nn;
n = nn; // subtractions with n potentially negative. say hello to all the bugs
int idx1, idx1_root;
int idx2;
int idx3, idx3_root;
int pad_root, pad, pad_r;
idx1 = ((int) mem->e_sz) - 1;
idx1_root = ((int) mem->r_sz);
// the conventio is that iteration i, idx1 points to the first element of line i+1
e[idx1++] = q[n];
// pad contains the padding for row i+1
// for row n it's always 3
pad = 3;
pad_root = 7;
for (i = n-1; i >= 0; --i) {
idx1 -= 2*(n-i)+1 + pad;
idx1_root -= 2*(n-i)+1 + pad_root;
idx2 = idx1 + 1;
e[idx1] = q[i];
w[idx1] = q[i];
for (j = i+1; j < n+1; ++j,++idx2) {
e[idx2] = INFINITY;
w[idx2] = w[idx2-1] + p[j-1] + q[j];
}
idx2 += pad; // padding of line i+1
// idx2 now points to the first element of the next line
idx3 = idx1;
idx3_root = idx1_root;
pad_r = pad;
for (r = i; r < n; ++r) {
pad_r = (pad_r+1)&3; // padding of line r+1
idx1 = idx3;
idx1_root = idx3_root;
l_end = idx2 + (n-r);
// l_end points to the first entry after the current row
e_tmp = e[idx1++];
idx1_root++;
// calculate until a multiple of 8 doubles is left
// 8 = 4 * 2 128-bit vectors
l_end_pre = idx2 + ((n-r)&15);
for( ; (idx2 < l_end_pre) && (idx2 < l_end); ++idx2 ) {
t = e_tmp + e[idx2] + w[idx1];
if (t < e[idx1]) {
e[idx1] = t;
root[idx1_root] = r;
}
idx1++;
idx1_root++;
}
v_tmp = _mm256_set_pd( e_tmp, e_tmp, e_tmp, e_tmp );
// execute the shit for 4 vectors of size 2
v_cur_roots = _mm256_set_epi32(r, r, r, r, r, r, r, r);
for( ; idx2 < l_end; idx2 += 16 ) {
v01 = _mm256_load_pd( &w[idx1 ] );
v11 = _mm256_load_pd( &w[idx1+ 4] );
v21 = _mm256_load_pd( &w[idx1+ 8] );
v31 = _mm256_load_pd( &w[idx1+12] );
v00 = _mm256_load_pd( &e[idx2 ] );
v01 = _mm256_add_pd( v01, v_tmp );
v10 = _mm256_load_pd( &e[idx2+ 4] );
v11 = _mm256_add_pd( v11, v_tmp );
v20 = _mm256_load_pd( &e[idx2+ 8] );
v21 = _mm256_add_pd( v21, v_tmp );
v30 = _mm256_load_pd( &e[idx2+12] );
v31 = _mm256_add_pd( v31, v_tmp );
v01 = _mm256_add_pd( v01, v00 );
v03 = _mm256_load_pd( &e[idx1 ] );
v11 = _mm256_add_pd( v11, v10 );
v13 = _mm256_load_pd( &e[idx1+ 4] );
v21 = _mm256_add_pd( v21, v20 );
v23 = _mm256_load_pd( &e[idx1+ 8] );
v31 = _mm256_add_pd( v31, v30 );
v33 = _mm256_load_pd( &e[idx1+12] );
v02 = _mm256_cmp_pd( v01, v03, _CMP_LT_OQ );
v12 = _mm256_cmp_pd( v11, v13, _CMP_LT_OQ );
v22 = _mm256_cmp_pd( v21, v23, _CMP_LT_OQ );
v32 = _mm256_cmp_pd( v31, v33, _CMP_LT_OQ );
_mm256_maskstore_pd( &e[idx1 ],
_mm256_castpd_si256( v02 ), v01 );
_mm256_maskstore_pd( &e[idx1+ 4],
_mm256_castpd_si256( v12 ), v11 );
v_rootmask0 = _mm256_insertf128_ps(
_mm256_castps128_ps256(
_mm256_cvtpd_ps(v02)),
_mm256_cvtpd_ps(v12) , 1
);
_mm256_maskstore_pd( &e[idx1+ 8],
_mm256_castpd_si256( v22 ), v21 );
_mm256_maskstore_pd( &e[idx1+12],
_mm256_castpd_si256( v32 ), v31 );
v_rootmask1 = _mm256_insertf128_ps(
_mm256_castps128_ps256(
_mm256_cvtpd_ps(v22)),
_mm256_cvtpd_ps(v32) , 1
);
_mm256_maskstore_ps( &root[idx1_root ],
_mm256_castps_si256( v_rootmask0 ),
_mm256_castsi256_ps( v_cur_roots ) );
_mm256_maskstore_ps( &root[idx1_root + 8],
_mm256_castps_si256( v_rootmask1 ),
_mm256_castsi256_ps( v_cur_roots ) );
idx1 += 16;
idx1_root += 16;
}
idx2 += pad_r;
idx3++;
idx3_root++;
}
pad = (pad -1)&3;
pad_root = (pad_root-1)&7;
}
// the index of the last item of the first row is ((n/4)+1)*4-1, due to the padding
// if n is even, the total number of entries in the first
// row of the table is odd, so we need padding
return e[ ((n/4)+1)*4 - 1 ];
}
/**
* Calculate all values in one step per pixel. Requires grabbing the neighboring pixels.
*/
FORCE_INLINE double single_pixel(
double *im, int center, int top, int left, int right, int bottom,
const __m256i mask1110,
const __m256d rgb0W,
const __m256d onehalf,
const __m256d minustwelvehalf){
// double r = im[center];
// double g = im[center+1];
// double b = im[center+2];
// double r1 = im[top];
// double g1 = im[top+1];
// double b1 = im[top+2];
// double r2 = im[left];
// double g2 = im[left+1];
// double b2 = im[left+2];
// double r3 = im[right];
// double g3 = im[right+1];
// double b3 = im[right+2];
// double r4 = im[bottom];
// double g4 = im[bottom+1];
// double b4 = im[bottom+2];
__m256d c = _mm256_maskload_pd(&(im[center]),mask1110);
__m256d c1 = _mm256_loadu_pd(&(im[top]));
__m256d c2 = _mm256_loadu_pd(&(im[left]));
__m256d c3 = _mm256_loadu_pd(&(im[right]));
__m256d c4 = _mm256_loadu_pd(&(im[bottom]));
COST_INC_LOAD(20);
// double grey = rw * r + gw * g + bw * b;
// double grey1 = rw * r1 + gw * g1 + bw * b1;
// double grey2 = rw * r2 + gw * g2 + bw * b2;
// double grey3 = rw * r3 + gw * g3 + bw * b3;
// double grey4 = rw * r4 + gw * g4 + bw * b4;
__m256d greyc = _mm256_mul_pd(c,rgb0W);
__m256d grey1 = _mm256_mul_pd(c1,rgb0W);
__m256d grey2 = _mm256_mul_pd(c2,rgb0W);
__m256d grey3 = _mm256_mul_pd(c3,rgb0W);
__m256d grey4 = _mm256_mul_pd(c4,rgb0W);
//AVX: double: horizontal add for 1 vector
__m256d c_perm = _mm256_permute2f128_pd(c, c, 0b00100001);//1,2
__m256d c_h = _mm256_hadd_pd(c,c_perm);
__m128d c_h_lo = _mm256_extractf128_pd (c_h, 0);// lo
__m128d c_h_hi = _mm256_extractf128_pd (c_h, 1);// hi
double c_hsum_lo = _mm_cvtsd_f64(c_h_lo);
double c_hsum_hi = _mm_cvtsd_f64(c_h_hi);
double c_hsum = c_hsum_lo + c_hsum_hi;
//AVX: double: horizontal add for 1 vector
__m256d greyc_perm = _mm256_permute2f128_pd(greyc, greyc, 0b00100001);//1,2
__m256d greyc_h = _mm256_hadd_pd(greyc,greyc_perm);
__m128d greyc_h_lo = _mm256_extractf128_pd (greyc_h, 0);// lo
__m128d greyc_h_hi = _mm256_extractf128_pd (greyc_h, 1);// hi
double greyc_hsum_lo = _mm_cvtsd_f64(greyc_h_lo);
double greyc_hsum_hi = _mm_cvtsd_f64(greyc_h_hi);
double greyc_hsum = greyc_hsum_lo + greyc_hsum_hi;
//AVX: _m256d: horizontal add for 4 vectors at once
__m256d grey12 = _mm256_hadd_pd(grey1,grey2);
__m256d grey34 = _mm256_hadd_pd(grey3,grey4);
__m256d grey_1234_blend = _mm256_blend_pd(grey12, grey34, 0b1100); //0011
__m256d grey_1234_perm = _mm256_permute2f128_pd(grey12, grey34, 0b00100001);//1,2
__m256d grey_1234 = _mm256_add_pd(grey_1234_perm, grey_1234_blend);
//AVX: double: horizontal add for 1 vector
__m256d grey1234_perm = _mm256_permute2f128_pd(grey_1234, grey_1234, 0b00100001);//1,2
__m256d grey1234_h = _mm256_hadd_pd(grey_1234,grey1234_perm);
__m128d grey1234_h_lo = _mm256_extractf128_pd (grey1234_h, 0);// lo
__m128d grey1234_h_hi = _mm256_extractf128_pd (grey1234_h, 1);// hi
double grey1234_hsum_lo = _mm_cvtsd_f64(grey1234_h_lo);
double grey1234_hsum_hi = _mm_cvtsd_f64(grey1234_h_hi);
double grey1234_sum = grey1234_hsum_lo + grey1234_hsum_hi;
COST_INC_ADD(10); //+ operations wasted on AVX
COST_INC_MUL(15); //+ operations wasted on AVX
double mu = c_hsum / 3.0;
COST_INC_ADD(2);
COST_INC_DIV(1);
// double rmu = r-mu;
// double gmu = g-mu;
// double bmu = b-mu;
__m256d c_mu = _mm256_set1_pd(mu);
__m256d c_rgbmu = _mm256_sub_pd(c,c_mu);
COST_INC_ADD(3); //+1 operations wasted on AVX
// double rz = r-0.5;
// double gz = g-0.5;
// double bz = b-0.5;
__m256d c_rgbz = _mm256_sub_pd(c,onehalf);
COST_INC_ADD(3); //+1 operations wasted on AVX
// double rzrz = rz*rz;
// double gzgz = gz*gz;
// double bzbz = bz*bz;
__m256d c_rgbz_sq = _mm256_mul_pd(c_rgbz,c_rgbz);
COST_INC_MUL(3); //+1 operations wasted on AVX
// double re = exp(-12.5*rzrz);
// double ge = exp(-12.5*gzgz);
// double be = exp(-12.5*bzbz);
__m256d c_rgbe_tmp = _mm256_mul_pd(minustwelvehalf,c_rgbz_sq);
__m128 c_rgbe_tmp_ps = _mm256_cvtpd_ps(c_rgbe_tmp);
__m128 c_rgbe_ps = exp_ps(c_rgbe_tmp_ps);
__m256d c_rgbe = _mm256_cvtps_pd(c_rgbe_ps);
COST_INC_EXP(3);
COST_INC_MUL(3); //+1 operations wasted on AVX
// double t1 = sqrt((rmu*rmu + gmu*gmu + bmu*bmu)/3.0);
__m256d c_rgbmu_sq = _mm256_mul_pd(c_rgbmu,c_rgbmu);
__m128d t1_tmp1_lo = _mm256_extractf128_pd (c_rgbmu_sq, 0);// lo
__m128d t1_tmp1_hi = _mm256_extractf128_pd (c_rgbmu_sq, 1);// hi
__m128d t1_tmp1_lo_sum = _mm_hadd_pd (t1_tmp1_lo, t1_tmp1_lo);
double t1_tmp1_hi_lo = _mm_cvtsd_f64(t1_tmp1_hi);
double t1_tmp1_lo_sum_lo = _mm_cvtsd_f64(t1_tmp1_lo_sum);
double t1_tmp1 = t1_tmp1_lo_sum_lo + t1_tmp1_hi_lo;
double t1_tmp2 = t1_tmp1 / 3.0;
double t1 = sqrt(t1_tmp2);
COST_INC_SQRT(1);
COST_INC_ADD(3);
COST_INC_MUL(3); //+1 operations wasted on AVX
COST_INC_DIV(1);
double t2 = fabs(t1);
COST_INC_ABS(1);
// double t3 = re*ge*be;
__m128d t3_tmp1_lo = _mm256_extractf128_pd (c_rgbe, 0);// lo
__m128d t3_tmp1_hi = _mm256_extractf128_pd (c_rgbe, 1);// hi
double t3_tmp1_lo_lo = _mm_cvtsd_f64(t3_tmp1_lo);
double t3_tmp1_hi_lo = _mm_cvtsd_f64(t3_tmp1_hi);
__m128d t3_tmp1_lo_swapped = _mm_permute_pd(t3_tmp1_lo, 1);// swap
double t3_tmp1_lo_hi = _mm_cvtsd_f64(t3_tmp1_lo_swapped);
double t3 = t3_tmp1_lo_lo * t3_tmp1_lo_hi * t3_tmp1_hi_lo;
COST_INC_MUL(2);
double t4 = fabs(t3);
COST_INC_ABS(1);
double t5 = t2 * t4;
COST_INC_MUL(1);
// double t6 = -4.0*grey+grey1+grey2+grey3+grey4;
double minusfour_times_grey = -4.0*greyc_hsum;
double t6 = minusfour_times_grey+grey1234_sum;
COST_INC_MUL(1);
COST_INC_ADD(2); //2 operations saved due to AVX
double t7 = fabs(t6);
COST_INC_ABS(1);
double t8 = t5 * t7;
COST_INC_MUL(1);
double t9 = t8 + 1.0E-12;
COST_INC_ADD(1);
return t9;
}
