__m128d test_mm_cmple_pd(__m128d A, __m128d B) {
// DAG-LABEL: test_mm_cmple_pd
// DAG: call <2 x double> @llvm.x86.sse2.cmp.pd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i8 2)
//
// ASM-LABEL: test_mm_cmple_pd
// ASM: cmplepd
return _mm_cmple_pd(A, B);
}
void _SIMD_cmpgt_pd(__SIMDd a, __SIMDd b, void** resultPtr)
{
__SIMDd* result = (__SIMDd*)malloc(sizeof(__SIMDd));
*resultPtr = result;
#ifdef USE_SSE
*result = _mm_cmple_pd(a,b);
#elif defined USE_AVX
*result = _mm256_cmp(a,b,30);
#elif defined USE_IBM
*result = vec_cmpgt(a,b);
#endif
}
BI_FORCE_INLINE inline sse_double operator<=(const sse_double& o1,
const sse_double& o2) {
sse_double res;
res.packed = _mm_cmple_pd(o1.packed, o2.packed);
return res;
}
int
calc_gb_rad_still_sse2_double(t_commrec *cr, t_forcerec *fr,
int natoms, gmx_localtop_t *top,
const t_atomtypes *atype, double *x, t_nblist *nl,
gmx_genborn_t *born)
{
int i,k,n,ii,is3,ii3,nj0,nj1,offset;
int jnrA,jnrB,j3A,j3B;
int *mdtype;
double shX,shY,shZ;
int *jjnr;
double *shiftvec;
double gpi_ai,gpi2;
double factor;
double *gb_radius;
double *vsolv;
double *work;
double *dadx;
__m128d ix,iy,iz;
__m128d jx,jy,jz;
__m128d dx,dy,dz;
__m128d tx,ty,tz;
__m128d rsq,rinv,rinv2,rinv4,rinv6;
__m128d ratio,gpi,rai,raj,vai,vaj,rvdw;
__m128d ccf,dccf,theta,cosq,term,sinq,res,prod,prod_ai,tmp;
__m128d mask,icf4,icf6,mask_cmp;
const __m128d half = _mm_set1_pd(0.5);
const __m128d three = _mm_set1_pd(3.0);
const __m128d one = _mm_set1_pd(1.0);
const __m128d two = _mm_set1_pd(2.0);
const __m128d zero = _mm_set1_pd(0.0);
const __m128d four = _mm_set1_pd(4.0);
const __m128d still_p5inv = _mm_set1_pd(STILL_P5INV);
const __m128d still_pip5 = _mm_set1_pd(STILL_PIP5);
const __m128d still_p4 = _mm_set1_pd(STILL_P4);
factor = 0.5 * ONE_4PI_EPS0;
gb_radius = born->gb_radius;
vsolv = born->vsolv;
work = born->gpol_still_work;
jjnr = nl->jjnr;
shiftvec = fr->shift_vec[0];
dadx = fr->dadx;
jnrA = jnrB = 0;
jx = _mm_setzero_pd();
jy = _mm_setzero_pd();
jz = _mm_setzero_pd();
n = 0;
for(i=0;i<natoms;i++)
{
work[i]=0;
}
for(i=0;i<nl->nri;i++)
{
ii = nl->iinr[i];
ii3 = ii*3;
is3 = 3*nl->shift[i];
shX = shiftvec[is3];
shY = shiftvec[is3+1];
shZ = shiftvec[is3+2];
nj0 = nl->jindex[i];
nj1 = nl->jindex[i+1];
ix = _mm_set1_pd(shX+x[ii3+0]);
iy = _mm_set1_pd(shY+x[ii3+1]);
iz = _mm_set1_pd(shZ+x[ii3+2]);
/* Polarization energy for atom ai */
gpi = _mm_setzero_pd();
rai = _mm_load1_pd(gb_radius+ii);
prod_ai = _mm_set1_pd(STILL_P4*vsolv[ii]);
for(k=nj0;k<nj1-1;k+=2)
{
jnrA = jjnr[k];
jnrB = jjnr[k+1];
j3A = 3*jnrA;
j3B = 3*jnrB;
GMX_MM_LOAD_1RVEC_2POINTERS_PD(x+j3A,x+j3B,jx,jy,jz);
GMX_MM_LOAD_2VALUES_PD(gb_radius+jnrA,gb_radius+jnrB,raj);
GMX_MM_LOAD_2VALUES_PD(vsolv+jnrA,vsolv+jnrB,vaj);
dx = _mm_sub_pd(ix,jx);
dy = _mm_sub_pd(iy,jy);
dz = _mm_sub_pd(iz,jz);
rsq = gmx_mm_calc_rsq_pd(dx,dy,dz);
rinv = gmx_mm_invsqrt_pd(rsq);
rinv2 = _mm_mul_pd(rinv,rinv);
rinv4 = _mm_mul_pd(rinv2,rinv2);
rinv6 = _mm_mul_pd(rinv4,rinv2);
rvdw = _mm_add_pd(rai,raj);
ratio = _mm_mul_pd(rsq, gmx_mm_inv_pd( _mm_mul_pd(rvdw,rvdw)));
mask_cmp = _mm_cmple_pd(ratio,still_p5inv);
/* gmx_mm_sincos_pd() is quite expensive, so avoid calculating it if we can! */
if( 0 == _mm_movemask_pd(mask_cmp) )
{
/* if ratio>still_p5inv for ALL elements */
ccf = one;
dccf = _mm_setzero_pd();
}
else
{
ratio = _mm_min_pd(ratio,still_p5inv);
theta = _mm_mul_pd(ratio,still_pip5);
gmx_mm_sincos_pd(theta,&sinq,&cosq);
term = _mm_mul_pd(half,_mm_sub_pd(one,cosq));
ccf = _mm_mul_pd(term,term);
dccf = _mm_mul_pd(_mm_mul_pd(two,term),
_mm_mul_pd(sinq,theta));
}
prod = _mm_mul_pd(still_p4,vaj);
icf4 = _mm_mul_pd(ccf,rinv4);
icf6 = _mm_mul_pd( _mm_sub_pd( _mm_mul_pd(four,ccf),dccf), rinv6);
GMX_MM_INCREMENT_2VALUES_PD(work+jnrA,work+jnrB,_mm_mul_pd(prod_ai,icf4));
gpi = _mm_add_pd(gpi, _mm_mul_pd(prod,icf4) );
_mm_store_pd(dadx,_mm_mul_pd(prod,icf6));
dadx+=2;
_mm_store_pd(dadx,_mm_mul_pd(prod_ai,icf6));
dadx+=2;
}
if(k<nj1)
{
jnrA = jjnr[k];
j3A = 3*jnrA;
GMX_MM_LOAD_1RVEC_1POINTER_PD(x+j3A,jx,jy,jz);
GMX_MM_LOAD_1VALUE_PD(gb_radius+jnrA,raj);
GMX_MM_LOAD_1VALUE_PD(vsolv+jnrA,vaj);
dx = _mm_sub_sd(ix,jx);
dy = _mm_sub_sd(iy,jy);
dz = _mm_sub_sd(iz,jz);
rsq = gmx_mm_calc_rsq_pd(dx,dy,dz);
rinv = gmx_mm_invsqrt_pd(rsq);
rinv2 = _mm_mul_sd(rinv,rinv);
rinv4 = _mm_mul_sd(rinv2,rinv2);
rinv6 = _mm_mul_sd(rinv4,rinv2);
rvdw = _mm_add_sd(rai,raj);
ratio = _mm_mul_sd(rsq, gmx_mm_inv_pd( _mm_mul_pd(rvdw,rvdw)));
mask_cmp = _mm_cmple_sd(ratio,still_p5inv);
/* gmx_mm_sincos_pd() is quite expensive, so avoid calculating it if we can! */
if( 0 == _mm_movemask_pd(mask_cmp) )
{
/* if ratio>still_p5inv for ALL elements */
ccf = one;
dccf = _mm_setzero_pd();
}
else
{
ratio = _mm_min_sd(ratio,still_p5inv);
theta = _mm_mul_sd(ratio,still_pip5);
gmx_mm_sincos_pd(theta,&sinq,&cosq);
term = _mm_mul_sd(half,_mm_sub_sd(one,cosq));
ccf = _mm_mul_sd(term,term);
dccf = _mm_mul_sd(_mm_mul_sd(two,term),
_mm_mul_sd(sinq,theta));
}
prod = _mm_mul_sd(still_p4,vaj);
icf4 = _mm_mul_sd(ccf,rinv4);
icf6 = _mm_mul_sd( _mm_sub_sd( _mm_mul_sd(four,ccf),dccf), rinv6);
GMX_MM_INCREMENT_1VALUE_PD(work+jnrA,_mm_mul_sd(prod_ai,icf4));
gpi = _mm_add_sd(gpi, _mm_mul_sd(prod,icf4) );
_mm_store_pd(dadx,_mm_mul_pd(prod,icf6));
dadx+=2;
_mm_store_pd(dadx,_mm_mul_pd(prod_ai,icf6));
dadx+=2;
}
gmx_mm_update_1pot_pd(gpi,work+ii);
}
/* Sum up the polarization energy from other nodes */
if(PARTDECOMP(cr))
{
gmx_sum(natoms, work, cr);
}
else if(DOMAINDECOMP(cr))
{
dd_atom_sum_real(cr->dd, work);
}
/* Compute the radii */
for(i=0;i<fr->natoms_force;i++) /* PELA born->nr */
{
if(born->use[i] != 0)
{
gpi_ai = born->gpol[i] + work[i]; /* add gpi to the initial pol energy gpi_ai*/
gpi2 = gpi_ai * gpi_ai;
born->bRad[i] = factor*gmx_invsqrt(gpi2);
fr->invsqrta[i] = gmx_invsqrt(born->bRad[i]);
}
}
/* Extra (local) communication required for DD */
if(DOMAINDECOMP(cr))
{
dd_atom_spread_real(cr->dd, born->bRad);
dd_atom_spread_real(cr->dd, fr->invsqrta);
}
return 0;
}
__m128d test_mm_cmple_pd(__m128d __a, __m128d __b) {
// CHECK-LABEL: @test_mm_cmple_pd
// CHECK: @llvm.x86.sse2.cmp.pd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i8 2)
return _mm_cmple_pd(__a, __b);
}
{
template<class Dummy>
struct call< tag::is_less_equal_ ( tag::simd_<tag::double_,tag::sse_>
, tag::simd_<tag::double_,tag::sse_>
)
, tag::cpu_, Dummy
>
: callable
{
template<class Sig> struct result;
template<class This,class A0>
struct result<This(A0,A0)> : meta::strip<A0> {};
NT2_FUNCTOR_CALL(2)
{
A0 that = { _mm_cmple_pd(a0,a1) };
return that;
}
};
} }
////////////////////////////////////////////////////////////////////////////////
// Overloads implementation for float
////////////////////////////////////////////////////////////////////////////////
NT2_REGISTER_DISPATCH ( tag::is_less_equal_, tag::cpu_, (A0)
, ((simd_<float_<A0>,tag::sse_>))
((simd_<float_<A0>,tag::sse_>))
);
namespace nt2 { namespace ext
{
