uint32_t sse_sumbytes_variant2(uint8_t* array, size_t size) {
const __m128i lobyte_mask = _mm_set1_epi32(0x000000ff);
__m128i accumulator = _mm_setzero_si128();
for (size_t i=0; i < size; i += 16) {
const __m128i v = _mm_loadu_si128((__m128i*)(array + i));
const __m128i v0 = v;
const __m128i v1 = _mm_srli_epi32(v, 1*8);
const __m128i v2 = _mm_srli_epi32(v, 2*8);
const __m128i v3 = _mm_srli_epi32(v, 3*8);
const __m128i t0 = _mm_and_si128(lobyte_mask, v0);
const __m128i t1 = _mm_and_si128(lobyte_mask, v1);
const __m128i t2 = _mm_and_si128(lobyte_mask, v2);
const __m128i t3 = v3;
accumulator = _mm_add_epi32(accumulator, t0);
accumulator = _mm_add_epi32(accumulator, t1);
accumulator = _mm_add_epi32(accumulator, t2);
accumulator = _mm_add_epi32(accumulator, t3);
}
return uint32_t(_mm_extract_epi32(accumulator, 0)) +
uint32_t(_mm_extract_epi32(accumulator, 1)) +
uint32_t(_mm_extract_epi32(accumulator, 2)) +
uint32_t(_mm_extract_epi32(accumulator, 3));
}
uint32_t sse_sumbytes(uint8_t* array, size_t size) {
__m128i accumulator = _mm_setzero_si128();
for (size_t i=0; i < size; i += 16) {
const __m128i v = _mm_loadu_si128((__m128i*)(array + i));
const __m128i v0_3 = v;
const __m128i v4_7 = _mm_bsrli_si128(v, 1*4);
const __m128i v8_11 = _mm_bsrli_si128(v, 2*4);
const __m128i v12_15 = _mm_bsrli_si128(v, 3*4);
const __m128i t0 = _mm_cvtepu8_epi32(v0_3);
const __m128i t1 = _mm_cvtepu8_epi32(v4_7);
const __m128i t2 = _mm_cvtepu8_epi32(v8_11);
const __m128i t3 = _mm_cvtepu8_epi32(v12_15);
const __m128i t01 = _mm_add_epi32(t0, t1);
const __m128i t23 = _mm_add_epi32(t2, t3);
accumulator = _mm_add_epi32(accumulator, t01);
accumulator = _mm_add_epi32(accumulator, t23);
}
return uint32_t(_mm_extract_epi32(accumulator, 0)) +
uint32_t(_mm_extract_epi32(accumulator, 1)) +
uint32_t(_mm_extract_epi32(accumulator, 2)) +
uint32_t(_mm_extract_epi32(accumulator, 3));
}
uint32_t probe(uint32_t key)
{
/* create a vector with all values initialized to key */
__m128i keyVector = _mm_set1_epi32(key);
/* find the appropriate buckets using multiplicative hashing */
__m128i bucketIds = _mm_mullo_epi32(keyVector, hashes.vec128);
bucketIds = _mm_srli_epi32(bucketIds, hashShift);
size_t b0 = _mm_extract_epi32(bucketIds, 0);
size_t b1 = _mm_extract_epi32(bucketIds, 1);
__m128i keys;
__m128i values0, values1;
/* load keys, compare with lookup key (to produce a bitmask).
* AND the result with the corresponding values. */
keys = _mm_load_si128((const __m128i *) buckets[b0].keys);
keys = _mm_cmpeq_epi32(keys, keyVector);
values0 = _mm_load_si128((const __m128i *) buckets[b0].values);
values0 = _mm_and_si128(values0, keys);
keys = _mm_load_si128((const __m128i *) buckets[b1].keys);
keys = _mm_cmpeq_epi32(keys, keyVector);
values1 = _mm_load_si128((const __m128i *) buckets[b1].values);
values1 = _mm_and_si128(values1, keys);
/* OR all of the (key AND value) pairs to get result */
union QuadInt qi;
qi.vec128 = _mm_or_si128(values0, values1);
qi.vec64[0] = _mm_or_si64(qi.vec64[0], qi.vec64[1]);
return qi.arr[0] | qi.arr[1];
}
int32_t sse_sadbw_sumsignedbytes(int8_t* array, size_t size) {
const __m128i zero = _mm_setzero_si128();
__m128i positive = zero;
__m128i negative = zero;
for (size_t i=0; i < size; i += 16) {
const __m128i v = _mm_loadu_si128((__m128i*)(array + i));
const __m128i m = _mm_cmplt_epi8(v, zero);
const __m128i va = _mm_abs_epi8(v);
// sum just positive numbers
const __m128i t0 = _mm_sad_epu8(_mm_andnot_si128(m, v), zero);
// sum just negative numbers
const __m128i t1 = _mm_sad_epu8(_mm_and_si128(m, va), zero);
positive = _mm_add_epi32(positive, t0);
negative = _mm_sub_epi32(negative, t1);
}
const __m128i accumulator = _mm_add_epi32(positive, negative);
return int32_t(_mm_extract_epi32(accumulator, 0)) +
int32_t(_mm_extract_epi32(accumulator, 2));
}
void t_print__m128i (__m128i a)
{
printf("%d = %d\n", 0, _mm_extract_epi32(a, 0));
printf("%d = %d\n", 1, _mm_extract_epi32(a, 1));
printf("%d = %d\n", 2, _mm_extract_epi32(a, 2));
printf("%d = %d\n", 3, _mm_extract_epi32(a, 3));
printf("\n");
}
int32_t sse_sadbw_unrolled4_sumsignedbytes(int8_t* array, size_t size) {
const __m128i zero = _mm_setzero_si128();
__m128i positive = zero;
__m128i negative = zero;
for (size_t i=0; i < size; i += 16*4) {
const __m128i v0 = _mm_loadu_si128((__m128i*)(array + i + 0*16));
const __m128i v1 = _mm_loadu_si128((__m128i*)(array + i + 1*16));
const __m128i v2 = _mm_loadu_si128((__m128i*)(array + i + 2*16));
const __m128i v3 = _mm_loadu_si128((__m128i*)(array + i + 3*16));
{
const __m128i v = v0;
const __m128i m = _mm_cmplt_epi8(v, zero);
const __m128i t0 = _mm_sad_epu8(_mm_andnot_si128(m, v), zero);
const __m128i va = _mm_abs_epi8(v);
const __m128i t1 = _mm_sad_epu8(_mm_and_si128(m, va), zero);
positive = _mm_add_epi32(positive, t0);
negative = _mm_sub_epi32(negative, t1);
}
{
const __m128i v = v1;
const __m128i m = _mm_cmplt_epi8(v, zero);
const __m128i t0 = _mm_sad_epu8(_mm_andnot_si128(m, v), zero);
const __m128i va = _mm_abs_epi8(v);
const __m128i t1 = _mm_sad_epu8(_mm_and_si128(m, va), zero);
positive = _mm_add_epi32(positive, t0);
negative = _mm_sub_epi32(negative, t1);
}
{
const __m128i v = v2;
const __m128i m = _mm_cmplt_epi8(v, zero);
const __m128i t0 = _mm_sad_epu8(_mm_andnot_si128(m, v), zero);
const __m128i va = _mm_abs_epi8(v);
const __m128i t1 = _mm_sad_epu8(_mm_and_si128(m, va), zero);
positive = _mm_add_epi32(positive, t0);
negative = _mm_sub_epi32(negative, t1);
}
{
const __m128i v = v3;
const __m128i m = _mm_cmplt_epi8(v, zero);
const __m128i t0 = _mm_sad_epu8(_mm_andnot_si128(m, v), zero);
const __m128i va = _mm_abs_epi8(v);
const __m128i t1 = _mm_sad_epu8(_mm_and_si128(m, va), zero);
positive = _mm_add_epi32(positive, t0);
negative = _mm_sub_epi32(negative, t1);
}
}
const __m128i accumulator = _mm_add_epi32(positive, negative);
return int32_t(_mm_extract_epi32(accumulator, 0)) +
int32_t(_mm_extract_epi32(accumulator, 2));
}
static inline void arr_store_col(
int *col,
__m128i vH,
int32_t t,
int32_t seglen)
{
col[0*seglen+t] = (int32_t)_mm_extract_epi32(vH, 0);
col[1*seglen+t] = (int32_t)_mm_extract_epi32(vH, 1);
col[2*seglen+t] = (int32_t)_mm_extract_epi32(vH, 2);
col[3*seglen+t] = (int32_t)_mm_extract_epi32(vH, 3);
}
static inline void do_encode_6bytes(const char (*alphabet)[2], char *out, __m128i chunk)
{
uint32_t v0, v1, v2, v3;
v0 = _mm_extract_epi32(chunk, 0);
v1 = _mm_extract_epi32(chunk, 1);
v2 = _mm_extract_epi32(chunk, 2);
v3 = _mm_extract_epi32(chunk, 3);
memcpy(out + 0, alphabet[v0], 2);
memcpy(out + 2, alphabet[v1], 2);
memcpy(out + 4, alphabet[v2], 2);
memcpy(out + 6, alphabet[v3], 2);
}
static inline void arr_store_si128(
int *array,
__m128i vH,
int32_t t,
int32_t seglen,
int32_t d,
int32_t dlen)
{
array[1LL*(0*seglen+t)*dlen + d] = (int32_t)_mm_extract_epi32(vH, 0);
array[1LL*(1*seglen+t)*dlen + d] = (int32_t)_mm_extract_epi32(vH, 1);
array[1LL*(2*seglen+t)*dlen + d] = (int32_t)_mm_extract_epi32(vH, 2);
array[1LL*(3*seglen+t)*dlen + d] = (int32_t)_mm_extract_epi32(vH, 3);
}
// useful for debugging
inline bool equal128(__m128i x, __m128i y)
{
return ((_mm_extract_epi32(x,0) == _mm_extract_epi32(y,0)) &&
(_mm_extract_epi32(x,1) == _mm_extract_epi32(y,1)) &&
(_mm_extract_epi32(x,2) == _mm_extract_epi32(y,2)) &&
(_mm_extract_epi32(x,3) == _mm_extract_epi32(y,3)));
}
// credit: Harold Aptroot
uint32_t maskedvectorsum(uint32_t * z, uint32_t N, uint32_t * accesses,
uint32_t nmbr) {
__m256i Nvec = _mm256_set1_epi32(N - 1);
__m256i sum = _mm256_setzero_si256();
for(uint32_t j = 0; j < nmbr ; j += 8) {
__m256i indexes = _mm256_loadu_si256((__m256i*)(accesses + j));
indexes = _mm256_and_si256(indexes, Nvec);
__m256i fi = _mm256_i32gather_epi32((int*)z, indexes, 4);
sum = _mm256_add_epi32(sum, fi);
}
__m128i sum128 = _mm_add_epi32(_mm256_extracti128_si256(sum, 0), _mm256_extracti128_si256(sum, 1));
sum128 = _mm_hadd_epi32(sum128, sum128);
return _mm_extract_epi32(sum128, 0) + _mm_extract_epi32(sum128, 1);
}
static inline void arr_store_rowcol(
int *row,
int *col,
__m128i vWH,
int32_t i,
int32_t s1Len,
int32_t j,
int32_t s2Len)
{
if (i+0 == s1Len-1 && 0 <= j-0 && j-0 < s2Len) {
row[j-0] = (int32_t)_mm_extract_epi32(vWH, 3);
}
if (j-0 == s2Len-1 && 0 <= i+0 && i+0 < s1Len) {
col[(i+0)] = (int32_t)_mm_extract_epi32(vWH, 3);
}
if (i+1 == s1Len-1 && 0 <= j-1 && j-1 < s2Len) {
row[j-1] = (int32_t)_mm_extract_epi32(vWH, 2);
}
if (j-1 == s2Len-1 && 0 <= i+1 && i+1 < s1Len) {
col[(i+1)] = (int32_t)_mm_extract_epi32(vWH, 2);
}
if (i+2 == s1Len-1 && 0 <= j-2 && j-2 < s2Len) {
row[j-2] = (int32_t)_mm_extract_epi32(vWH, 1);
}
if (j-2 == s2Len-1 && 0 <= i+2 && i+2 < s1Len) {
col[(i+2)] = (int32_t)_mm_extract_epi32(vWH, 1);
}
if (i+3 == s1Len-1 && 0 <= j-3 && j-3 < s2Len) {
row[j-3] = (int32_t)_mm_extract_epi32(vWH, 0);
}
if (j-3 == s2Len-1 && 0 <= i+3 && i+3 < s1Len) {
col[(i+3)] = (int32_t)_mm_extract_epi32(vWH, 0);
}
}
int vector_ps_short (const short* pa,const short* pb,size_t n)
{
size_t k;
size_t q = n / 16;
size_t r = n % 16;
int w;
if (q > 0) {
__m128i acc1 = _mm_setzero_si128();
__m128i acc2 = _mm_setzero_si128();
if (ALGEBRA_IS_ALIGNED(pa) && ALGEBRA_IS_ALIGNED(pb)) {
for (k=0;k<q;k++) {
/* Charge 16 mots dans chaque tableau */
__m128i a1 = _mm_load_si128((__m128i*)pa);
__m128i b1 = _mm_load_si128((__m128i*)pb);
__m128i a2 = _mm_load_si128((__m128i*)(pa+8));
__m128i b2 = _mm_load_si128((__m128i*)(pb+8));
/* Multiple, somme et converti en double word */
__m128i s1 = _mm_madd_epi16(a1,b1);
__m128i s2 = _mm_madd_epi16(a2,b2);
pa += 16;
pb += 16;
/* Accumule */
acc1 = _mm_add_epi32(acc1,s1);
acc2 = _mm_add_epi32(acc2,s2);
}
}
else {
for (k=0;k<q;k++) {
/* Charge 16 mots dans chaque tableau */
__m128i a1 = _mm_loadu_si128((__m128i*)pa);
__m128i b1 = _mm_loadu_si128((__m128i*)pb);
__m128i a2 = _mm_loadu_si128((__m128i*)(pa+8));
__m128i b2 = _mm_loadu_si128((__m128i*)(pb+8));
/* Multiple, somme et converti en double word */
__m128i s1 = _mm_madd_epi16(a1,b1);
__m128i s2 = _mm_madd_epi16(a2,b2);
pa += 16;
pb += 16;
/* Accumule */
acc1 = _mm_add_epi32(acc1,s1);
acc2 = _mm_add_epi32(acc2,s2);
}
}
/* Somme finale */
acc1 = _mm_add_epi32(acc1,acc2);
acc1 = _mm_hadd_epi32(acc1,acc1);
acc1 = _mm_hadd_epi32(acc1,acc1);
w = _mm_extract_epi32(acc1,0);
}
else {
w = 0;
}
for (k=0;k<r;k++)
w += (*pa++) * (*pb++);
return w;
}
uint seqRank ( uint * vector , byte searchedByte , uint position ){
register uint i , cont = 0;
__m128i patt , window , returnValue ;
byte * c1 , patt_code [16];
uint d = position > >4 , r = position & 0 xf ;
for ( i =0; i <16; i ++)
patt_code [i ]= searchedByte ;
long long * pat_array = ( long long *) patt_code ;
patt = _mm_set_epi64x ( pat_array [1] , pat_array [0]) ;
long long * text_array = ( long long *) vector ;
for ( i =0; i <d; i ++) {
window = _mm_set_epi64x ( text_array [1] , text_array [0]) ;
returnValue = _mm_cmpestrm ( patt , 16 , window , 16 , mode ) ;
cont += _mm_popcnt_u32 ( _mm_extract_epi32 ( returnValue ,0) );
text_array += 2;
}
window = _mm_set_epi64x ( text_array [1] , text_array [0]) ;
returnValue = _mm_cmpestrm ( patt , r , window , r , mode );
cont += _mm_popcnt_u32 ( _mm_extract_epi32 ( returnValue ,0) ) +r -16;
return cont ;
}
static inline void arr_store_si128(
int *array,
__m128i vWH,
int32_t i,
int32_t s1Len,
int32_t j,
int32_t s2Len)
{
if (0 <= i+0 && i+0 < s1Len && 0 <= j-0 && j-0 < s2Len) {
array[1LL*(i+0)*s2Len + (j-0)] = (int32_t)_mm_extract_epi32(vWH, 3);
}
if (0 <= i+1 && i+1 < s1Len && 0 <= j-1 && j-1 < s2Len) {
array[1LL*(i+1)*s2Len + (j-1)] = (int32_t)_mm_extract_epi32(vWH, 2);
}
if (0 <= i+2 && i+2 < s1Len && 0 <= j-2 && j-2 < s2Len) {
array[1LL*(i+2)*s2Len + (j-2)] = (int32_t)_mm_extract_epi32(vWH, 1);
}
if (0 <= i+3 && i+3 < s1Len && 0 <= j-3 && j-3 < s2Len) {
array[1LL*(i+3)*s2Len + (j-3)] = (int32_t)_mm_extract_epi32(vWH, 0);
}
}
static void
sfid_render_cache_rt_write_simd8_r_uint8_ymajor(struct thread *t,
const struct sfid_render_cache_args *args)
{
const int slice_y = args->rt.minimum_array_element * args->rt.qpitch;
const int x = t->grf[1].uw[4];
const int y = t->grf[1].uw[5] + slice_y;
const int cpp = 1;
void *base = ymajor_offset(args->rt.pixels, x, y, args->rt.stride, cpp);
struct reg *src = &t->grf[args->src];
__m256i r32 = _mm256_permute4x64_epi64(src[0].ireg, SWIZZLE(0, 2, 1, 3));
__m128i lo = _mm256_extractf128_si256(r32, 0);
__m128i hi = _mm256_extractf128_si256(r32, 1);
__m128i r16 = _mm_packus_epi32(lo, hi);
__m128i r8 = _mm_packus_epi16(r16, r16);
/* FIXME: Needs masking. */
*(uint32_t *) (base + 0) = _mm_extract_epi32(r8, 0);
*(uint32_t *) (base + 16) = _mm_extract_epi32(r8, 1);
}
static INLINE int variance_final_from_32bit_sum_avx2(__m256i vsse, __m128i vsum,
unsigned int *const sse) {
// extract the low lane and add it to the high lane
const __m128i sse_reg_128 = mm256_add_hi_lo_epi32(vsse);
// unpack sse and sum registers and add
const __m128i sse_sum_lo = _mm_unpacklo_epi32(sse_reg_128, vsum);
const __m128i sse_sum_hi = _mm_unpackhi_epi32(sse_reg_128, vsum);
const __m128i sse_sum = _mm_add_epi32(sse_sum_lo, sse_sum_hi);
// perform the final summation and extract the results
const __m128i res = _mm_add_epi32(sse_sum, _mm_srli_si128(sse_sum, 8));
*((int *)sse) = _mm_cvtsi128_si32(res);
return _mm_extract_epi32(res, 1);
}
static __m128i cielabv (union hvrgbpix rgb)
{
__m128 xvxyz[2] = {_mm_set1_ps(0.5),_mm_set1_ps(0.5) }; //,0.5,0.5,0.5);
__m128 vcam0 = _mm_setr_ps(cielab_xyz_cam[0][0],cielab_xyz_cam[1][0],cielab_xyz_cam[2][0],0);
__m128 vcam1 = _mm_setr_ps(cielab_xyz_cam[0][1],cielab_xyz_cam[1][1],cielab_xyz_cam[2][1],0);
__m128 vcam2 = _mm_setr_ps(cielab_xyz_cam[0][2],cielab_xyz_cam[1][2],cielab_xyz_cam[2][2],0);
__m128 vrgb0h = _mm_set1_ps(rgb.h.c[0]);
__m128 vrgb1h = _mm_set1_ps(rgb.h.c[1]);
__m128 vrgb2h = _mm_set1_ps(rgb.h.c[2]);
__m128 vrgb0v = _mm_set1_ps(rgb.v.c[0]);
__m128 vrgb1v = _mm_set1_ps(rgb.v.c[1]);
__m128 vrgb2v = _mm_set1_ps(rgb.v.c[2]);
xvxyz[0] = _mm_add_ps(xvxyz[0], _mm_mul_ps(vcam0,vrgb0h));
xvxyz[0] = _mm_add_ps(xvxyz[0], _mm_mul_ps(vcam1,vrgb1h));
xvxyz[0] = _mm_add_ps(xvxyz[0], _mm_mul_ps(vcam2,vrgb2h));
xvxyz[1] = _mm_add_ps(xvxyz[1], _mm_mul_ps(vcam0,vrgb0v));
xvxyz[1] = _mm_add_ps(xvxyz[1], _mm_mul_ps(vcam1,vrgb1v));
xvxyz[1] = _mm_add_ps(xvxyz[1], _mm_mul_ps(vcam2,vrgb2v));
xvxyz[0] = _mm_max_ps(_mm_set1_ps(0),
_mm_min_ps(_mm_set1_ps(0xffff),
_mm_round_ps(xvxyz[0], _MM_FROUND_TO_ZERO)));
xvxyz[1] = _mm_max_ps(_mm_set1_ps(0),
_mm_min_ps(_mm_set1_ps(0xffff),
_mm_round_ps(xvxyz[1], _MM_FROUND_TO_ZERO)));
__m128i loadaddrh = _mm_cvttps_epi32(xvxyz[0]);
__m128i loadaddrv = _mm_cvttps_epi32(xvxyz[1]);
#ifdef __AVX__
__m256 vlab,
vxyz = { cielab_cbrt[_mm_extract_epi32(loadaddrh,1)],
cielab_cbrt[_mm_extract_epi32(loadaddrh,0)],
cielab_cbrt[_mm_extract_epi32(loadaddrh,1)],
0,
cielab_cbrt[_mm_extract_epi32(loadaddrv,1)],
cielab_cbrt[_mm_extract_epi32(loadaddrv,0)],
cielab_cbrt[_mm_extract_epi32(loadaddrv,1)],
0},
vxyz2 = {0,
cielab_cbrt[_mm_extract_epi32(loadaddrh,1)],
cielab_cbrt[_mm_extract_epi32(loadaddrh,2)],
cielab_cbrt[_mm_extract_epi32(loadaddrh,0)],
0,
cielab_cbrt[_mm_extract_epi32(loadaddrv,1)],
cielab_cbrt[_mm_extract_epi32(loadaddrv,2)],
cielab_cbrt[_mm_extract_epi32(loadaddrv,0)]};
vlab = _mm256_sub_ps(vxyz,vxyz2);
vlab = _mm256_mul_ps(vlab, _mm256_setr_ps(116,500,200,0,116,500,200,0));
vlab = _mm256_sub_ps(vlab, _mm256_setr_ps(16,0,0,0,16,0,0,0));
vlab = _mm256_mul_ps(vlab,_mm256_set1_ps(64));
vlab = _mm256_round_ps(vlab, _MM_FROUND_TO_ZERO);
__m256i vlabi = _mm256_cvtps_epi32(vlab);
return _mm_packs_epi32(_mm256_castsi256_si128(vlabi), ((__m128i*)&vlabi)[1]);
#else
__m128 vlabh, vxyzh = {cielab_cbrt[_mm_extract_epi32(loadaddrh,0)],
cielab_cbrt[_mm_extract_epi32(loadaddrh,1)],
cielab_cbrt[_mm_extract_epi32(loadaddrh,2)],
0};
__m128 vlabv, vxyzv = {cielab_cbrt[_mm_extract_epi32(loadaddrv,0)],
cielab_cbrt[_mm_extract_epi32(loadaddrv,1)],
cielab_cbrt[_mm_extract_epi32(loadaddrv,2)],
0};
vlabh = _mm_sub_ps(_mm_shuffle_ps(vxyzh,vxyzh,_MM_SHUFFLE(0,1,0,1)),
_mm_shuffle_ps(vxyzh,vxyzh,_MM_SHUFFLE(0,2,1,3)));
vlabh = _mm_mul_ps(vlabh,_mm_setr_ps(116,500,200,0));
vlabh = _mm_sub_ps(vlabh,_mm_setr_ps(16,0,0,0));
vlabh = _mm_mul_ps(vlabh,_mm_set_ps1(64));
vlabh = _mm_round_ps(vlabh, _MM_FROUND_TO_ZERO);
vlabv = _mm_sub_ps(_mm_shuffle_ps(vxyzv,vxyzv,_MM_SHUFFLE(0,1,0,1)),
_mm_shuffle_ps(vxyzv,vxyzv,_MM_SHUFFLE(0,2,1,3)));
vlabv = _mm_mul_ps(vlabv,_mm_setr_ps(116,500,200,0));
vlabv = _mm_sub_ps(vlabv,_mm_setr_ps(16,0,0,0));
vlabv = _mm_mul_ps(vlabv,_mm_set_ps1(64));
vlabv = _mm_round_ps(vlabv, _MM_FROUND_TO_ZERO);
return _mm_set_epi64(_mm_cvtps_pi16(vlabv),_mm_cvtps_pi16(vlabh));
#endif
}
/*****************************************************************************
* This function utilises 3 properties of the cost function lookup tables, *
* constructed in using 'cal_nmvjointsadcost' and 'cal_nmvsadcosts' in *
* vp9_encoder.c. *
* For the joint cost: *
* - mvjointsadcost[1] == mvjointsadcost[2] == mvjointsadcost[3] *
* For the component costs: *
* - For all i: mvsadcost[0][i] == mvsadcost[1][i] *
* (Equal costs for both components) *
* - For all i: mvsadcost[0][i] == mvsadcost[0][-i] *
* (Cost function is even) *
* If these do not hold, then this function cannot be used without *
* modification, in which case you can revert to using the C implementation, *
* which does not rely on these properties. *
*****************************************************************************/
int vp9_diamond_search_sad_avx(const MACROBLOCK *x,
const search_site_config *cfg,
MV *ref_mv, MV *best_mv, int search_param,
int sad_per_bit, int *num00,
const vp9_variance_fn_ptr_t *fn_ptr,
const MV *center_mv) {
const int_mv maxmv = pack_int_mv(x->mv_row_max, x->mv_col_max);
const __m128i v_max_mv_w = _mm_set1_epi32(maxmv.as_int);
const int_mv minmv = pack_int_mv(x->mv_row_min, x->mv_col_min);
const __m128i v_min_mv_w = _mm_set1_epi32(minmv.as_int);
const __m128i v_spb_d = _mm_set1_epi32(sad_per_bit);
const __m128i v_joint_cost_0_d = _mm_set1_epi32(x->nmvjointsadcost[0]);
const __m128i v_joint_cost_1_d = _mm_set1_epi32(x->nmvjointsadcost[1]);
// search_param determines the length of the initial step and hence the number
// of iterations.
// 0 = initial step (MAX_FIRST_STEP) pel
// 1 = (MAX_FIRST_STEP/2) pel,
// 2 = (MAX_FIRST_STEP/4) pel...
const MV *ss_mv = &cfg->ss_mv[cfg->searches_per_step * search_param];
const intptr_t *ss_os = &cfg->ss_os[cfg->searches_per_step * search_param];
const int tot_steps = cfg->total_steps - search_param;
const int_mv fcenter_mv = pack_int_mv(center_mv->row >> 3,
center_mv->col >> 3);
const __m128i vfcmv = _mm_set1_epi32(fcenter_mv.as_int);
const int ref_row = clamp(ref_mv->row, minmv.as_mv.row, maxmv.as_mv.row);
const int ref_col = clamp(ref_mv->col, minmv.as_mv.col, maxmv.as_mv.col);
int_mv bmv = pack_int_mv(ref_row, ref_col);
int_mv new_bmv = bmv;
__m128i v_bmv_w = _mm_set1_epi32(bmv.as_int);
const int what_stride = x->plane[0].src.stride;
const int in_what_stride = x->e_mbd.plane[0].pre[0].stride;
const uint8_t *const what = x->plane[0].src.buf;
const uint8_t *const in_what = x->e_mbd.plane[0].pre[0].buf +
ref_row * in_what_stride + ref_col;
// Work out the start point for the search
const uint8_t *best_address = in_what;
const uint8_t *new_best_address = best_address;
#if ARCH_X86_64
__m128i v_ba_q = _mm_set1_epi64x((intptr_t)best_address);
#else
__m128i v_ba_d = _mm_set1_epi32((intptr_t)best_address);
#endif
unsigned int best_sad;
int i;
int j;
int step;
// Check the prerequisite cost function properties that are easy to check
// in an assert. See the function-level documentation for details on all
// prerequisites.
assert(x->nmvjointsadcost[1] == x->nmvjointsadcost[2]);
assert(x->nmvjointsadcost[1] == x->nmvjointsadcost[3]);
// Check the starting position
best_sad = fn_ptr->sdf(what, what_stride, in_what, in_what_stride);
best_sad += mvsad_err_cost(x, bmv, &fcenter_mv.as_mv, sad_per_bit);
*num00 = 0;
for (i = 0, step = 0; step < tot_steps; step++) {
for (j = 0; j < cfg->searches_per_step; j += 4, i += 4) {
__m128i v_sad_d;
__m128i v_cost_d;
__m128i v_outside_d;
__m128i v_inside_d;
__m128i v_diff_mv_w;
#if ARCH_X86_64
__m128i v_blocka[2];
#else
__m128i v_blocka[1];
#endif
// Compute the candidate motion vectors
const __m128i v_ss_mv_w = _mm_loadu_si128((const __m128i*)&ss_mv[i]);
const __m128i v_these_mv_w = _mm_add_epi16(v_bmv_w, v_ss_mv_w);
// Clamp them to the search bounds
__m128i v_these_mv_clamp_w = v_these_mv_w;
v_these_mv_clamp_w = _mm_min_epi16(v_these_mv_clamp_w, v_max_mv_w);
v_these_mv_clamp_w = _mm_max_epi16(v_these_mv_clamp_w, v_min_mv_w);
// The ones that did not change are inside the search area
v_inside_d = _mm_cmpeq_epi32(v_these_mv_clamp_w, v_these_mv_w);
// If none of them are inside, then move on
if (__likely__(_mm_test_all_zeros(v_inside_d, v_inside_d))) {
continue;
}
// The inverse mask indicates which of the MVs are outside
v_outside_d = _mm_xor_si128(v_inside_d, _mm_set1_epi8(0xff));
// Shift right to keep the sign bit clear, we will use this later
// to set the cost to the maximum value.
v_outside_d = _mm_srli_epi32(v_outside_d, 1);
// Compute the difference MV
v_diff_mv_w = _mm_sub_epi16(v_these_mv_clamp_w, vfcmv);
// We utilise the fact that the cost function is even, and use the
// absolute difference. This allows us to use unsigned indexes later
// and reduces cache pressure somewhat as only a half of the table
// is ever referenced.
v_diff_mv_w = _mm_abs_epi16(v_diff_mv_w);
// Compute the SIMD pointer offsets.
{
#if ARCH_X86_64 // sizeof(intptr_t) == 8
// Load the offsets
__m128i v_bo10_q = _mm_loadu_si128((const __m128i*)&ss_os[i+0]);
__m128i v_bo32_q = _mm_loadu_si128((const __m128i*)&ss_os[i+2]);
// Set the ones falling outside to zero
v_bo10_q = _mm_and_si128(v_bo10_q,
_mm_cvtepi32_epi64(v_inside_d));
v_bo32_q = _mm_and_si128(v_bo32_q,
_mm_unpackhi_epi32(v_inside_d, v_inside_d));
// Compute the candidate addresses
v_blocka[0] = _mm_add_epi64(v_ba_q, v_bo10_q);
v_blocka[1] = _mm_add_epi64(v_ba_q, v_bo32_q);
#else // ARCH_X86 // sizeof(intptr_t) == 4
__m128i v_bo_d = _mm_loadu_si128((const __m128i*)&ss_os[i]);
v_bo_d = _mm_and_si128(v_bo_d, v_inside_d);
v_blocka[0] = _mm_add_epi32(v_ba_d, v_bo_d);
#endif
}
fn_ptr->sdx4df(what, what_stride,
(const uint8_t **)&v_blocka[0], in_what_stride,
(uint32_t*)&v_sad_d);
// Look up the component cost of the residual motion vector
{
const int32_t row0 = _mm_extract_epi16(v_diff_mv_w, 0);
const int32_t col0 = _mm_extract_epi16(v_diff_mv_w, 1);
const int32_t row1 = _mm_extract_epi16(v_diff_mv_w, 2);
const int32_t col1 = _mm_extract_epi16(v_diff_mv_w, 3);
const int32_t row2 = _mm_extract_epi16(v_diff_mv_w, 4);
const int32_t col2 = _mm_extract_epi16(v_diff_mv_w, 5);
const int32_t row3 = _mm_extract_epi16(v_diff_mv_w, 6);
const int32_t col3 = _mm_extract_epi16(v_diff_mv_w, 7);
// Note: This is a use case for vpgather in AVX2
const uint32_t cost0 = x->nmvsadcost[0][row0] + x->nmvsadcost[0][col0];
const uint32_t cost1 = x->nmvsadcost[0][row1] + x->nmvsadcost[0][col1];
const uint32_t cost2 = x->nmvsadcost[0][row2] + x->nmvsadcost[0][col2];
const uint32_t cost3 = x->nmvsadcost[0][row3] + x->nmvsadcost[0][col3];
__m128i v_cost_10_d, v_cost_32_d;
v_cost_10_d = _mm_cvtsi32_si128(cost0);
v_cost_10_d = _mm_insert_epi32(v_cost_10_d, cost1, 1);
v_cost_32_d = _mm_cvtsi32_si128(cost2);
v_cost_32_d = _mm_insert_epi32(v_cost_32_d, cost3, 1);
v_cost_d = _mm_unpacklo_epi64(v_cost_10_d, v_cost_32_d);
}
// Now add in the joint cost
{
const __m128i v_sel_d = _mm_cmpeq_epi32(v_diff_mv_w,
_mm_setzero_si128());
const __m128i v_joint_cost_d = _mm_blendv_epi8(v_joint_cost_1_d,
v_joint_cost_0_d,
v_sel_d);
v_cost_d = _mm_add_epi32(v_cost_d, v_joint_cost_d);
}
// Multiply by sad_per_bit
v_cost_d = _mm_mullo_epi32(v_cost_d, v_spb_d);
// ROUND_POWER_OF_TWO(v_cost_d, 8)
v_cost_d = _mm_add_epi32(v_cost_d, _mm_set1_epi32(0x80));
v_cost_d = _mm_srai_epi32(v_cost_d, 8);
// Add the cost to the sad
v_sad_d = _mm_add_epi32(v_sad_d, v_cost_d);
// Make the motion vectors outside the search area have max cost
// by or'ing in the comparison mask, this way the minimum search won't
// pick them.
v_sad_d = _mm_or_si128(v_sad_d, v_outside_d);
// Find the minimum value and index horizontally in v_sad_d
{
// Try speculatively on 16 bits, so we can use the minpos intrinsic
const __m128i v_sad_w = _mm_packus_epi32(v_sad_d, v_sad_d);
const __m128i v_minp_w = _mm_minpos_epu16(v_sad_w);
uint32_t local_best_sad = _mm_extract_epi16(v_minp_w, 0);
uint32_t local_best_idx = _mm_extract_epi16(v_minp_w, 1);
// If the local best value is not saturated, just use it, otherwise
// find the horizontal minimum again the hard way on 32 bits.
// This is executed rarely.
if (__unlikely__(local_best_sad == 0xffff)) {
__m128i v_loval_d, v_hival_d, v_loidx_d, v_hiidx_d, v_sel_d;
v_loval_d = v_sad_d;
v_loidx_d = _mm_set_epi32(3, 2, 1, 0);
v_hival_d = _mm_srli_si128(v_loval_d, 8);
v_hiidx_d = _mm_srli_si128(v_loidx_d, 8);
v_sel_d = _mm_cmplt_epi32(v_hival_d, v_loval_d);
v_loval_d = _mm_blendv_epi8(v_loval_d, v_hival_d, v_sel_d);
v_loidx_d = _mm_blendv_epi8(v_loidx_d, v_hiidx_d, v_sel_d);
v_hival_d = _mm_srli_si128(v_loval_d, 4);
v_hiidx_d = _mm_srli_si128(v_loidx_d, 4);
v_sel_d = _mm_cmplt_epi32(v_hival_d, v_loval_d);
v_loval_d = _mm_blendv_epi8(v_loval_d, v_hival_d, v_sel_d);
v_loidx_d = _mm_blendv_epi8(v_loidx_d, v_hiidx_d, v_sel_d);
local_best_sad = _mm_extract_epi32(v_loval_d, 0);
local_best_idx = _mm_extract_epi32(v_loidx_d, 0);
}
// Update the global minimum if the local minimum is smaller
if (__likely__(local_best_sad < best_sad)) {
new_bmv = ((const int_mv *)&v_these_mv_w)[local_best_idx];
new_best_address = ((const uint8_t **)v_blocka)[local_best_idx];
best_sad = local_best_sad;
}
}
}
bmv = new_bmv;
best_address = new_best_address;
v_bmv_w = _mm_set1_epi32(bmv.as_int);
#if ARCH_X86_64
v_ba_q = _mm_set1_epi64x((intptr_t)best_address);
#else
v_ba_d = _mm_set1_epi32((intptr_t)best_address);
#endif
if (__unlikely__(best_address == in_what)) {
(*num00)++;
}
}
*best_mv = bmv.as_mv;
return best_sad;
}
unsigned long long int foo32(__m128i x)
{
return (unsigned int) _mm_extract_epi32(x, 2);
}
ARGB32 SoftTexture2D::Sample_Point_Wrap( F4 u, F4 v ) const
{
//F4 fU = clampf(u,0,1);
//F4 fV = clampf(v,0,1);
F4 fWidth = SIZE_X;
F4 fHeight = SIZE_Y;
//
const int iU = iround( u * fWidth ) & (SIZE_X-1); // [0..1] -> [0..H]
const int iV = iround( v * fHeight ) & (SIZE_Y-1); // [0..1] -> [0..H]
return m_data.ToPtr()[ iV * SIZE_X + iU ];
/*
// 28.4 fixed-point coordinates
const INT32 FU = iround( 16.0f * u );
const INT32 FV = iround( 16.0f * v );
enum { FIX_POINT_PRE = 9 };
const UINT offset =
(( ( FU & t->textureYMask ) >> FIX_POINT_PRE ) << t->pitchlog2)
| (( FV & t->textureXMask ) >> FIX_POINT_PRE)
;
//*(span++) += textureBuffer[((iv>>10)&0xffffffC0) + (iu>>16)];
*/
#if 0
float4 res;
__m128i tU, tV;
__m128 conv = _mm_rcp_ps( f255 );
// Formula for computing U and V:
// tX = (int)( min(iX/iW, 1.0f) * (tex_width - 1) );
{
__m128 t2 = _mm_mul_ps( MSR_Wrap(u), _mm_set_ps1((float)(tex->clip_rect.w-1)) );
__m128 t3 = _mm_mul_ps( MSR_Wrap(v), _mm_set_ps1((float)(tex->clip_rect.h-1)) );
tU = _mm_cvtps_epi32(t2);
tV = _mm_cvtps_epi32(t3);
}
// tSample = tV * pitch + tU * bytesPerPixel
__m128i tIdx = _mm_add_epi32( mul_epi32(tU, _mm_set1_epi32(tex->format->BytesPerPixel)), mul_epi32(tV, _mm_set1_epi32(tex->pitch)));
// Since SSE doesn't support arbitrary indexing out of an array, we have to extract the indexes,
// grab the sample, and recreate an SSE register with the new samples.
Uint8 *ptr = (Uint8*&)tex->pixels;
Uint8 *sample3 = &ptr[_mm_extract_epi32(tIdx, 3)];
Uint8 *sample2 = &ptr[_mm_extract_epi32(tIdx, 2)];
Uint8 *sample1 = &ptr[_mm_extract_epi32(tIdx, 1)];
Uint8 *sample0 = &ptr[_mm_extract_epi32(tIdx, 0)];
__m128i tSample = _mm_set_epi32( *(Uint32*)sample3, *(Uint32*)sample2, *(Uint32*)sample1, *(Uint32*)sample0 );
// Finally, grab each of the channels out by shifting and masking.
res.r = _mm_cvtepi32_ps(_mm_srl_epi32( _mm_and_si128( tSample, _mm_set1_epi32(tex->format->Rmask) ), _mm_set_epi32(0, 0, 0, tex->format->Rshift) ) );
res.g = _mm_cvtepi32_ps(_mm_srl_epi32( _mm_and_si128( tSample, _mm_set1_epi32(tex->format->Gmask) ), _mm_set_epi32(0, 0, 0, tex->format->Gshift) ) );
res.b = _mm_cvtepi32_ps(_mm_srl_epi32( _mm_and_si128( tSample, _mm_set1_epi32(tex->format->Bmask) ), _mm_set_epi32(0, 0, 0, tex->format->Bshift) ) );
*res.r = _mm_mul_ps( *res.r, conv );
*res.g = _mm_mul_ps( *res.g, conv );
*res.b = _mm_mul_ps( *res.b, conv );
return res;
#endif
}
static inline int32_t _mm_hmax_epi32_rpl(__m128i a) {
a = _mm_max_epi32(a, _mm_srli_si128(a, 8));
a = _mm_max_epi32(a, _mm_srli_si128(a, 4));
return _mm_extract_epi32(a, 0);
}
void AVXFMA4DNoise(Vector3d& result, const Vector3d& EPoint)
{
DBL x, y, z;
int ix, iy, iz;
int ixiy_hash, ixjy_hash, jxiy_hash, jxjy_hash;
// TODO FIXME - global statistics reference
// Stats[Calls_To_DNoise]++;
x = EPoint[X];
y = EPoint[Y];
z = EPoint[Z];
/* its equivalent integer lattice point. */
/*ix = (int)x; iy = (int)y; iz = (int)z;
x_ix = x - ix; y_iy = y - iy; z_iz = z - iz;*/
/* JB fix for the range problem */
__m128d xy = _mm_setr_pd(x, y);
__m128d zn = _mm_set_sd(z);
__m128d epsy = _mm_set1_pd(1.0 - EPSILON);
__m128d xy_e = _mm_sub_pd(xy, epsy);
__m128d zn_e = _mm_sub_sd(zn, epsy);
__m128i tmp_xy = _mm_cvttpd_epi32(_mm_blendv_pd(xy, xy_e, xy));
__m128i tmp_zn = _mm_cvttpd_epi32(_mm_blendv_pd(zn, zn_e, zn));
__m128i noise_min_xy = _mm_setr_epi32(NOISE_MINX, NOISE_MINY, 0, 0);
__m128i noise_min_zn = _mm_set1_epi32(NOISE_MINZ);
__m128d xy_ixy = _mm_sub_pd(xy, _mm_cvtepi32_pd(tmp_xy));
__m128d zn_izn = _mm_sub_sd(zn, _mm_cvtepi32_pd(tmp_zn));
const __m128i fff = _mm_set1_epi32(0xfff);
__m128i i_xy = _mm_and_si128(_mm_sub_epi32(tmp_xy, noise_min_xy), fff);
__m128i i_zn = _mm_and_si128(_mm_sub_epi32(tmp_zn, noise_min_zn), fff);
ix = _mm_extract_epi32(i_xy, 0);
iy = _mm_extract_epi32(i_xy, 1);
iz = _mm_extract_epi32(i_zn, 0);
ixiy_hash = Hash2d(ix, iy);
jxiy_hash = Hash2d(ix + 1, iy);
ixjy_hash = Hash2d(ix, iy + 1);
jxjy_hash = Hash2d(ix + 1, iy + 1);
DBL* mp1 = &RTable[Hash1dRTableIndex(ixiy_hash, iz)];
DBL* mp2 = &RTable[Hash1dRTableIndex(jxiy_hash, iz)];
DBL* mp3 = &RTable[Hash1dRTableIndex(jxjy_hash, iz)];
DBL* mp4 = &RTable[Hash1dRTableIndex(ixjy_hash, iz)];
DBL* mp5 = &RTable[Hash1dRTableIndex(ixjy_hash, iz + 1)];
DBL* mp6 = &RTable[Hash1dRTableIndex(jxjy_hash, iz + 1)];
DBL* mp7 = &RTable[Hash1dRTableIndex(jxiy_hash, iz + 1)];
DBL* mp8 = &RTable[Hash1dRTableIndex(ixiy_hash, iz + 1)];
const __m128d three = _mm_set1_pd(3.0);
const __m128d two = _mm_set1_pd(2.0);
const __m128d one = _mm_set1_pd(1.0);
__m128d ix_mm = _mm_unpacklo_pd(xy_ixy, xy_ixy);
__m128d iy_mm = _mm_unpackhi_pd(xy_ixy, xy_ixy);
__m128d iz_mm = _mm_unpacklo_pd(zn_izn, zn_izn);
__m128d jx_mm = _mm_sub_pd(ix_mm, one);
__m128d jy_mm = _mm_sub_pd(iy_mm, one);
__m128d jz_mm = _mm_sub_pd(iz_mm, one);
__m128d mm_sz = _mm_mul_pd(_mm_mul_pd(iz_mm, iz_mm), _mm_nmacc_pd(two, iz_mm, three));
__m128d mm_tz = _mm_sub_pd(one, mm_sz);
__m128d mm_sxy = _mm_mul_pd(_mm_mul_pd(xy_ixy, xy_ixy), _mm_nmacc_pd(two, xy_ixy, three));
__m128d mm_txy = _mm_sub_pd(one, mm_sxy);
__m128d mm_tysy = _mm_unpackhi_pd(mm_txy, mm_sxy);
__m128d mm_txty_txsy = _mm_mul_pd(_mm_unpacklo_pd(mm_txy, mm_txy), mm_tysy);
__m128d mm_sxty_sxsy = _mm_mul_pd(_mm_unpacklo_pd(mm_sxy, mm_sxy), mm_tysy);
__m128d mm_txty_txsy_tz = _mm_mul_pd(mm_txty_txsy, mm_tz);
__m128d mm_txty_txsy_sz = _mm_mul_pd(mm_txty_txsy, mm_sz);
__m128d mm_sxty_sxsy_tz = _mm_mul_pd(mm_sxty_sxsy, mm_tz);
__m128d mm_sxty_sxsy_sz = _mm_mul_pd(mm_sxty_sxsy, mm_sz);
__m128d mp_t1, mp_t2, mp1_mm, mp2_mm, mp4_mm, mp6_mm, sum_p;
__m128d sum_X_Y = _mm_setzero_pd();
__m128d sum__Z = _mm_setzero_pd();
__m128d mm_s1 = _mm_unpacklo_pd(mm_txty_txsy_tz, mm_txty_txsy_tz);
INCRSUMP2(mp1, mp1 + 8, mm_s1, ix_mm, iy_mm, iz_mm, sum_X_Y);
__m128d mm_s2 = _mm_unpacklo_pd(mm_sxty_sxsy_tz, mm_sxty_sxsy_tz);
INCRSUMP2(mp2, mp2 + 8, mm_s2, jx_mm, iy_mm, iz_mm, sum_X_Y);
__m128d mm_s3 = _mm_unpackhi_pd(mm_sxty_sxsy_tz, mm_sxty_sxsy_tz);
INCRSUMP2(mp3, mp3 + 8, mm_s3, jx_mm, jy_mm, iz_mm, sum_X_Y);
__m128d mm_s4 = _mm_unpackhi_pd(mm_txty_txsy_tz, mm_txty_txsy_tz);
INCRSUMP2(mp4, mp4 + 8, mm_s4, ix_mm, jy_mm, iz_mm, sum_X_Y);
__m128d mm_s5 = _mm_unpackhi_pd(mm_txty_txsy_sz, mm_txty_txsy_sz);
INCRSUMP2(mp5, mp5 + 8, mm_s5, ix_mm, jy_mm, jz_mm, sum_X_Y);
__m128d mm_s6 = _mm_unpackhi_pd(mm_sxty_sxsy_sz, mm_sxty_sxsy_sz);
INCRSUMP2(mp6, mp6 + 8, mm_s6, jx_mm, jy_mm, jz_mm, sum_X_Y);
__m128d mm_s7 = _mm_unpacklo_pd(mm_sxty_sxsy_sz, mm_sxty_sxsy_sz);
INCRSUMP2(mp7, mp7 + 8, mm_s7, jx_mm, iy_mm, jz_mm, sum_X_Y);
__m128d mm_s8 = _mm_unpacklo_pd(mm_txty_txsy_sz, mm_txty_txsy_sz);
INCRSUMP2(mp8, mp8 + 8, mm_s8, ix_mm, iy_mm, jz_mm, sum_X_Y);
__m128d iy_jy = _mm_unpacklo_pd(iy_mm, jy_mm);
INCRSUMP2(mp1 + 16, mp4 + 16, mm_txty_txsy_tz, ix_mm, iy_jy, iz_mm, sum__Z);
INCRSUMP2(mp8 + 16, mp5 + 16, mm_txty_txsy_sz, ix_mm, iy_jy, jz_mm, sum__Z);
INCRSUMP2(mp2 + 16, mp3 + 16, mm_sxty_sxsy_tz, jx_mm, iy_jy, iz_mm, sum__Z);
INCRSUMP2(mp7 + 16, mp6 + 16, mm_sxty_sxsy_sz, jx_mm, iy_jy, jz_mm, sum__Z);
sum__Z = _mm_hadd_pd(sum__Z, sum__Z);
_mm_storeu_pd(*result, sum_X_Y);
_mm_store_sd(&result[Z], sum__Z);
}
DBL AVXFMA4Noise(const Vector3d& EPoint, int noise_generator)
{
DBL x, y, z;
DBL *mp;
int ix, iy, iz;
int ixiy_hash, ixjy_hash, jxiy_hash, jxjy_hash;
DBL sum;
// TODO FIXME - global statistics reference
// Stats[Calls_To_Noise]++;
if (noise_generator==kNoiseGen_Perlin)
{
// The 1.59 and 0.985 are to correct for some biasing problems with
// the random # generator used to create the noise tables. Final
// range of values is about 5.0e-4 below 0.0 and above 1.0. Mean
// value is 0.49 (ideally it would be 0.5).
sum = 0.5 * (1.59 * SolidNoise(EPoint) + 0.985);
// Clamp final value to 0-1 range
if (sum < 0.0) sum = 0.0;
if (sum > 1.0) sum = 1.0;
return sum;
}
x = EPoint[X];
y = EPoint[Y];
z = EPoint[Z];
/* its equivalent integer lattice point. */
/* ix = (int)x; iy = (int)y; iz = (long)z; */
/* JB fix for the range problem */
__m128d xy = _mm_setr_pd(x, y);
__m128d zn = _mm_set_sd(z);
__m128d epsy = _mm_set1_pd(1.0 - EPSILON);
__m128d xy_e = _mm_sub_pd(xy, epsy);
__m128d zn_e = _mm_sub_sd(zn, epsy);
__m128i tmp_xy = _mm_cvttpd_epi32(_mm_blendv_pd(xy, xy_e, xy));
__m128i tmp_zn = _mm_cvttpd_epi32(_mm_blendv_pd(zn, zn_e, zn));
__m128i noise_min_xy = _mm_setr_epi32(NOISE_MINX, NOISE_MINY, 0, 0);
__m128i noise_min_zn = _mm_set1_epi32(NOISE_MINZ);
__m128d xy_ixy = _mm_sub_pd(xy, _mm_cvtepi32_pd(tmp_xy));
__m128d zn_izn = _mm_sub_sd(zn, _mm_cvtepi32_pd(tmp_zn));
const __m128i fff = _mm_set1_epi32(0xfff);
__m128i i_xy = _mm_and_si128(_mm_sub_epi32(tmp_xy, noise_min_xy), fff);
__m128i i_zn = _mm_and_si128(_mm_sub_epi32(tmp_zn, noise_min_zn), fff);
ix = _mm_extract_epi32(i_xy, 0);
iy = _mm_extract_epi32(i_xy, 1);
iz = _mm_extract_epi32(i_zn, 0);
ixiy_hash = Hash2d(ix, iy);
jxiy_hash = Hash2d(ix + 1, iy);
ixjy_hash = Hash2d(ix, iy + 1);
jxjy_hash = Hash2d(ix + 1, iy + 1);
mp = &RTable[Hash1dRTableIndex(ixiy_hash, iz)];
DBL *mp2 = &RTable[Hash1dRTableIndex(ixjy_hash, iz)];
DBL *mp3 = &RTable[Hash1dRTableIndex(ixiy_hash, iz + 1)];
DBL *mp4 = &RTable[Hash1dRTableIndex(ixjy_hash, iz + 1)];
DBL *mp5 = &RTable[Hash1dRTableIndex(jxiy_hash, iz)];
DBL *mp6 = &RTable[Hash1dRTableIndex(jxjy_hash, iz)];
DBL *mp7 = &RTable[Hash1dRTableIndex(jxiy_hash, iz + 1)];
DBL *mp8 = &RTable[Hash1dRTableIndex(jxjy_hash, iz + 1)];
const __m128d three = _mm_set1_pd(3.0);
const __m128d two = _mm_set1_pd(2.0);
const __m128d one = _mm_set1_pd(1.0);
__m128d ix_mm = _mm_unpacklo_pd(xy_ixy, xy_ixy);
__m128d iy_mm = _mm_unpackhi_pd(xy_ixy, xy_ixy);
__m128d iz_mm = _mm_unpacklo_pd(zn_izn, zn_izn);
__m128d jx_mm = _mm_sub_pd(ix_mm, one);
__m128d jy_mm = _mm_sub_pd(iy_mm, one);
__m128d jz_mm = _mm_sub_pd(iz_mm, one);
__m128d mm_sxy = _mm_mul_pd(_mm_mul_pd(xy_ixy, xy_ixy), _mm_nmacc_pd(two, xy_ixy, three));
__m128d mm_sz = _mm_mul_pd(_mm_mul_pd(iz_mm, iz_mm), _mm_nmacc_pd(two, iz_mm, three));
__m128d mm_tz = _mm_sub_pd(one, mm_sz);
__m128d mm_txy = _mm_sub_pd(one, mm_sxy);
__m128d mm_tysy = _mm_unpackhi_pd(mm_txy, mm_sxy);
__m128d mm_txty_txsy = _mm_mul_pd(_mm_unpacklo_pd(mm_txy, mm_txy), mm_tysy);
__m128d mm_sxty_sxsy = _mm_mul_pd(_mm_unpacklo_pd(mm_sxy, mm_sxy), mm_tysy);
__m128d y_mm = _mm_unpacklo_pd(iy_mm, jy_mm);
__m128d mp_t1, mp_t2, mp1_mm, mp2_mm, mp4_mm, mp6_mm, sum_p, s_mm;
__m128d int_sum1 = _mm_setzero_pd();
s_mm = _mm_mul_pd(mm_txty_txsy, mm_tz);
INCRSUMP2(mp, mp2, s_mm, ix_mm, y_mm, iz_mm, int_sum1);
s_mm = _mm_mul_pd(mm_txty_txsy, mm_sz);
INCRSUMP2(mp3, mp4, s_mm, ix_mm, y_mm, jz_mm, int_sum1);
s_mm = _mm_mul_pd(mm_sxty_sxsy, mm_tz);
INCRSUMP2(mp5, mp6, s_mm, jx_mm, y_mm, iz_mm, int_sum1);
s_mm = _mm_mul_pd(mm_sxty_sxsy, mm_sz);
INCRSUMP2(mp7, mp8, s_mm, jx_mm, y_mm, jz_mm, int_sum1);
int_sum1 = _mm_hadd_pd(int_sum1, int_sum1);
if(noise_generator==kNoiseGen_RangeCorrected)
{
/* details of range here:
Min, max: -1.05242, 0.988997
Mean: -0.0191481, Median: -0.535493, Std Dev: 0.256828
We want to change it to as close to [0,1] as possible.
*/
const __m128d r2 = _mm_set_sd(0.48985582);
const __m128d r1r2 = _mm_set_sd(1.05242*0.48985582);
int_sum1 = _mm_macc_sd(int_sum1, r2, r1r2);
}
else
{
int_sum1 = _mm_add_sd(int_sum1, _mm_set_sd(0.5));
}
int_sum1 = _mm_min_sd(one, int_sum1);
int_sum1 = _mm_max_sd(_mm_setzero_pd(), int_sum1);
_mm_store_sd(&sum, int_sum1);
return (sum);
}
long long int foo32(__m128i x) { return (int) _mm_extract_epi32(x, 2); }
int check(size_t N, size_t Nq) {
int * queries = (int*)malloc(Nq*sizeof(int));
int * source = (int*)malloc(N*sizeof(int));
size_t i, k;
int displaytest = 0;
for(i = 0; i < N; ++i) {
source[i] = rand();
}
qsort (source, N, sizeof(int), compare);
if(displaytest) {
for(i = 0; i < N; ++i) {
printf(" %d ",source[i]);
}
printf("\n");
}
int maxval = source[N-1];
for(i = 0; i < Nq; ++i) {
queries[i] = rand()%(maxval+1);
}
for(k = 0; k < Nq; ++k)
if(branchy_search(source,N,queries[k]) != branchfree_search(source,N,queries[k])) {
printf("bug1\n");
free(source);
free(queries);
return -1;
}
for(k = 0; k+1 < Nq; k+=2) {
size_t i1, i2;
branchfree_search2(source,N,queries[k],queries[k+1],&i1,&i2);
if(branchfree_search(source,N,queries[k]) != i1) {
printf("bug2\n");
free(source);
free(queries);
return -1;
}
if(branchfree_search(source,N,queries[k+1]) != i2) {
printf("bug3\n");
free(source);
free(queries);
return -1;
}
}
#ifdef MYAVX
for(k = 0; k+3 < Nq; k+=4) {
size_t i1, i2, i3, i4;
__m128i q = _mm_lddqu_si128((__m128i const*)(queries +k));
__m128i bog = branchfree_search4_avx(source,N,q);
branchfree_search4(source,N,queries[k],queries[k+1],queries[k+2],queries[k+3],&i1,&i2,&i3,&i4);
if((_mm_extract_epi32(bog,0)!= i1) || (_mm_extract_epi32(bog,1)!= i2) || (_mm_extract_epi32(bog,2)!= i3) || (_mm_extract_epi32(bog,3)!= i4)) {
printf("bug3\n");
printf("%zu %zu %zu %zu\n",i1,i2,i3,i4);
printf("%d %d %d %d\n",_mm_extract_epi32(bog,0),_mm_extract_epi32(bog,1),_mm_extract_epi32(bog,2),_mm_extract_epi32(bog,3));
return -1;
}
}
#endif
free(source);
free(queries);
return 0;
}
int demo(size_t N, size_t Nq) {
int * queries = (int*)malloc(Nq*sizeof(int));
int * source = (int*)malloc(N*sizeof(int));
size_t bogus = 0;
size_t bogus1 = 0;
size_t bogus2 = 0;
size_t bogus3 = 0;
size_t bogus4 = 0;
__m128i bog = _mm_setzero_si128();
size_t i, k, ti;
printf("===============\n");
printf("array size (N)=%zu, number of queries (Nq)=%zu...\n",N,Nq);
printf("preparing data...\n");
for(i = 0; i < N; ++i) {
source[i] = rand();
}
qsort (source, N, sizeof(int), compare);
int maxval = source[N-1];
for(i = 0; i < Nq; ++i) {
queries[i] = rand()%(maxval+1);
}
printf("beginning tests...\n");
printf("\n");
for(ti = 0; ti < 3; ++ti) {
struct timeval t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, t12, t13;
gettimeofday(&t6, 0);
for(k = 0; k+1 < Nq; k+=2)
branchfree_search2_prefetch(source,N,queries[k],queries[k+1],&bogus1,&bogus2);
gettimeofday(&t1, 0);
for(k = 0; k < Nq; ++k)
bogus += branchfree_search(source,N,queries[k]);
gettimeofday(&t2, 0);
for(k = 0; k < Nq; ++k)
bogus += branchy_search(source,N,queries[k]);
gettimeofday(&t3, 0);
for(k = 0; k < Nq; ++k)
bogus += branchfree_search_prefetch(source,N,queries[k]);
gettimeofday(&t4, 0);
for(k = 0; k+1 < Nq; k+=2)
branchfree_search2(source,N,queries[k],queries[k+1],&bogus1,&bogus2);
gettimeofday(&t5, 0);
for(k = 0; k+3 < Nq; k+=4)
branchfree_search4(source,N,queries[k],queries[k+1],queries[k+2],queries[k+3],&bogus1,&bogus2,&bogus3,&bogus4);
gettimeofday(&t7, 0);
for(k = 0; k+3 < Nq; k+=4)
branchfree_search4_prefetch(source,N,queries[k],queries[k+1],queries[k+2],queries[k+3],&bogus1,&bogus2,&bogus3,&bogus4);
gettimeofday(&t8, 0);
#ifdef MYAVX
for(k = 0; k+3 < Nq; k+=4) {
__m128i q = _mm_lddqu_si128((__m128i const*)(queries +k));
bog = _mm_add_epi32(bog,branchfree_search4_avx(source,N,q));
}
gettimeofday(&t9, 0);
for(k = 0; k+7 < Nq; k+=8) {
__m256i q = _mm256_lddqu_si256((__m256i const*)(queries +k));
bog = _mm_add_epi32(bog,_mm256_castsi256_si128(branchfree_search8_avx(source,N,q)));
}
#endif
gettimeofday(&t10, 0);
for(k = 0; k+7 < Nq; k+=8) {
branchfree_search8(source,N,queries[k],queries[k+1],queries[k+2],queries[k+3],queries[k+4],queries[k+5],queries[k+6],queries[k+7],&bogus1,&bogus2,&bogus3,&bogus4,&bogus1,&bogus2,&bogus3,&bogus4);
}
gettimeofday(&t11, 0);
for(k = 0; k+7 < Nq; k+=8) {
branchfree_search8_prefetch(source,N,queries[k],queries[k+1],queries[k+2],queries[k+3],queries[k+4],queries[k+5],queries[k+6],queries[k+7],&bogus1,&bogus2,&bogus3,&bogus4,&bogus1,&bogus2,&bogus3,&bogus4);
}
gettimeofday(&t12, 0);
for(k = 0; k < Nq; ++k)
bogus += hackedbranchfree_search(source,N,queries[k]);
gettimeofday(&t13, 0);
printf("branchless time=%llu \n",t2.tv_sec * 1000ULL * 1000ULL + t2.tv_usec - (t1.tv_sec * 1000ULL * 1000ULL + t1.tv_usec));
printf("branchy time=%llu \n",t3.tv_sec * 1000ULL * 1000ULL + t3.tv_usec - (t2.tv_sec * 1000ULL * 1000ULL + t2.tv_usec));
printf("branchless time with prefetch=%llu \n",t4.tv_sec * 1000ULL * 1000ULL + t4.tv_usec - (t3.tv_sec * 1000ULL * 1000ULL + t3.tv_usec));
printf("branchless interleaved (2) time=%llu \n",t5.tv_sec * 1000ULL * 1000ULL + t5.tv_usec - (t4.tv_sec * 1000ULL * 1000ULL + t4.tv_usec));
printf("branchless interleaved (2) (prefetch) time=%llu \n",t1.tv_sec * 1000ULL * 1000ULL + t1.tv_usec - (t6.tv_sec * 1000ULL * 1000ULL + t6.tv_usec));
printf("branchless interleaved (4) time=%llu \n",t7.tv_sec * 1000ULL * 1000ULL + t7.tv_usec - (t5.tv_sec * 1000ULL * 1000ULL + t5.tv_usec));
printf("branchless interleaved (4) (prefetch) time=%llu \n",t8.tv_sec * 1000ULL * 1000ULL + t8.tv_usec - (t7.tv_sec * 1000ULL * 1000ULL + t7.tv_usec));
#ifdef MYAVX
printf("branchless interleaved (4) (AVX) time=%llu \n",t9.tv_sec * 1000ULL * 1000ULL + t9.tv_usec - (t8.tv_sec * 1000ULL * 1000ULL + t8.tv_usec));
printf("branchless interleaved (8) (AVX) time=%llu \n",t10.tv_sec * 1000ULL * 1000ULL + t10.tv_usec - (t9.tv_sec * 1000ULL * 1000ULL + t9.tv_usec));
#endif
printf("branchless interleaved (8) time=%llu \n",t11.tv_sec * 1000ULL * 1000ULL + t11.tv_usec - (t10.tv_sec * 1000ULL * 1000ULL + t10.tv_usec));
printf("branchless interleaved (8) (prefetch) time=%llu \n",t12.tv_sec * 1000ULL * 1000ULL + t12.tv_usec - (t11.tv_sec * 1000ULL * 1000ULL + t11.tv_usec));
printf("hacked branchless time=%llu \n",t13.tv_sec * 1000ULL * 1000ULL + t13.tv_usec - (t12.tv_sec * 1000ULL * 1000ULL + t12.tv_usec));
printf("\n");
}
#ifdef MYAVX
bogus += _mm_extract_epi32(bog,0);
#endif
free(source);
free(queries);
return (int) bogus+bogus1+bogus2+bogus3+bogus4;
}
void print(__m128i bog) {
printf("%u %u %u %u \n",_mm_extract_epi32(bog,0),_mm_extract_epi32(bog,1),_mm_extract_epi32(bog,2),_mm_extract_epi32(bog,3));
}
float vector_cos_short (const short* pa,const short* pb,size_t n)
{
size_t k;
double norm;
size_t q = n / 16;
size_t r = n % 16;
int ps,na,nb;
if (q > 0) {
__m128i acc;
__m128i acc_ps1 = _mm_setzero_si128();
__m128i acc_ps2 = _mm_setzero_si128();
__m128i acc_na1 = _mm_setzero_si128();
__m128i acc_na2 = _mm_setzero_si128();
__m128i acc_nb1 = _mm_setzero_si128();
__m128i acc_nb2 = _mm_setzero_si128();
if (ALGEBRA_IS_ALIGNED(pa) && ALGEBRA_IS_ALIGNED(pb)) {
for (k=0;k<q;k++) {
/* Charge 16 mots dans chaque tableau */
__m128i a1 = _mm_load_si128((__m128i*)pa);
__m128i b1 = _mm_load_si128((__m128i*)pb);
__m128i a2 = _mm_load_si128((__m128i*)(pa+8));
__m128i b2 = _mm_load_si128((__m128i*)(pb+8));
/* Multiple, somme et converti en double word */
__m128i ps1 = _mm_madd_epi16(a1,b1);
__m128i ps2 = _mm_madd_epi16(a2,b2);
__m128i na1 = _mm_madd_epi16(a1,a1);
__m128i na2 = _mm_madd_epi16(a2,a2);
__m128i nb1 = _mm_madd_epi16(b1,b1);
__m128i nb2 = _mm_madd_epi16(b2,b2);
pa += 16;
pb += 16;
/* Accumule */
acc_ps1 = _mm_add_epi32(acc_ps1,ps1);
acc_ps2 = _mm_add_epi32(acc_ps2,ps2);
acc_na1 = _mm_add_epi32(acc_na1,na1);
acc_na2 = _mm_add_epi32(acc_na2,na2);
acc_nb1 = _mm_add_epi32(acc_nb1,nb1);
acc_nb2 = _mm_add_epi32(acc_nb2,nb2);
}
}
else {
for (k=0;k<q;k++) {
}
}
/* Somme finale */
acc = _mm_add_epi32(acc_ps1,acc_ps2);
acc = _mm_hadd_epi32(acc,acc);
acc = _mm_hadd_epi32(acc,acc);
ps = _mm_extract_epi32(acc,0);
acc = _mm_add_epi32(acc_na1,acc_na2);
acc = _mm_hadd_epi32(acc,acc);
acc = _mm_hadd_epi32(acc,acc);
na = _mm_extract_epi32(acc,0);
acc = _mm_add_epi32(acc_nb1,acc_nb2);
acc = _mm_hadd_epi32(acc,acc);
acc = _mm_hadd_epi32(acc,acc);
nb = _mm_extract_epi32(acc,0);
}
else {
ps = 0;
na = 0;
nb = 0;
}
for (k=0;k<r;k++) {
int a = *pa++;
int b = *pb++;
ps += a*b;
na += a*a;
nb += b*b;
}
norm = sqrt( ((double)na) * ((double)nb) );
if (norm < 1E-5f)
return 0;
return ps / norm;
}
void ahd_interpolate_tile(int top, char * buffer)
{
int row, col, tr, tc, c, val;
const int dir[4] = { -1, 1, -width, width };
__m128i ldiff[2], abdiff[2];
union hvrgbpix (*rgb)[width] = (union hvrgbpix (*)[width])buffer;
union hvrgbpix *rix;
union rgbpix * pix;
union hvrgbpix (*lab)[width];
short (*lix)[8];
char (*h**o)[width][2];
lab = (union hvrgbpix (*)[width])(buffer + 16*width*TS);
h**o = (char (*)[width][2])(buffer + 32*width*TS);
const int left=2;
if ((uintptr_t)(image+top*width)&0xf || (uintptr_t)buffer&0xf) {
fprintf(stderr, "unaligned buffers defeat speed!\n"); abort();
}
/* Interpolate gren horz&vert, red and blue, and convert to CIELab: */
//do the first two rows of green first.
//then one green, and rgb through the tile.. this because R/B needs down-right green value
for (row=top; row < top+2 && row < height-2; row++) {
col = left + (FC(row,left) & 1);
for (c = FC(row,col); col < width-2; col+=2) {
pix = (union rgbpix*)image + row*width+col;
val = ((pix[-1].g + pix[0].c[c] + pix[1].g) * 2 - pix[-2].c[c] - pix[2].c[c]) >> 2;
rgb[row-top][col-left].h.g = ULIM(val,pix[-1].g,pix[1].g);
val = ((pix[-width].g + pix[0].c[c] + pix[width].g) * 2 - pix[-2*width].c[c] - pix[2*width].c[c]) >> 2;
rgb[row-top][col-left].v.g = ULIM(val,pix[-width].g,pix[width].g);
}
}
for (; row < top+TS && row < height-2; row++) {
int rowx = row-1;
if (FC(rowx,left+1)==1) {
int c1 = FC(rowx+1,left+1),
c2 = FC(rowx,left+2);
pix = (union rgbpix*)image + row*width+left+1;
rix = &rgb[row-top][1];
val = ((pix[-1].g + pix[0].c[c1] + pix[1].g) * 2 - pix[-2].c[c1] - pix[2].c[c1]) >> 2;
rix[0].h.g = ULIM(val,pix[-1].g,pix[1].g);
val = ((pix[-width].g + pix[0].c[c1] + pix[width].g) * 2 - pix[-2*width].c[c1] - pix[2*width].c[c1]) >> 2;
rix[0].v.g = ULIM(val,pix[-width].g,pix[width].g);
for (col=left+1; col < width-3; col+=2) {
pix = (union rgbpix*)image + rowx*width+col+1;
union hvrgbpix rixr, rix0;
rix = &rgb[rowx-top][col-left]+1;
signed pix_diag = pix[-width-1].c[c1] + pix[-width+1].c[c1];
signed pix_ul = pix[-width-1].c[c1];
rixr.vec = _mm_set1_epi16(pix[-1].g);
signed pix_lr = pix[-2].c[c2] + pix[0].c[c2];
rix0.h.c[c2] = rix0.v.c[c2] = pix[0].c[c2];
pix_diag += pix[width-1].c[c1] + pix[width+1].c[c1] + 1;
signed pix_dl = pix[width-1].c[c1];
//fully loaded
__m128i rix_dr = _mm_setr_epi32(pix[width].g, pix[width-1].c[c1], pix[1].g, pix[-width+1].c[c1]);
rix_dr = _mm_add_epi32(rix_dr,_mm_setr_epi32(pix[width+1].c[c1], pix[width+3].c[c1], pix[width+1].c[c1], 0));
rix_dr = _mm_add_epi32(rix_dr,_mm_setr_epi32(pix[width+2].g, 0, pix[2*width+1].g, pix[3*width+1].c[c1]));
rix_dr = _mm_mullo_epi32(rix_dr,_mm_setr_epi32(2,1,2,1));
//half loaded
rix_dr = _mm_hsub_epi32(rix_dr,_mm_setzero_si128());
rix_dr = _mm_srai_epi32(rix_dr,2);
__m128i a = _mm_setr_epi32(pix[width].g,pix[1].g,0,0);
__m128i b = _mm_setr_epi32(pix[width+2].g,pix[2*width+1].g,0,0);
__m128i m = _mm_min_epi32(a,b);
__m128i M = _mm_max_epi32(a,b);
rix_dr = _mm_min_epi32(rix_dr,M);
rix_dr = _mm_max_epi32(rix_dr,m);
signed pix_udr = pix_ul + pix_dl;
signed rix0_ul = rix[-width-1].h.g;
signed rix1_ul = rix[-width-1].v.g;
__m128i rix_ur = _mm_setr_epi32(rix[-width+1].h.g, rix[-width+1].v.g, 0, 0);
signed rix0_rr = rix[-2].h.g;
signed rix1_rr = rix[-2].v.g;
rix0.h.g = rix[0].h.g;
rix0.v.g = rix[0].v.g;
signed rix0_dl = rix[width-1].h.g;
signed rix1_dl = rix[width-1].v.g;
// fully loaded
__m128i rix_udr = _mm_setr_epi32(rix0_ul, rix1_ul, rix0_rr, rix1_rr);
rix_udr = _mm_add_epi32(rix_udr, _mm_setr_epi32(rix0_dl, rix1_dl, rix0.h.g, rix0.v.g));
__m128i v2 = _mm_set_epi32(pix_lr, pix_lr, pix_udr, pix_udr);
v2 = _mm_sub_epi32(v2, rix_udr);
v2 = _mm_srai_epi32(v2,1);
v2 = _mm_add_epi32(v2,_mm_cvtepu16_epi32(rixr.vec));
v2 = _mm_max_epi32(v2, _mm_setzero_si128());
v2 = _mm_min_epi32(v2, _mm_set1_epi32(0xffff));
rixr.h.c[c2] = _mm_extract_epi32(v2,2);
rixr.v.c[c2] = _mm_extract_epi32(v2,3);
rixr.h.c[c1] = _mm_extract_epi32(v2,0);
rixr.v.c[c1] = _mm_extract_epi32(v2,1);
// following only uses 64 bit
__m128i v1 = _mm_set1_epi32(pix_diag);
v1 = _mm_sub_epi32(v1, rix_ur);
v1 = _mm_sub_epi32(v1, rix_dr);
v1 = _mm_sub_epi32(v1, rix_udr);
v1 = _mm_srai_epi32(v1,2);
v1 = _mm_add_epi32(v1, _mm_setr_epi32(rix0.h.g, rix0.v.g, 0, 0));
v1 = _mm_max_epi32(v1, _mm_setzero_si128());
v1 = _mm_min_epi32(v1, _mm_set1_epi32(0xffff));
rix0.h.c[c1] = _mm_extract_epi32(v1,0);
rix0.v.c[c1] = _mm_extract_epi32(v1,1);
lab[rowx-top][col-left].vec = cielabv(rixr);
lab[rowx-top][col-left+1].vec = cielabv(rix0);
_mm_store_si128(&rix[-1].vec,rixr.vec);
_mm_store_si128(&rix[0].vec,rix0.vec);
rix[width+1].h.g = _mm_extract_epi32(rix_dr,0);
rix[width+1].v.g = _mm_extract_epi32(rix_dr,1);
}
} else {