static INLINE unsigned int highbd_masked_sad16xh_avx2(
const uint8_t *src8, int src_stride, const uint8_t *a8, int a_stride,
const uint8_t *b8, int b_stride, const uint8_t *m_ptr, int m_stride,
int width, int height) {
const uint16_t *src_ptr = CONVERT_TO_SHORTPTR(src8);
const uint16_t *a_ptr = CONVERT_TO_SHORTPTR(a8);
const uint16_t *b_ptr = CONVERT_TO_SHORTPTR(b8);
int x, y;
__m256i res = _mm256_setzero_si256();
const __m256i mask_max = _mm256_set1_epi16((1 << AOM_BLEND_A64_ROUND_BITS));
const __m256i round_const =
_mm256_set1_epi32((1 << AOM_BLEND_A64_ROUND_BITS) >> 1);
const __m256i one = _mm256_set1_epi16(1);
for (y = 0; y < height; y++) {
for (x = 0; x < width; x += 16) {
const __m256i src = _mm256_lddqu_si256((const __m256i *)&src_ptr[x]);
const __m256i a = _mm256_lddqu_si256((const __m256i *)&a_ptr[x]);
const __m256i b = _mm256_lddqu_si256((const __m256i *)&b_ptr[x]);
// Zero-extend mask to 16 bits
const __m256i m =
_mm256_cvtepu8_epi16(_mm_lddqu_si128((const __m128i *)&m_ptr[x]));
const __m256i m_inv = _mm256_sub_epi16(mask_max, m);
const __m256i data_l = _mm256_unpacklo_epi16(a, b);
const __m256i mask_l = _mm256_unpacklo_epi16(m, m_inv);
__m256i pred_l = _mm256_madd_epi16(data_l, mask_l);
pred_l = _mm256_srai_epi32(_mm256_add_epi32(pred_l, round_const),
AOM_BLEND_A64_ROUND_BITS);
const __m256i data_r = _mm256_unpackhi_epi16(a, b);
const __m256i mask_r = _mm256_unpackhi_epi16(m, m_inv);
__m256i pred_r = _mm256_madd_epi16(data_r, mask_r);
pred_r = _mm256_srai_epi32(_mm256_add_epi32(pred_r, round_const),
AOM_BLEND_A64_ROUND_BITS);
// Note: the maximum value in pred_l/r is (2^bd)-1 < 2^15,
// so it is safe to do signed saturation here.
const __m256i pred = _mm256_packs_epi32(pred_l, pred_r);
// There is no 16-bit SAD instruction, so we have to synthesize
// an 8-element SAD. We do this by storing 4 32-bit partial SADs,
// and accumulating them at the end
const __m256i diff = _mm256_abs_epi16(_mm256_sub_epi16(pred, src));
res = _mm256_add_epi32(res, _mm256_madd_epi16(diff, one));
}
src_ptr += src_stride;
a_ptr += a_stride;
b_ptr += b_stride;
m_ptr += m_stride;
}
// At this point, we have four 32-bit partial SADs stored in 'res'.
res = _mm256_hadd_epi32(res, res);
res = _mm256_hadd_epi32(res, res);
int sad = _mm256_extract_epi32(res, 0) + _mm256_extract_epi32(res, 4);
return (sad + 31) >> 6;
}
static INLINE unsigned int masked_sad32xh_avx2(
const uint8_t *src_ptr, int src_stride, const uint8_t *a_ptr, int a_stride,
const uint8_t *b_ptr, int b_stride, const uint8_t *m_ptr, int m_stride,
int width, int height) {
int x, y;
__m256i res = _mm256_setzero_si256();
const __m256i mask_max = _mm256_set1_epi8((1 << AOM_BLEND_A64_ROUND_BITS));
const __m256i round_scale =
_mm256_set1_epi16(1 << (15 - AOM_BLEND_A64_ROUND_BITS));
for (y = 0; y < height; y++) {
for (x = 0; x < width; x += 32) {
const __m256i src = _mm256_lddqu_si256((const __m256i *)&src_ptr[x]);
const __m256i a = _mm256_lddqu_si256((const __m256i *)&a_ptr[x]);
const __m256i b = _mm256_lddqu_si256((const __m256i *)&b_ptr[x]);
const __m256i m = _mm256_lddqu_si256((const __m256i *)&m_ptr[x]);
const __m256i m_inv = _mm256_sub_epi8(mask_max, m);
// Calculate 16 predicted pixels.
// Note that the maximum value of any entry of 'pred_l' or 'pred_r'
// is 64 * 255, so we have plenty of space to add rounding constants.
const __m256i data_l = _mm256_unpacklo_epi8(a, b);
const __m256i mask_l = _mm256_unpacklo_epi8(m, m_inv);
__m256i pred_l = _mm256_maddubs_epi16(data_l, mask_l);
pred_l = _mm256_mulhrs_epi16(pred_l, round_scale);
const __m256i data_r = _mm256_unpackhi_epi8(a, b);
const __m256i mask_r = _mm256_unpackhi_epi8(m, m_inv);
__m256i pred_r = _mm256_maddubs_epi16(data_r, mask_r);
pred_r = _mm256_mulhrs_epi16(pred_r, round_scale);
const __m256i pred = _mm256_packus_epi16(pred_l, pred_r);
res = _mm256_add_epi32(res, _mm256_sad_epu8(pred, src));
}
src_ptr += src_stride;
a_ptr += a_stride;
b_ptr += b_stride;
m_ptr += m_stride;
}
// At this point, we have two 32-bit partial SADs in lanes 0 and 2 of 'res'.
res = _mm256_shuffle_epi32(res, 0xd8);
res = _mm256_permute4x64_epi64(res, 0xd8);
res = _mm256_hadd_epi32(res, res);
res = _mm256_hadd_epi32(res, res);
int32_t sad = _mm256_extract_epi32(res, 0);
return (sad + 31) >> 6;
}
void aom_comp_mask_pred_avx2(uint8_t *comp_pred, const uint8_t *pred, int width,
int height, const uint8_t *ref, int ref_stride,
const uint8_t *mask, int mask_stride,
int invert_mask) {
int i = 0;
const uint8_t *src0 = invert_mask ? pred : ref;
const uint8_t *src1 = invert_mask ? ref : pred;
const int stride0 = invert_mask ? width : ref_stride;
const int stride1 = invert_mask ? ref_stride : width;
if (width == 8) {
comp_mask_pred_8_ssse3(comp_pred, height, src0, stride0, src1, stride1,
mask, mask_stride);
} else if (width == 16) {
do {
const __m256i sA0 = mm256_loadu2(src0 + stride0, src0);
const __m256i sA1 = mm256_loadu2(src1 + stride1, src1);
const __m256i aA = mm256_loadu2(mask + mask_stride, mask);
src0 += (stride0 << 1);
src1 += (stride1 << 1);
mask += (mask_stride << 1);
const __m256i sB0 = mm256_loadu2(src0 + stride0, src0);
const __m256i sB1 = mm256_loadu2(src1 + stride1, src1);
const __m256i aB = mm256_loadu2(mask + mask_stride, mask);
src0 += (stride0 << 1);
src1 += (stride1 << 1);
mask += (mask_stride << 1);
// comp_pred's stride == width == 16
comp_mask_pred_line_avx2(sA0, sA1, aA, comp_pred);
comp_mask_pred_line_avx2(sB0, sB1, aB, comp_pred + 32);
comp_pred += (16 << 2);
i += 4;
} while (i < height);
} else { // for width == 32
do {
const __m256i sA0 = _mm256_lddqu_si256((const __m256i *)(src0));
const __m256i sA1 = _mm256_lddqu_si256((const __m256i *)(src1));
const __m256i aA = _mm256_lddqu_si256((const __m256i *)(mask));
const __m256i sB0 = _mm256_lddqu_si256((const __m256i *)(src0 + stride0));
const __m256i sB1 = _mm256_lddqu_si256((const __m256i *)(src1 + stride1));
const __m256i aB =
_mm256_lddqu_si256((const __m256i *)(mask + mask_stride));
comp_mask_pred_line_avx2(sA0, sA1, aA, comp_pred);
comp_mask_pred_line_avx2(sB0, sB1, aB, comp_pred + 32);
comp_pred += (32 << 1);
src0 += (stride0 << 1);
src1 += (stride1 << 1);
mask += (mask_stride << 1);
i += 2;
} while (i < height);
}
}
int demo(size_t N, size_t Nq) {
int * queries = (int*)malloc(Nq*sizeof(int));
int * source = (int*)malloc(N*sizeof(int));
size_t bogus = 0;
size_t bogus1 = 0;
size_t bogus2 = 0;
size_t bogus3 = 0;
size_t bogus4 = 0;
__m128i bog = _mm_setzero_si128();
size_t i, k, ti;
printf("===============\n");
printf("array size (N)=%zu, number of queries (Nq)=%zu...\n",N,Nq);
printf("preparing data...\n");
for(i = 0; i < N; ++i) {
source[i] = rand();
}
qsort (source, N, sizeof(int), compare);
int maxval = source[N-1];
for(i = 0; i < Nq; ++i) {
queries[i] = rand()%(maxval+1);
}
printf("beginning tests...\n");
printf("\n");
for(ti = 0; ti < 3; ++ti) {
struct timeval t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, t12, t13;
gettimeofday(&t6, 0);
for(k = 0; k+1 < Nq; k+=2)
branchfree_search2_prefetch(source,N,queries[k],queries[k+1],&bogus1,&bogus2);
gettimeofday(&t1, 0);
for(k = 0; k < Nq; ++k)
bogus += branchfree_search(source,N,queries[k]);
gettimeofday(&t2, 0);
for(k = 0; k < Nq; ++k)
bogus += branchy_search(source,N,queries[k]);
gettimeofday(&t3, 0);
for(k = 0; k < Nq; ++k)
bogus += branchfree_search_prefetch(source,N,queries[k]);
gettimeofday(&t4, 0);
for(k = 0; k+1 < Nq; k+=2)
branchfree_search2(source,N,queries[k],queries[k+1],&bogus1,&bogus2);
gettimeofday(&t5, 0);
for(k = 0; k+3 < Nq; k+=4)
branchfree_search4(source,N,queries[k],queries[k+1],queries[k+2],queries[k+3],&bogus1,&bogus2,&bogus3,&bogus4);
gettimeofday(&t7, 0);
for(k = 0; k+3 < Nq; k+=4)
branchfree_search4_prefetch(source,N,queries[k],queries[k+1],queries[k+2],queries[k+3],&bogus1,&bogus2,&bogus3,&bogus4);
gettimeofday(&t8, 0);
#ifdef MYAVX
for(k = 0; k+3 < Nq; k+=4) {
__m128i q = _mm_lddqu_si128((__m128i const*)(queries +k));
bog = _mm_add_epi32(bog,branchfree_search4_avx(source,N,q));
}
gettimeofday(&t9, 0);
for(k = 0; k+7 < Nq; k+=8) {
__m256i q = _mm256_lddqu_si256((__m256i const*)(queries +k));
bog = _mm_add_epi32(bog,_mm256_castsi256_si128(branchfree_search8_avx(source,N,q)));
}
#endif
gettimeofday(&t10, 0);
for(k = 0; k+7 < Nq; k+=8) {
branchfree_search8(source,N,queries[k],queries[k+1],queries[k+2],queries[k+3],queries[k+4],queries[k+5],queries[k+6],queries[k+7],&bogus1,&bogus2,&bogus3,&bogus4,&bogus1,&bogus2,&bogus3,&bogus4);
}
gettimeofday(&t11, 0);
for(k = 0; k+7 < Nq; k+=8) {
branchfree_search8_prefetch(source,N,queries[k],queries[k+1],queries[k+2],queries[k+3],queries[k+4],queries[k+5],queries[k+6],queries[k+7],&bogus1,&bogus2,&bogus3,&bogus4,&bogus1,&bogus2,&bogus3,&bogus4);
}
gettimeofday(&t12, 0);
for(k = 0; k < Nq; ++k)
bogus += hackedbranchfree_search(source,N,queries[k]);
gettimeofday(&t13, 0);
printf("branchless time=%llu \n",t2.tv_sec * 1000ULL * 1000ULL + t2.tv_usec - (t1.tv_sec * 1000ULL * 1000ULL + t1.tv_usec));
printf("branchy time=%llu \n",t3.tv_sec * 1000ULL * 1000ULL + t3.tv_usec - (t2.tv_sec * 1000ULL * 1000ULL + t2.tv_usec));
printf("branchless time with prefetch=%llu \n",t4.tv_sec * 1000ULL * 1000ULL + t4.tv_usec - (t3.tv_sec * 1000ULL * 1000ULL + t3.tv_usec));
printf("branchless interleaved (2) time=%llu \n",t5.tv_sec * 1000ULL * 1000ULL + t5.tv_usec - (t4.tv_sec * 1000ULL * 1000ULL + t4.tv_usec));
printf("branchless interleaved (2) (prefetch) time=%llu \n",t1.tv_sec * 1000ULL * 1000ULL + t1.tv_usec - (t6.tv_sec * 1000ULL * 1000ULL + t6.tv_usec));
printf("branchless interleaved (4) time=%llu \n",t7.tv_sec * 1000ULL * 1000ULL + t7.tv_usec - (t5.tv_sec * 1000ULL * 1000ULL + t5.tv_usec));
printf("branchless interleaved (4) (prefetch) time=%llu \n",t8.tv_sec * 1000ULL * 1000ULL + t8.tv_usec - (t7.tv_sec * 1000ULL * 1000ULL + t7.tv_usec));
#ifdef MYAVX
printf("branchless interleaved (4) (AVX) time=%llu \n",t9.tv_sec * 1000ULL * 1000ULL + t9.tv_usec - (t8.tv_sec * 1000ULL * 1000ULL + t8.tv_usec));
printf("branchless interleaved (8) (AVX) time=%llu \n",t10.tv_sec * 1000ULL * 1000ULL + t10.tv_usec - (t9.tv_sec * 1000ULL * 1000ULL + t9.tv_usec));
#endif
printf("branchless interleaved (8) time=%llu \n",t11.tv_sec * 1000ULL * 1000ULL + t11.tv_usec - (t10.tv_sec * 1000ULL * 1000ULL + t10.tv_usec));
printf("branchless interleaved (8) (prefetch) time=%llu \n",t12.tv_sec * 1000ULL * 1000ULL + t12.tv_usec - (t11.tv_sec * 1000ULL * 1000ULL + t11.tv_usec));
printf("hacked branchless time=%llu \n",t13.tv_sec * 1000ULL * 1000ULL + t13.tv_usec - (t12.tv_sec * 1000ULL * 1000ULL + t12.tv_usec));
printf("\n");
}
#ifdef MYAVX
bogus += _mm_extract_epi32(bog,0);
#endif
free(source);
free(queries);
return (int) bogus+bogus1+bogus2+bogus3+bogus4;
}
