INLINE static __m256i diff_row_dual_avx2(const kvz_pixel *buf1, const kvz_pixel *buf2, const kvz_pixel *orig)
{
__m128i temp1 = _mm_loadl_epi64((__m128i*)buf1);
__m128i temp2 = _mm_loadl_epi64((__m128i*)buf2);
__m128i temp3 = _mm_loadl_epi64((__m128i*)orig);
__m256i buf1_row = _mm256_cvtepu8_epi16(_mm_unpacklo_epi64(temp1, temp2));
__m256i buf2_row = _mm256_cvtepu8_epi16(_mm_broadcastq_epi64(temp3));
return _mm256_sub_epi16(buf1_row, buf2_row);
}
static void satd_8bit_4x4_dual_avx2(
const pred_buffer preds, const kvz_pixel * const orig, unsigned num_modes, unsigned *satds_out)
{
__m256i original = _mm256_broadcastsi128_si256(_mm_cvtepu8_epi16(_mm_loadl_epi64((__m128i*)orig)));
__m256i pred = _mm256_cvtepu8_epi16(_mm_loadl_epi64((__m128i*)preds[0]));
pred = _mm256_inserti128_si256(pred, _mm_cvtepu8_epi16(_mm_loadl_epi64((__m128i*)preds[1])), 1);
__m256i diff_lo = _mm256_sub_epi16(pred, original);
original = _mm256_broadcastsi128_si256(_mm_cvtepu8_epi16(_mm_loadl_epi64((__m128i*)(orig + 8))));
pred = _mm256_cvtepu8_epi16(_mm_loadl_epi64((__m128i*)(preds[0] + 8)));
pred = _mm256_inserti128_si256(pred, _mm_cvtepu8_epi16(_mm_loadl_epi64((__m128i*)(preds[1] + 8))), 1);
__m256i diff_hi = _mm256_sub_epi16(pred, original);
//Hor
__m256i row0 = _mm256_hadd_epi16(diff_lo, diff_hi);
__m256i row1 = _mm256_hsub_epi16(diff_lo, diff_hi);
__m256i row2 = _mm256_hadd_epi16(row0, row1);
__m256i row3 = _mm256_hsub_epi16(row0, row1);
//Ver
row0 = _mm256_hadd_epi16(row2, row3);
row1 = _mm256_hsub_epi16(row2, row3);
row2 = _mm256_hadd_epi16(row0, row1);
row3 = _mm256_hsub_epi16(row0, row1);
//Abs and sum
row2 = _mm256_abs_epi16(row2);
row3 = _mm256_abs_epi16(row3);
row3 = _mm256_add_epi16(row2, row3);
row3 = _mm256_add_epi16(row3, _mm256_shuffle_epi32(row3, KVZ_PERMUTE(2, 3, 0, 1) ));
row3 = _mm256_add_epi16(row3, _mm256_shuffle_epi32(row3, KVZ_PERMUTE(1, 0, 1, 0) ));
row3 = _mm256_add_epi16(row3, _mm256_shufflelo_epi16(row3, KVZ_PERMUTE(1, 0, 1, 0) ));
unsigned sum1 = _mm_extract_epi16(_mm256_castsi256_si128(row3), 0);
sum1 = (sum1 + 1) >> 1;
unsigned sum2 = _mm_extract_epi16(_mm256_extracti128_si256(row3, 1), 0);
sum2 = (sum2 + 1) >> 1;
satds_out[0] = sum1;
satds_out[1] = sum2;
}
static INLINE unsigned int highbd_masked_sad16xh_avx2(
const uint8_t *src8, int src_stride, const uint8_t *a8, int a_stride,
const uint8_t *b8, int b_stride, const uint8_t *m_ptr, int m_stride,
int width, int height) {
const uint16_t *src_ptr = CONVERT_TO_SHORTPTR(src8);
const uint16_t *a_ptr = CONVERT_TO_SHORTPTR(a8);
const uint16_t *b_ptr = CONVERT_TO_SHORTPTR(b8);
int x, y;
__m256i res = _mm256_setzero_si256();
const __m256i mask_max = _mm256_set1_epi16((1 << AOM_BLEND_A64_ROUND_BITS));
const __m256i round_const =
_mm256_set1_epi32((1 << AOM_BLEND_A64_ROUND_BITS) >> 1);
const __m256i one = _mm256_set1_epi16(1);
for (y = 0; y < height; y++) {
for (x = 0; x < width; x += 16) {
const __m256i src = _mm256_lddqu_si256((const __m256i *)&src_ptr[x]);
const __m256i a = _mm256_lddqu_si256((const __m256i *)&a_ptr[x]);
const __m256i b = _mm256_lddqu_si256((const __m256i *)&b_ptr[x]);
// Zero-extend mask to 16 bits
const __m256i m =
_mm256_cvtepu8_epi16(_mm_lddqu_si128((const __m128i *)&m_ptr[x]));
const __m256i m_inv = _mm256_sub_epi16(mask_max, m);
const __m256i data_l = _mm256_unpacklo_epi16(a, b);
const __m256i mask_l = _mm256_unpacklo_epi16(m, m_inv);
__m256i pred_l = _mm256_madd_epi16(data_l, mask_l);
pred_l = _mm256_srai_epi32(_mm256_add_epi32(pred_l, round_const),
AOM_BLEND_A64_ROUND_BITS);
const __m256i data_r = _mm256_unpackhi_epi16(a, b);
const __m256i mask_r = _mm256_unpackhi_epi16(m, m_inv);
__m256i pred_r = _mm256_madd_epi16(data_r, mask_r);
pred_r = _mm256_srai_epi32(_mm256_add_epi32(pred_r, round_const),
AOM_BLEND_A64_ROUND_BITS);
// Note: the maximum value in pred_l/r is (2^bd)-1 < 2^15,
// so it is safe to do signed saturation here.
const __m256i pred = _mm256_packs_epi32(pred_l, pred_r);
// There is no 16-bit SAD instruction, so we have to synthesize
// an 8-element SAD. We do this by storing 4 32-bit partial SADs,
// and accumulating them at the end
const __m256i diff = _mm256_abs_epi16(_mm256_sub_epi16(pred, src));
res = _mm256_add_epi32(res, _mm256_madd_epi16(diff, one));
}
src_ptr += src_stride;
a_ptr += a_stride;
b_ptr += b_stride;
m_ptr += m_stride;
}
// At this point, we have four 32-bit partial SADs stored in 'res'.
res = _mm256_hadd_epi32(res, res);
res = _mm256_hadd_epi32(res, res);
int sad = _mm256_extract_epi32(res, 0) + _mm256_extract_epi32(res, 4);
return (sad + 31) >> 6;
}
static void
avx2_test (void)
{
union128i_b s;
union256i_w res;
short res_ref[16];
s.x = _mm_set_epi8 (1, 2, 3, 4, 200, 50, 6, 8, 1, 2, 3, 4, 200, 5, 6, 8);
res.x = _mm256_cvtepu8_epi16 (s.x);
compute_movzxbw (s.a, res_ref);
if (check_union256i_w (res, res_ref))
abort ();
}
template <bool align> SIMD_INLINE __m256i LoadUnpacked(const void * src)
{
return _mm256_cvtepu8_epi16(LoadHalf<align>((const __m128i*)src));
}
__m256i test_mm256_cvtepu8_epi16(__m128i a) {
// CHECK: @llvm.x86.avx2.pmovzxbw
return _mm256_cvtepu8_epi16(a);
}
__m256i test_mm256_cvtepu8_epi16(__m128i a) {
// CHECK-LABEL: test_mm256_cvtepu8_epi16
// CHECK: zext <16 x i8> %{{.*}} to <16 x i16>
return _mm256_cvtepu8_epi16(a);
}
void aom_highbd_comp_mask_pred_avx2(uint8_t *comp_pred8, const uint8_t *pred8,
int width, int height, const uint8_t *ref8,
int ref_stride, const uint8_t *mask,
int mask_stride, int invert_mask) {
int i = 0;
uint16_t *pred = CONVERT_TO_SHORTPTR(pred8);
uint16_t *ref = CONVERT_TO_SHORTPTR(ref8);
uint16_t *comp_pred = CONVERT_TO_SHORTPTR(comp_pred8);
const uint16_t *src0 = invert_mask ? pred : ref;
const uint16_t *src1 = invert_mask ? ref : pred;
const int stride0 = invert_mask ? width : ref_stride;
const int stride1 = invert_mask ? ref_stride : width;
const __m256i zero = _mm256_setzero_si256();
if (width == 8) {
do {
const __m256i s0 = mm256_loadu2_16(src0 + stride0, src0);
const __m256i s1 = mm256_loadu2_16(src1 + stride1, src1);
const __m128i m_l = _mm_loadl_epi64((const __m128i *)mask);
const __m128i m_h = _mm_loadl_epi64((const __m128i *)(mask + 8));
__m256i m = _mm256_castsi128_si256(m_l);
m = _mm256_insertf128_si256(m, m_h, 1);
const __m256i m_16 = _mm256_unpacklo_epi8(m, zero);
const __m256i comp = highbd_comp_mask_pred_line_avx2(s0, s1, m_16);
_mm_storeu_si128((__m128i *)(comp_pred), _mm256_castsi256_si128(comp));
_mm_storeu_si128((__m128i *)(comp_pred + width),
_mm256_extractf128_si256(comp, 1));
src0 += (stride0 << 1);
src1 += (stride1 << 1);
mask += (mask_stride << 1);
comp_pred += (width << 1);
i += 2;
} while (i < height);
} else if (width == 16) {
do {
const __m256i s0 = _mm256_loadu_si256((const __m256i *)(src0));
const __m256i s1 = _mm256_loadu_si256((const __m256i *)(src1));
const __m256i m_16 =
_mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i *)mask));
const __m256i comp = highbd_comp_mask_pred_line_avx2(s0, s1, m_16);
_mm256_storeu_si256((__m256i *)comp_pred, comp);
src0 += stride0;
src1 += stride1;
mask += mask_stride;
comp_pred += width;
i += 1;
} while (i < height);
} else if (width == 32) {
do {
const __m256i s0 = _mm256_loadu_si256((const __m256i *)src0);
const __m256i s2 = _mm256_loadu_si256((const __m256i *)(src0 + 16));
const __m256i s1 = _mm256_loadu_si256((const __m256i *)src1);
const __m256i s3 = _mm256_loadu_si256((const __m256i *)(src1 + 16));
const __m256i m01_16 =
_mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i *)mask));
const __m256i m23_16 =
_mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i *)(mask + 16)));
const __m256i comp = highbd_comp_mask_pred_line_avx2(s0, s1, m01_16);
const __m256i comp1 = highbd_comp_mask_pred_line_avx2(s2, s3, m23_16);
_mm256_storeu_si256((__m256i *)comp_pred, comp);
_mm256_storeu_si256((__m256i *)(comp_pred + 16), comp1);
src0 += stride0;
src1 += stride1;
mask += mask_stride;
comp_pred += width;
i += 1;
} while (i < height);
}
}
void av1_build_compound_diffwtd_mask_avx2(uint8_t *mask,
DIFFWTD_MASK_TYPE mask_type,
const uint8_t *src0, int stride0,
const uint8_t *src1, int stride1,
int h, int w) {
const int mb = (mask_type == DIFFWTD_38_INV) ? AOM_BLEND_A64_MAX_ALPHA : 0;
const __m256i y_mask_base = _mm256_set1_epi16(38 - mb);
int i = 0;
if (4 == w) {
do {
const __m128i s0A = xx_loadl_32(src0);
const __m128i s0B = xx_loadl_32(src0 + stride0);
const __m128i s0C = xx_loadl_32(src0 + stride0 * 2);
const __m128i s0D = xx_loadl_32(src0 + stride0 * 3);
const __m128i s0AB = _mm_unpacklo_epi32(s0A, s0B);
const __m128i s0CD = _mm_unpacklo_epi32(s0C, s0D);
const __m128i s0ABCD = _mm_unpacklo_epi64(s0AB, s0CD);
const __m256i s0ABCD_w = _mm256_cvtepu8_epi16(s0ABCD);
const __m128i s1A = xx_loadl_32(src1);
const __m128i s1B = xx_loadl_32(src1 + stride1);
const __m128i s1C = xx_loadl_32(src1 + stride1 * 2);
const __m128i s1D = xx_loadl_32(src1 + stride1 * 3);
const __m128i s1AB = _mm_unpacklo_epi32(s1A, s1B);
const __m128i s1CD = _mm_unpacklo_epi32(s1C, s1D);
const __m128i s1ABCD = _mm_unpacklo_epi64(s1AB, s1CD);
const __m256i s1ABCD_w = _mm256_cvtepu8_epi16(s1ABCD);
const __m256i m16 = calc_mask_avx2(y_mask_base, s0ABCD_w, s1ABCD_w);
const __m256i m8 = _mm256_packus_epi16(m16, _mm256_setzero_si256());
const __m128i x_m8 =
_mm256_castsi256_si128(_mm256_permute4x64_epi64(m8, 0xd8));
xx_storeu_128(mask, x_m8);
src0 += (stride0 << 2);
src1 += (stride1 << 2);
mask += 16;
i += 4;
} while (i < h);
} else if (8 == w) {
do {
const __m128i s0A = xx_loadl_64(src0);
const __m128i s0B = xx_loadl_64(src0 + stride0);
const __m128i s0C = xx_loadl_64(src0 + stride0 * 2);
const __m128i s0D = xx_loadl_64(src0 + stride0 * 3);
const __m256i s0AC_w = _mm256_cvtepu8_epi16(_mm_unpacklo_epi64(s0A, s0C));
const __m256i s0BD_w = _mm256_cvtepu8_epi16(_mm_unpacklo_epi64(s0B, s0D));
const __m128i s1A = xx_loadl_64(src1);
const __m128i s1B = xx_loadl_64(src1 + stride1);
const __m128i s1C = xx_loadl_64(src1 + stride1 * 2);
const __m128i s1D = xx_loadl_64(src1 + stride1 * 3);
const __m256i s1AB_w = _mm256_cvtepu8_epi16(_mm_unpacklo_epi64(s1A, s1C));
const __m256i s1CD_w = _mm256_cvtepu8_epi16(_mm_unpacklo_epi64(s1B, s1D));
const __m256i m16AC = calc_mask_avx2(y_mask_base, s0AC_w, s1AB_w);
const __m256i m16BD = calc_mask_avx2(y_mask_base, s0BD_w, s1CD_w);
const __m256i m8 = _mm256_packus_epi16(m16AC, m16BD);
yy_storeu_256(mask, m8);
src0 += stride0 << 2;
src1 += stride1 << 2;
mask += 32;
i += 4;
} while (i < h);
} else if (16 == w) {
do {
const __m128i s0A = xx_load_128(src0);
const __m128i s0B = xx_load_128(src0 + stride0);
const __m128i s1A = xx_load_128(src1);
const __m128i s1B = xx_load_128(src1 + stride1);
const __m256i s0AL = _mm256_cvtepu8_epi16(s0A);
const __m256i s0BL = _mm256_cvtepu8_epi16(s0B);
const __m256i s1AL = _mm256_cvtepu8_epi16(s1A);
const __m256i s1BL = _mm256_cvtepu8_epi16(s1B);
const __m256i m16AL = calc_mask_avx2(y_mask_base, s0AL, s1AL);
const __m256i m16BL = calc_mask_avx2(y_mask_base, s0BL, s1BL);
const __m256i m8 =
_mm256_permute4x64_epi64(_mm256_packus_epi16(m16AL, m16BL), 0xd8);
yy_storeu_256(mask, m8);
src0 += stride0 << 1;
src1 += stride1 << 1;
mask += 32;
i += 2;
} while (i < h);
} else {
do {
int j = 0;
do {
const __m256i s0 = yy_loadu_256(src0 + j);
const __m256i s1 = yy_loadu_256(src1 + j);
const __m256i s0L = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(s0));
const __m256i s1L = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(s1));
const __m256i s0H =
_mm256_cvtepu8_epi16(_mm256_extracti128_si256(s0, 1));
const __m256i s1H =
_mm256_cvtepu8_epi16(_mm256_extracti128_si256(s1, 1));
const __m256i m16L = calc_mask_avx2(y_mask_base, s0L, s1L);
const __m256i m16H = calc_mask_avx2(y_mask_base, s0H, s1H);
const __m256i m8 =
_mm256_permute4x64_epi64(_mm256_packus_epi16(m16L, m16H), 0xd8);
yy_storeu_256(mask + j, m8);
j += 32;
} while (j < w);
src0 += stride0;
src1 += stride1;
mask += w;
i += 1;
} while (i < h);
}
}